Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

chore(lint): clear all 146 outstanding clippy + dead-code lints

xerj-org42e784c423c1
diff --git a/engine/crates/xerj-ai/src/chunker.rs b/engine/crates/xerj-ai/src/chunker.rs
index 37d93a96a..acb67ad13 100644
--- a/engine/crates/xerj-ai/src/chunker.rs
+++ b/engine/crates/xerj-ai/src/chunker.rs
@@ -55,8 +55,14 @@ pub struct TextChunker {
impl TextChunker {
pub fn new(chunk_size: usize, overlap: usize) -> Self {
- assert!(overlap < chunk_size, "overlap must be smaller than chunk_size");
- Self { chunk_size, overlap }
+ assert!(
+ overlap < chunk_size,
+ "overlap must be smaller than chunk_size"
+ );
+ Self {
+ chunk_size,
+ overlap,
+ }
}
/// Split `text` into chunks.
diff --git a/engine/crates/xerj-ai/src/embed.rs b/engine/crates/xerj-ai/src/embed.rs
index 8f1f9d31d..7fc23bb66 100644
--- a/engine/crates/xerj-ai/src/embed.rs
+++ b/engine/crates/xerj-ai/src/embed.rs
@@ -41,9 +41,15 @@ pub struct EmbeddingProxyConfig {
pub max_retries: u32,
}
-fn default_timeout_secs() -> u64 { 30 }
-fn default_max_concurrent() -> usize { 4 }
-fn default_max_retries() -> u32 { 3 }
+fn default_timeout_secs() -> u64 {
+ 30
+}
+fn default_max_concurrent() -> usize {
+ 4
+}
+fn default_max_retries() -> u32 {
+ 3
+}
impl EmbeddingProxyConfig {
pub fn new(endpoint: impl Into<String>, model: impl Into<String>) -> Self {
@@ -109,14 +115,19 @@ impl EmbeddingProxy {
let semaphore = Arc::new(Semaphore::new(config.max_concurrent));
- Ok(Self { config, client, semaphore })
+ Ok(Self {
+ config,
+ client,
+ semaphore,
+ })
}
/// Embed a batch of texts using the configured model.
///
/// Returns one embedding vector per input text, in the same order.
pub async fn embed_batch(&self, texts: Vec<String>) -> Result<Vec<Vec<f32>>> {
- self.embed_batch_with_model(texts, &self.config.model.clone()).await
+ self.embed_batch_with_model(texts, &self.config.model.clone())
+ .await
}
/// Embed texts using a specific model (overrides the config default).
@@ -163,7 +174,10 @@ impl EmbeddingProxy {
}
async fn send_once(&self, texts: &[String], model: &str) -> Result<Vec<Vec<f32>>> {
- let body = EmbedRequest { input: texts, model };
+ let body = EmbedRequest {
+ input: texts,
+ model,
+ };
let mut req = self
.client
@@ -227,7 +241,8 @@ mod tests {
#[test]
fn config_defaults() {
- let cfg = EmbeddingProxyConfig::new("http://localhost/v1/embeddings", "text-embedding-3-small");
+ let cfg =
+ EmbeddingProxyConfig::new("http://localhost/v1/embeddings", "text-embedding-3-small");
assert_eq!(cfg.timeout_secs, 30);
assert_eq!(cfg.max_concurrent, 4);
assert_eq!(cfg.max_retries, 3);
@@ -236,8 +251,7 @@ mod tests {
#[test]
fn config_with_api_key() {
- let cfg = EmbeddingProxyConfig::new("http://localhost", "model")
- .with_api_key("sk-test");
+ let cfg = EmbeddingProxyConfig::new("http://localhost", "model").with_api_key("sk-test");
assert_eq!(cfg.api_key.as_deref(), Some("sk-test"));
}
diff --git a/engine/crates/xerj-ai/src/local.rs b/engine/crates/xerj-ai/src/local.rs
index 2c1ee2745..9ac1836a9 100644
--- a/engine/crates/xerj-ai/src/local.rs
+++ b/engine/crates/xerj-ai/src/local.rs
@@ -67,7 +67,10 @@ pub fn local_embed(text: &str, dims: usize) -> Vec<f32> {
}
let lower = text.to_lowercase();
- for token in lower.split(|c: char| !c.is_alphanumeric()).filter(|t| !t.is_empty()) {
+ for token in lower
+ .split(|c: char| !c.is_alphanumeric())
+ .filter(|t| !t.is_empty())
+ {
// Whole-word unigram — the dominant lexical signal.
add_feature(&mut acc, token.as_bytes(), 1.0);
@@ -145,7 +148,10 @@ mod tests {
// on for kNN retrieval.
let dims = DEFAULT_DIMS;
let doc_relevant = local_embed("a hungry dog chased the ball across the green park", dims);
- let doc_unrelated = local_embed("investors sold technology shares amid rising interest rates", dims);
+ let doc_unrelated = local_embed(
+ "investors sold technology shares amid rising interest rates",
+ dims,
+ );
let query = local_embed("a dog ran after a ball in the park", dims);
let sim_relevant = dot(&query, &doc_relevant);
diff --git a/engine/crates/xerj-ai/src/memory.rs b/engine/crates/xerj-ai/src/memory.rs
index 27cd30c83..2f26e0dca 100644
--- a/engine/crates/xerj-ai/src/memory.rs
+++ b/engine/crates/xerj-ai/src/memory.rs
@@ -65,7 +65,11 @@ fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
nb += bi * bi;
}
let denom = (na * nb).sqrt();
- if denom == 0.0 { 0.0 } else { dot / denom }
+ if denom == 0.0 {
+ 0.0
+ } else {
+ dot / denom
+ }
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -179,9 +183,7 @@ impl AgentMemory {
.collect();
// Sort descending by score
- scored.sort_unstable_by(|a, b| {
- b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal)
- });
+ scored.sort_unstable_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
scored.into_iter().take(k).map(|(_, e)| e.clone()).collect()
}
@@ -216,11 +218,19 @@ mod tests {
use super::*;
use serde_json::json;
- fn vec_a() -> Vec<f32> { vec![1.0, 0.0, 0.0, 0.0] }
- fn vec_b() -> Vec<f32> { vec![0.0, 1.0, 0.0, 0.0] }
- fn vec_c() -> Vec<f32> { vec![0.0, 0.0, 1.0, 0.0] }
+ fn vec_a() -> Vec<f32> {
+ vec![1.0, 0.0, 0.0, 0.0]
+ }
+ fn vec_b() -> Vec<f32> {
+ vec![0.0, 1.0, 0.0, 0.0]
+ }
+ fn vec_c() -> Vec<f32> {
+ vec![0.0, 0.0, 1.0, 0.0]
+ }
// Very similar to vec_a
- fn vec_a_near() -> Vec<f32> { vec![0.999, 0.001, 0.0, 0.0] }
+ fn vec_a_near() -> Vec<f32> {
+ vec![0.999, 0.001, 0.0, 0.0]
+ }
#[test]
fn store_and_recall_basic() {
diff --git a/engine/crates/xerj-api/src/auth.rs b/engine/crates/xerj-api/src/auth.rs
index d9baa0b73..11406d9da 100644
--- a/engine/crates/xerj-api/src/auth.rs
+++ b/engine/crates/xerj-api/src/auth.rs
@@ -24,11 +24,7 @@ use crate::state::AppState;
///
/// Call via `middleware::from_fn_with_state(state, auth_middleware)` in the
/// router builders.
-pub async fn auth_middleware(
- State(state): State<AppState>,
- req: Request,
- next: Next,
-) -> Response {
+pub async fn auth_middleware(State(state): State<AppState>, req: Request, next: Next) -> Response {
let cfg = &state.config.auth;
// Skip auth when disabled or no admin key is configured.
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 3932b70bb..874f5cb0b 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -21,10 +21,8 @@ use crate::{
extract::OptionalJson,
responses::{
BulkItemError, EsBulkItem, EsBulkItemAction, EsBulkItemResult, EsBulkResponse,
- EsDeleteDocResponse, EsDeleteIndexResponse, EsDocResponse, EsGetResponse,
- EsHealthResponse, EsHit, EsHits, EsHitsTotal, EsIndexMapping, EsIndexResponse,
- EsIndexSettings, EsIndexSettingsInner, EsIndexVersion, EsInfoResponse, EsMappingResponse,
- EsMappings, EsSearchResponse, EsSettingsBlock, EsSettingsResponse,
+ EsDeleteDocResponse, EsDeleteIndexResponse, EsDocResponse, EsGetResponse, EsHit, EsHits,
+ EsHitsTotal, EsIndexResponse, EsInfoResponse, EsSearchResponse,
},
state::AppState,
};
@@ -99,7 +97,10 @@ async fn cluster_health_inner(
.iter()
.filter(|info| {
want.iter().any(|s| {
- *s == "_all" || *s == "*" || glob_match_simple(s, &info.name) || *s == info.name
+ *s == "_all"
+ || *s == "*"
+ || glob_match_simple(s, &info.name)
+ || *s == info.name
})
})
.collect()
@@ -133,7 +134,11 @@ async fn cluster_health_inner(
.get(&info.name)
.map(|v| *v)
.unwrap_or(false);
- if is_closed { want_closed } else { want_open }
+ if is_closed {
+ want_closed
+ } else {
+ want_open
+ }
})
.collect()
} else {
@@ -211,17 +216,18 @@ async fn cluster_health_inner(
// keep the *cluster* status driven by replicas only. Our tests cover
// both — we currently only track a single "closed" flag per index and
// don't differentiate the closed-replication mode.
- let status = if unassigned_replicas > 0 { "yellow" } else { "green" };
+ let status = if unassigned_replicas > 0 {
+ "yellow"
+ } else {
+ "green"
+ };
// active_primary_shards / active_shards sum per-index shard counts.
// Post-7.2 ES keeps closed replicated indices' shards "active" from
// the cluster's perspective, so we count them the same way as open
// ones here (the test suite asserts this explicitly via
// `expand_wildcards: closed` + `active_shards: N`).
- let active: u32 = selected
- .iter()
- .map(|info| shard_count(&info.name))
- .sum();
+ let active: u32 = selected.iter().map(|info| shard_count(&info.name)).sum();
let _ = closed_count;
// Wait-for assertions: single-node cluster, nothing relocating /
@@ -257,7 +263,11 @@ async fn cluster_health_inner(
// check runs (we don't block), so we set timed_out accordingly.
let wait_for_active_shards_unmet = match params.wait_for_active_shards.as_deref() {
Some("all") => unassigned_replicas > 0,
- Some(s) => s.parse::<u64>().ok().map(|n| n > active as u64).unwrap_or(false),
+ Some(s) => s
+ .parse::<u64>()
+ .ok()
+ .map(|n| n > active as u64)
+ .unwrap_or(false),
None => false,
};
// When the caller explicitly requests `wait_for_nodes>=N`, we
@@ -273,20 +283,21 @@ async fn cluster_health_inner(
let s = s.trim();
if let Some(ms) = s.strip_suffix("ms") {
ms.parse::<u64>().ok().map(|v| v <= 100).unwrap_or(false)
- } else if s == "0" {
- true
} else {
- false
+ s == "0"
}
})
.unwrap_or(false);
let declared_nodes: u32 = if aggressive_timeout {
1
} else {
- params.wait_for_nodes.as_deref().map(|_| wait_for_nodes).unwrap_or(1)
+ params
+ .wait_for_nodes
+ .as_deref()
+ .map(|_| wait_for_nodes)
+ .unwrap_or(1)
};
- let timed_out = wait_for_active_shards_unmet
- || (aggressive_timeout && wait_for_nodes > 1);
+ let timed_out = wait_for_active_shards_unmet || (aggressive_timeout && wait_for_nodes > 1);
let mut resp = json!({
"cluster_name": "xerj",
@@ -333,13 +344,7 @@ async fn cluster_health_inner(
})
})
.unwrap_or(0) as u32;
- let idx_status = if replicas > 0 {
- "yellow"
- } else if is_closed {
- "green"
- } else {
- "green"
- };
+ let idx_status = if replicas > 0 { "yellow" } else { "green" };
let shards = shard_count(&info.name);
let mut idx_obj = json!({
"status": idx_status,
@@ -406,7 +411,10 @@ pub async fn cat_indices(State(state): State<AppState>) -> impl IntoResponse {
let body = lines.join("\n") + "\n";
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -483,7 +491,8 @@ pub async fn create_index(
// the original mapping declares a `@timestamp` date field. Encode the
// decision at create time so the search path can default-sort on it
// even after @timestamp is later extended by dynamic mapping.
- let declared_timestamp_date = mappings_val.pointer("/properties/@timestamp/type")
+ let declared_timestamp_date = mappings_val
+ .pointer("/properties/@timestamp/type")
.and_then(Value::as_str)
.map(|t| t == "date" || t == "date_nanos")
.unwrap_or(false);
@@ -530,7 +539,10 @@ pub async fn create_index(
if let Some(f) = explicit_sort_field.as_ref() {
if let Some(o) = s.as_object_mut() {
o.insert("__xy_index_sort_field".to_string(), json!(f));
- o.insert("__xy_index_sort_order".to_string(), json!(explicit_sort_order));
+ o.insert(
+ "__xy_index_sort_order".to_string(),
+ json!(explicit_sort_order),
+ );
// Mark this as an EXPLICIT index sort (vs the
// @timestamp auto-heuristic) so the search path can
// map a lone `_doc` sort onto the index-sort field.
@@ -542,28 +554,29 @@ pub async fn create_index(
o.insert("__xy_index_sort_order".to_string(), json!("desc"));
}
}
- state
- .engine
- .index_settings
- .insert(index.clone(), s);
+ state.engine.index_settings.insert(index.clone(), s);
} else if let Some(f) = explicit_sort_field.as_ref() {
- state.engine.index_settings.insert(index.clone(), json!({
- "__xy_index_sort_field": f,
- "__xy_index_sort_order": explicit_sort_order,
- "__xy_index_sort_explicit": true,
- }));
+ state.engine.index_settings.insert(
+ index.clone(),
+ json!({
+ "__xy_index_sort_field": f,
+ "__xy_index_sort_order": explicit_sort_order,
+ "__xy_index_sort_explicit": true,
+ }),
+ );
} else if declared_timestamp_date {
- state.engine.index_settings.insert(index.clone(), json!({
- "__xy_index_sort_field": "@timestamp",
- "__xy_index_sort_order": "desc",
- }));
+ state.engine.index_settings.insert(
+ index.clone(),
+ json!({
+ "__xy_index_sort_field": "@timestamp",
+ "__xy_index_sort_order": "desc",
+ }),
+ );
}
if !mappings_val.is_null() {
// Persists es_mapping.json into the index dir (atomic) so
// the create-time mapping survives a restart.
- state
- .engine
- .put_index_mapping(&index, mappings_val.clone());
+ state.engine.put_index_mapping(&index, mappings_val.clone());
}
if let Some(aliases) = body.get("aliases").and_then(Value::as_object) {
// Alias keys can also contain date math
@@ -594,7 +607,9 @@ pub async fn create_index(
/// that YAML tests exercise as "invalid" (`baz`, `foobar`, etc.).
fn validate_properties(props: &serde_json::Map<String, Value>) -> Result<(), String> {
for (name, spec) in props {
- let Some(obj) = spec.as_object() else { continue };
+ let Some(obj) = spec.as_object() else {
+ continue;
+ };
if let Some(t) = obj.get("type").and_then(Value::as_str) {
if !is_supported_field_type(t) {
return Err(format!(
@@ -615,7 +630,9 @@ fn validate_properties(props: &serde_json::Map<String, Value>) -> Result<(), Str
fn validate_runtime_fields(rt: &serde_json::Map<String, Value>) -> Result<(), String> {
for (name, spec) in rt {
- let Some(obj) = spec.as_object() else { continue };
+ let Some(obj) = spec.as_object() else {
+ continue;
+ };
if let Some(t) = obj.get("type").and_then(Value::as_str) {
if !is_supported_field_type(t) {
return Err(format!(
@@ -710,7 +727,11 @@ pub async fn delete_index(
let all_names: Vec<String> = all.iter().map(|i| i.name.clone()).collect();
let mut to_delete: Vec<String> = Vec::new();
- let parts: Vec<&str> = index.split(',').map(str::trim).filter(|s| !s.is_empty()).collect();
+ let parts: Vec<&str> = index
+ .split(',')
+ .map(str::trim)
+ .filter(|s| !s.is_empty())
+ .collect();
// Resolve each selector part independently. ES semantics for delete:
// - Wildcards / `_all` / `*`: expand against *indices only* (aliases
@@ -822,8 +843,12 @@ async fn get_index_inner(
.as_deref()
.unwrap_or("open")
.to_string();
- let include_closed = expand_wildcards.split(',').any(|w| w == "closed" || w == "all");
- let include_open = expand_wildcards.split(',').any(|w| w == "open" || w == "all");
+ let include_closed = expand_wildcards
+ .split(',')
+ .any(|w| w == "closed" || w == "all");
+ let include_open = expand_wildcards
+ .split(',')
+ .any(|w| w == "open" || w == "all");
let all = state.engine.list_indices().await;
@@ -921,11 +946,7 @@ async fn get_index_inner(
// Mappings: prefer the raw blob written at create; fall back to the
// schema-derived properties (which tracks subsequent put_mapping).
- let stored_mappings = state
- .engine
- .index_mappings
- .get(name)
- .map(|v| v.clone());
+ let stored_mappings = state.engine.index_mappings.get(name).map(|v| v.clone());
let mappings = match stored_mappings {
Some(m) if !m.is_null() => m,
_ => {
@@ -1025,7 +1046,9 @@ fn merge_settings_defaults(user_settings: &Value, provided_name: &str, human: bo
let ts = chrono::DateTime::<Utc>::from_timestamp_millis(cd).unwrap_or_else(Utc::now);
inner
.entry("creation_date_string".to_string())
- .or_insert_with(|| Value::String(ts.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)));
+ .or_insert_with(|| {
+ Value::String(ts.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))
+ });
}
}
@@ -1062,11 +1085,19 @@ pub async fn put_mapping(
// the call is rejected with illegal_argument_exception.
if let Some(obj) = body.as_object() {
for (k, v) in obj {
- if k == "properties" || k == "dynamic" || k == "_source"
- || k == "_meta" || k == "_routing" || k == "_size"
- || k == "_field_names" || k == "numeric_detection"
- || k == "date_detection" || k == "dynamic_date_formats"
- || k == "dynamic_templates" || k == "runtime" || k == "subobjects"
+ if k == "properties"
+ || k == "dynamic"
+ || k == "_source"
+ || k == "_meta"
+ || k == "_routing"
+ || k == "_size"
+ || k == "_field_names"
+ || k == "numeric_detection"
+ || k == "date_detection"
+ || k == "dynamic_date_formats"
+ || k == "dynamic_templates"
+ || k == "runtime"
+ || k == "subobjects"
{
continue;
}
@@ -1087,7 +1118,8 @@ pub async fn put_mapping(
},
"status": 400,
})),
- ).into_response();
+ )
+ .into_response();
}
}
}
@@ -1211,7 +1243,11 @@ fn collect_leaf_types(mapping: &Value) -> std::collections::HashMap<String, Stri
for (key, val) in props {
// Both literal-dotted keys and nested object shapes need
// visiting — we recurse from either one.
- let full = if prefix.is_empty() { key.clone() } else { format!("{}.{}", prefix, key) };
+ let full = if prefix.is_empty() {
+ key.clone()
+ } else {
+ format!("{}.{}", prefix, key)
+ };
if let Some(ftype) = val.get("type").and_then(Value::as_str) {
out.insert(full.clone(), ftype.to_string());
// Also register the literal-dotted variant the user may
@@ -1231,7 +1267,9 @@ fn collect_leaf_types(mapping: &Value) -> std::collections::HashMap<String, Stri
.and_then(|m| m.get("properties"))
.or_else(|| mapping.get("properties"))
.and_then(Value::as_object);
- if let Some(p) = props { walk(p, "", &mut out); }
+ if let Some(p) = props {
+ walk(p, "", &mut out);
+ }
out
}
@@ -1342,17 +1380,18 @@ fn inject_bbq_rescore_defaults(node: &mut Value) {
Value::Object(obj) => {
let has_type = obj.get("type").and_then(Value::as_str) == Some("dense_vector");
if has_type {
- let bbq = obj.get("index_options")
+ let bbq = obj
+ .get("index_options")
.and_then(|io| io.get("type"))
.and_then(Value::as_str);
if is_bbq(bbq) {
- let io = obj.entry("index_options".to_string()).or_insert_with(|| json!({}));
+ let io = obj
+ .entry("index_options".to_string())
+ .or_insert_with(|| json!({}));
if let Some(io_obj) = io.as_object_mut() {
if !io_obj.contains_key("rescore_vector") {
- io_obj.insert(
- "rescore_vector".to_string(),
- json!({ "oversample": 3.0 }),
- );
+ io_obj
+ .insert("rescore_vector".to_string(), json!({ "oversample": 3.0 }));
}
}
}
@@ -1438,7 +1477,8 @@ pub async fn index_doc_auto(
// create time but fails the first doc that actually references
// the range field, since copy_to on a range is meaningless
// (ES: "Copy-to currently only works for value-type fields").
- if let Some(range_field) = first_range_copy_to_field_in_mapping(&state, &index, doc.as_object()) {
+ if let Some(range_field) = first_range_copy_to_field_in_mapping(&state, &index, doc.as_object())
+ {
return (
StatusCode::BAD_REQUEST,
Json(json!({
@@ -1473,7 +1513,8 @@ pub async fn index_doc_auto(
Ok(mut results) if !results.is_empty() => {
let (action, transformed) = results.remove(0);
if matches!(action, xerj_wasm::pipeline::ProcessAction::Drop) {
- return Json(json!({"result": "noop", "_id": "", "_version": 0})).into_response();
+ return Json(json!({"result": "noop", "_id": "", "_version": 0}))
+ .into_response();
}
transformed
}
@@ -1543,7 +1584,8 @@ pub async fn index_doc(
let doc = apply_ignore_malformed(&state, &index, doc);
// Range-type + copy_to rejection — see index_doc_auto for
// context.
- if let Some(range_field) = first_range_copy_to_field_in_mapping(&state, &index, doc.as_object()) {
+ if let Some(range_field) = first_range_copy_to_field_in_mapping(&state, &index, doc.as_object())
+ {
return (
StatusCode::BAD_REQUEST,
Json(json!({
@@ -1584,11 +1626,15 @@ pub async fn index_doc(
};
// Execute ingest pipeline if specified.
let doc = if let Some(ref pipeline) = params.pipeline {
- match state.engine.process_through_pipeline(pipeline, vec![doc.clone()]) {
+ match state
+ .engine
+ .process_through_pipeline(pipeline, vec![doc.clone()])
+ {
Ok(mut results) if !results.is_empty() => {
let (action, transformed) = results.remove(0);
if matches!(action, xerj_wasm::pipeline::ProcessAction::Drop) {
- return Json(json!({"result": "noop", "_id": id, "_version": 0})).into_response();
+ return Json(json!({"result": "noop", "_id": id, "_version": 0}))
+ .into_response();
}
transformed
}
@@ -1620,14 +1666,16 @@ pub async fn index_doc(
let ext_type = params.version_type.as_deref();
let result = if let (Some(v), Some(vt)) = (params.version, ext_type) {
if vt == "external" || vt == "external_gte" {
- idx.index_document_external(Some(id.clone()), doc, v, vt).await
+ idx.index_document_external(Some(id.clone()), doc, v, vt)
+ .await
} else {
idx.index_document_with_version(
Some(id.clone()),
doc,
params.if_seq_no,
params.if_primary_term,
- ).await
+ )
+ .await
}
} else {
idx.index_document_with_version(
@@ -1635,7 +1683,8 @@ pub async fn index_doc(
doc,
params.if_seq_no,
params.if_primary_term,
- ).await
+ )
+ .await
};
match result {
Ok(resp) => {
@@ -1643,7 +1692,11 @@ pub async fn index_doc(
let is_update = resp.result == "updated";
let mut er = EsDocResponse::created(&index, &resp.id, resp.seq_no);
er.version = resp.version;
- let status = if is_update { StatusCode::OK } else { StatusCode::CREATED };
+ let status = if is_update {
+ StatusCode::OK
+ } else {
+ StatusCode::CREATED
+ };
(status, Json(er)).into_response()
}
Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
@@ -1963,7 +2016,9 @@ impl Default for EsSearchBody {
/// value unchanged. Handles `strict_date_optional_time_nanos` (and its
/// aliases) — the most common format used in ES YAML sort tests.
fn format_sort_value(raw: &Value, format: Option<&str>) -> Value {
- let Some(fmt) = format else { return raw.clone() };
+ let Some(fmt) = format else {
+ return raw.clone();
+ };
// Numeric sort value → treat as epoch; need to decide ms vs ns by
// magnitude. Values above ~2 * 10^13 are nanoseconds (more than 600
// years past epoch in ms); everything else is milliseconds.
@@ -1992,8 +2047,8 @@ fn format_sort_value(raw: &Value, format: Option<&str>) -> Value {
let trimmed = trimmed.trim_end_matches('.');
format!("{}Z", trimmed)
} else {
- let dt = chrono::DateTime::<chrono::Utc>::from_timestamp_millis(n)
- .unwrap_or_default();
+ let dt =
+ chrono::DateTime::<chrono::Utc>::from_timestamp_millis(n).unwrap_or_default();
dt.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()
}
}
@@ -2011,8 +2066,7 @@ fn format_sort_value(raw: &Value, format: Option<&str>) -> Value {
chrono::DateTime::<chrono::Utc>::from_timestamp(secs, sub_ns as u32)
.unwrap_or_default()
} else {
- chrono::DateTime::<chrono::Utc>::from_timestamp_millis(n)
- .unwrap_or_default()
+ chrono::DateTime::<chrono::Utc>::from_timestamp_millis(n).unwrap_or_default()
};
dt.format(&pat).to_string()
}
@@ -2029,24 +2083,54 @@ fn java_pattern_to_strftime(fmt: &str) -> String {
let mut i = 0;
while i < bytes.len() {
let rest = &fmt[i..];
- if rest.starts_with("yyyy") { out.push_str("%Y"); i += 4; }
- else if rest.starts_with("uuuu") { out.push_str("%Y"); i += 4; }
- else if rest.starts_with("yy") { out.push_str("%y"); i += 2; }
- else if rest.starts_with("MM") { out.push_str("%m"); i += 2; }
- else if rest.starts_with("dd") { out.push_str("%d"); i += 2; }
- else if rest.starts_with("HH") { out.push_str("%H"); i += 2; }
- else if rest.starts_with("mm") { out.push_str("%M"); i += 2; }
- else if rest.starts_with("ss") { out.push_str("%S"); i += 2; }
- else if rest.starts_with("SSSSSSSSS") { out.push_str("%9f"); i += 9; }
- else if rest.starts_with("SSSSSS") { out.push_str("%6f"); i += 6; }
- else if rest.starts_with("SSS") { out.push_str("%3f"); i += 3; }
- else if rest.starts_with("ZZZZZ") { out.push_str("%:z"); i += 5; }
- else if rest.starts_with("XXX") { out.push_str("%:z"); i += 3; }
- else if rest.starts_with('Z') { out.push_str("%z"); i += 1; }
- else if rest.starts_with('\'') {
+ if rest.starts_with("yyyy") || rest.starts_with("uuuu") {
+ out.push_str("%Y");
+ i += 4;
+ } else if rest.starts_with("yy") {
+ out.push_str("%y");
+ i += 2;
+ } else if rest.starts_with("MM") {
+ out.push_str("%m");
+ i += 2;
+ } else if rest.starts_with("dd") {
+ out.push_str("%d");
+ i += 2;
+ } else if rest.starts_with("HH") {
+ out.push_str("%H");
+ i += 2;
+ } else if rest.starts_with("mm") {
+ out.push_str("%M");
+ i += 2;
+ } else if rest.starts_with("ss") {
+ out.push_str("%S");
+ i += 2;
+ } else if rest.starts_with("SSSSSSSSS") {
+ out.push_str("%9f");
+ i += 9;
+ } else if rest.starts_with("SSSSSS") {
+ out.push_str("%6f");
+ i += 6;
+ } else if rest.starts_with("SSS") {
+ out.push_str("%3f");
+ i += 3;
+ } else if rest.starts_with("ZZZZZ") {
+ out.push_str("%:z");
+ i += 5;
+ } else if rest.starts_with("XXX") {
+ out.push_str("%:z");
+ i += 3;
+ } else if rest.starts_with('Z') {
+ out.push_str("%z");
i += 1;
- while i < bytes.len() && bytes[i] != b'\'' { out.push(bytes[i] as char); i += 1; }
- if i < bytes.len() { i += 1; }
+ } else if rest.starts_with('\'') {
+ i += 1;
+ while i < bytes.len() && bytes[i] != b'\'' {
+ out.push(bytes[i] as char);
+ i += 1;
+ }
+ if i < bytes.len() {
+ i += 1;
+ }
} else {
out.push(bytes[i] as char);
i += 1;
@@ -2069,7 +2153,10 @@ fn java_pattern_to_strftime(fmt: &str) -> String {
/// only knows inline sources, so resolution happens at the coord layer.
fn resolve_stored_scripts(v: &mut Value, state: &AppState) {
fn is_script_key(k: &str) -> bool {
- matches!(k, "init_script" | "map_script" | "combine_script" | "reduce_script" | "script")
+ matches!(
+ k,
+ "init_script" | "map_script" | "combine_script" | "reduce_script" | "script"
+ )
}
fn walk(v: &mut Value, state: &AppState) {
match v {
@@ -2079,12 +2166,22 @@ fn resolve_stored_scripts(v: &mut Value, state: &AppState) {
let child = obj.get_mut(&k).unwrap();
if is_script_key(&k) {
if let Some(script_obj) = child.as_object_mut() {
- if let Some(id) = script_obj.get("id").and_then(Value::as_str).map(String::from) {
- if let Some(src) = state.engine.search_templates.get(&id).map(|v| v.clone()) {
+ if let Some(id) = script_obj
+ .get("id")
+ .and_then(Value::as_str)
+ .map(String::from)
+ {
+ if let Some(src) =
+ state.engine.search_templates.get(&id).map(|v| v.clone())
+ {
let src_str = src
.as_str()
.map(String::from)
- .or_else(|| src.get("source").and_then(Value::as_str).map(String::from))
+ .or_else(|| {
+ src.get("source")
+ .and_then(Value::as_str)
+ .map(String::from)
+ })
.unwrap_or_default();
script_obj.remove("id");
script_obj.insert("source".to_string(), Value::String(src_str));
@@ -2096,7 +2193,9 @@ fn resolve_stored_scripts(v: &mut Value, state: &AppState) {
}
}
Value::Array(arr) => {
- for item in arr.iter_mut() { walk(item, state); }
+ for item in arr.iter_mut() {
+ walk(item, state);
+ }
}
_ => {}
}
@@ -2114,11 +2213,11 @@ fn build_matched_queries_value(
include_scores: bool,
hit_score: Option<f64>,
) -> Value {
- if names.is_empty() { return Value::Null; }
+ if names.is_empty() {
+ return Value::Null;
+ }
if !include_scores {
- return Value::Array(
- names.iter().cloned().map(Value::String).collect(),
- );
+ return Value::Array(names.iter().cloned().map(Value::String).collect());
}
let mut boosts: std::collections::BTreeMap<String, f64> = std::collections::BTreeMap::new();
let mut fs_names: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
@@ -2137,8 +2236,7 @@ fn build_matched_queries_value(
} else {
boosts.get(name).copied().unwrap_or(1.0)
};
- let n = serde_json::Number::from_f64(score)
- .unwrap_or_else(|| serde_json::Number::from(1));
+ let n = serde_json::Number::from_f64(score).unwrap_or_else(|| serde_json::Number::from(1));
out.insert(name.clone(), Value::Number(n));
}
Value::Object(out)
@@ -2157,9 +2255,15 @@ fn collect_function_score_names(q: &Value, out: &mut std::collections::BTreeSet<
out.insert(n.to_string());
}
}
- for (_, v) in obj { collect_function_score_names(v, out); }
+ for (_, v) in obj {
+ collect_function_score_names(v, out);
+ }
+ }
+ Value::Array(arr) => {
+ for item in arr {
+ collect_function_score_names(item, out);
+ }
}
- Value::Array(arr) => { for item in arr { collect_function_score_names(item, out); } }
_ => {}
}
}
@@ -2209,7 +2313,9 @@ fn collect_named_query_scores(
.and_then(|s| s.trim().parse::<f64>().ok());
let fn_score = fn_weight.or(fn_script_score);
for (k, v) in obj {
- if k == "_name" || k == "boost" { continue; }
+ if k == "_name" || k == "boost" {
+ continue;
+ }
// Descend into the child, propagating the boost along the
// way so ES's multiplicative semantics (outer_boost *
// inner_boost) carry through for nested named queries.
@@ -2227,7 +2333,9 @@ fn collect_named_query_scores(
}
}
Value::Array(arr) => {
- for item in arr { collect_named_query_scores(item, parent_boost, out); }
+ for item in arr {
+ collect_named_query_scores(item, parent_boost, out);
+ }
}
_ => {}
}
@@ -2238,11 +2346,7 @@ fn collect_named_query_scores(
/// mapping declares `index: false` AND `doc_values: false` (or the field
/// type doesn't have implicit doc_values support). When unqueryable across
/// every index, the clause is rewritten to `{"match_none": {}}`.
-fn rewrite_unqueryable_exists(
- q: &mut Value,
- state: &AppState,
- indices: &[String],
-) {
+fn rewrite_unqueryable_exists(q: &mut Value, state: &AppState, indices: &[String]) {
fn field_is_queryable(state: &AppState, idx: &str, field: &str) -> bool {
let Some(mapping) = state.engine.index_mappings.get(idx).map(|v| v.clone()) else {
return true; // unknown mapping — be permissive
@@ -2260,10 +2364,10 @@ fn rewrite_unqueryable_exists(
) -> Option<&'a Value> {
let first = segs.first()?;
let node = props.get(*first)?;
- if segs.len() == 1 { return Some(node); }
- let child = node
- .get("properties")
- .and_then(Value::as_object)?;
+ if segs.len() == 1 {
+ return Some(node);
+ }
+ let child = node.get("properties").and_then(Value::as_object)?;
resolve(child, &segs[1..])
}
let Some(field_map) = resolve(props, &segs) else {
@@ -2280,11 +2384,11 @@ fn rewrite_unqueryable_exists(
// Default doc_values per field type. Numeric/keyword/date/ip/
// geo_point/boolean all default to true; text defaults to
// false.
- let ftype = field_map
- .get("type")
- .and_then(Value::as_str)
- .unwrap_or("");
- !matches!(ftype, "text" | "annotated_text" | "match_only_text" | "binary")
+ let ftype = field_map.get("type").and_then(Value::as_str).unwrap_or("");
+ !matches!(
+ ftype,
+ "text" | "annotated_text" | "match_only_text" | "binary"
+ )
});
indexed || has_dv
}
@@ -2300,7 +2404,9 @@ fn rewrite_unqueryable_exists(
.and_then(Value::as_str)
.map(String::from)
{
- let any_queryable = indices.iter().any(|ix| field_is_queryable(state, ix, &field));
+ let any_queryable = indices
+ .iter()
+ .any(|ix| field_is_queryable(state, ix, &field));
if !any_queryable {
obj.remove("exists");
obj.insert("match_none".to_string(), json!({}));
@@ -2363,13 +2469,20 @@ fn collect_terms_lookups(q: &Value, out: &mut Vec<(String, String, String, Strin
if k == "terms" {
if let Some(terms_obj) = v.as_object() {
for (field, spec) in terms_obj.iter() {
- if field == "boost" { continue; }
+ if field == "boost" {
+ continue;
+ }
if let Some(s) = spec.as_object() {
let ix = s.get("index").and_then(Value::as_str);
let id = s.get("id").and_then(Value::as_str);
let path = s.get("path").and_then(Value::as_str);
if let (Some(ix), Some(id), Some(path)) = (ix, id, path) {
- out.push((field.clone(), ix.to_string(), id.to_string(), path.to_string()));
+ out.push((
+ field.clone(),
+ ix.to_string(),
+ id.to_string(),
+ path.to_string(),
+ ));
}
}
}
@@ -2398,7 +2511,9 @@ fn apply_terms_lookups(
if let Some(terms_obj) = terms_val.as_object_mut() {
let fields: Vec<String> = terms_obj.keys().cloned().collect();
for field in fields {
- if field == "boost" { continue; }
+ if field == "boost" {
+ continue;
+ }
let spec = terms_obj.get(&field).cloned();
if let Some(Value::Object(s)) = spec {
let ix = s.get("index").and_then(Value::as_str);
@@ -2480,22 +2595,22 @@ fn extract_field_values_from_source(src: &Value, path: &str) -> Vec<Value> {
/// xerj-engine aggs so the coordinator picks from the same palette.
const AUTO_DATE_INTERVALS: &[(&str, i64)] = &[
("1ms", 1),
- ("1s", 1_000),
+ ("1s", 1_000),
("10s", 10_000),
("30s", 30_000),
- ("1m", 60_000),
- ("5m", 300_000),
+ ("1m", 60_000),
+ ("5m", 300_000),
("10m", 600_000),
("15m", 900_000),
("30m", 1_800_000),
- ("1h", 3_600_000),
- ("3h", 10_800_000),
+ ("1h", 3_600_000),
+ ("3h", 10_800_000),
("12h", 43_200_000),
- ("1d", 86_400_000),
- ("7d", 604_800_000),
+ ("1d", 86_400_000),
+ ("7d", 604_800_000),
("30d", 2_592_000_000),
("90d", 7_776_000_000),
- ("1y", 31_536_000_000),
+ ("1y", 31_536_000_000),
];
fn epoch_ms_to_iso8601_utc(ms: i64) -> String {
@@ -2510,15 +2625,22 @@ fn epoch_ms_to_iso8601_utc(ms: i64) -> String {
/// per-shard bucket keys. Recursive over sub-aggs so nested agg trees
/// (e.g. `filter { auto_date_histogram {...} }`) get fixed too.
fn rebucket_auto_date_histograms(results: &mut Value, aggs_req: &Value) {
- let Some(req_obj) = aggs_req.as_object() else { return };
- let Some(res_obj) = results.as_object_mut() else { return };
+ let Some(req_obj) = aggs_req.as_object() else {
+ return;
+ };
+ let Some(res_obj) = results.as_object_mut() else {
+ return;
+ };
for (name, req_spec) in req_obj {
- let Some(spec_obj) = req_spec.as_object() else { continue };
- if let Some(params) = spec_obj.get("auto_date_histogram").and_then(Value::as_object) {
- let target_buckets = params
- .get("buckets")
- .and_then(Value::as_u64)
- .unwrap_or(10) as usize;
+ let Some(spec_obj) = req_spec.as_object() else {
+ continue;
+ };
+ if let Some(params) = spec_obj
+ .get("auto_date_histogram")
+ .and_then(Value::as_object)
+ {
+ let target_buckets =
+ params.get("buckets").and_then(Value::as_u64).unwrap_or(10) as usize;
if let Some(res_val) = res_obj.get_mut(name) {
recoordinate_auto_date_histogram(res_val, target_buckets);
}
@@ -2527,7 +2649,9 @@ fn rebucket_auto_date_histograms(results: &mut Value, aggs_req: &Value) {
// sibling. In the merged response, nested aggs live next to the
// bucket in each bucket's sub-object OR top-level alongside
// metric results.
- let nested_req = spec_obj.get("aggs").or_else(|| spec_obj.get("aggregations"));
+ let nested_req = spec_obj
+ .get("aggs")
+ .or_else(|| spec_obj.get("aggregations"));
if let (Some(nested_req), Some(res_val)) = (nested_req, res_obj.get_mut(name)) {
// Descend into buckets (bucket sub-aggs).
if let Some(buckets) = res_val.get_mut("buckets") {
@@ -2553,13 +2677,16 @@ fn rebucket_auto_date_histograms(results: &mut Value, aggs_req: &Value) {
}
fn recoordinate_auto_date_histogram(res: &mut Value, target_buckets: usize) {
- let Some(obj) = res.as_object_mut() else { return };
+ let Some(obj) = res.as_object_mut() else {
+ return;
+ };
let buckets = match obj.get("buckets").and_then(|b| b.as_array()) {
Some(b) if !b.is_empty() => b.clone(),
_ => return,
};
// Collect (key_ms, doc_count, sub_aggs_map) per bucket.
- let mut entries: Vec<(i64, u64, serde_json::Map<String, Value>)> = Vec::with_capacity(buckets.len());
+ let mut entries: Vec<(i64, u64, serde_json::Map<String, Value>)> =
+ Vec::with_capacity(buckets.len());
for b in &buckets {
let bo = match b.as_object() {
Some(o) => o,
@@ -2574,7 +2701,9 @@ fn recoordinate_auto_date_histogram(res: &mut Value, target_buckets: usize) {
for (k, v) in bo.iter() {
match k.as_str() {
"key" | "key_as_string" | "doc_count" => {}
- _ => { subs.insert(k.clone(), v.clone()); }
+ _ => {
+ subs.insert(k.clone(), v.clone());
+ }
}
}
entries.push((key, doc_count, subs));
@@ -2606,10 +2735,13 @@ fn recoordinate_auto_date_histogram(res: &mut Value, target_buckets: usize) {
return;
}
// Re-bucket at the coarser grid. Sum doc_counts and merge sub-aggs.
- let mut grid: std::collections::BTreeMap<i64, (u64, serde_json::Map<String, Value>)> = std::collections::BTreeMap::new();
+ let mut grid: std::collections::BTreeMap<i64, (u64, serde_json::Map<String, Value>)> =
+ std::collections::BTreeMap::new();
for (key, dc, subs) in entries {
let bucket_start = key.div_euclid(new_interval) * new_interval;
- let e = grid.entry(bucket_start).or_insert_with(|| (0, serde_json::Map::new()));
+ let e = grid
+ .entry(bucket_start)
+ .or_insert_with(|| (0, serde_json::Map::new()));
e.0 += dc;
// Sub-agg merge: if existing, delegate to merge_metric_agg /
// merge_bucket_agg when possible; otherwise keep the first.
@@ -2633,10 +2765,15 @@ fn recoordinate_auto_date_histogram(res: &mut Value, target_buckets: usize) {
let mut new_buckets: Vec<Value> = Vec::with_capacity(grid.len());
for (bucket_start, (dc, subs)) in grid {
let mut bo = serde_json::Map::new();
- bo.insert("key_as_string".to_string(), Value::String(epoch_ms_to_iso8601_utc(bucket_start)));
+ bo.insert(
+ "key_as_string".to_string(),
+ Value::String(epoch_ms_to_iso8601_utc(bucket_start)),
+ );
bo.insert("key".to_string(), json!(bucket_start));
bo.insert("doc_count".to_string(), json!(dc));
- for (k, v) in subs { bo.insert(k, v); }
+ for (k, v) in subs {
+ bo.insert(k, v);
+ }
new_buckets.push(Value::Object(bo));
}
obj.insert("interval".to_string(), Value::String(new_label.to_string()));
@@ -2667,8 +2804,14 @@ fn merge_single_bucket_agg(old: &Value, new: &Value) -> Option<Value> {
return None;
}
let mut merged = old_obj.clone();
- let a_count = old_obj.get("doc_count").and_then(Value::as_u64).unwrap_or(0);
- let b_count = new_obj.get("doc_count").and_then(Value::as_u64).unwrap_or(0);
+ let a_count = old_obj
+ .get("doc_count")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
+ let b_count = new_obj
+ .get("doc_count")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
merged.insert("doc_count".to_string(), json!(a_count + b_count));
for (k, v) in new_obj {
if k == "doc_count" || k.starts_with("__") {
@@ -2820,9 +2963,7 @@ fn merge_bucket_agg(old: &Value, new: &Value) -> Option<Value> {
// Canonicalize numbers so long 10 and double 10.0 collide.
if let Some(f) = n.as_f64() {
let trunc = f.trunc();
- if (f - trunc).abs() < f64::EPSILON
- && trunc.abs() < (1u64 << 53) as f64
- {
+ if (f - trunc).abs() < f64::EPSILON && trunc.abs() < (1u64 << 53) as f64 {
return (trunc as i64).to_string();
}
// Round-trip through f64 to normalize "2.0" vs "2" variants.
@@ -2846,10 +2987,8 @@ fn merge_bucket_agg(old: &Value, new: &Value) -> Option<Value> {
})
.collect();
pairs.sort();
- let joined: Vec<String> = pairs
- .into_iter()
- .map(|(k, v)| format!("{k}={v}"))
- .collect();
+ let joined: Vec<String> =
+ pairs.into_iter().map(|(k, v)| format!("{k}={v}")).collect();
format!("o:{}", joined.join(","))
}
Some(Value::Array(a)) => format!("a:{}", serde_json::to_string(a).unwrap_or_default()),
@@ -2934,10 +3073,7 @@ fn merge_bucket_agg(old: &Value, new: &Value) -> Option<Value> {
let summable = |k: &str| -> bool {
matches!(
k,
- "sum_other_doc_count"
- | "doc_count_error_upper_bound"
- | "bg_count"
- | "doc_count"
+ "sum_other_doc_count" | "doc_count_error_upper_bound" | "bg_count" | "doc_count"
)
};
for (k, v) in old_obj.iter().chain(new_obj.iter()) {
@@ -2946,8 +3082,7 @@ fn merge_bucket_agg(old: &Value, new: &Value) -> Option<Value> {
}
match v {
Value::Number(n) if summable(k) => {
- let acc = out.get(k).and_then(Value::as_u64).unwrap_or(0)
- + n.as_u64().unwrap_or(0);
+ let acc = out.get(k).and_then(Value::as_u64).unwrap_or(0) + n.as_u64().unwrap_or(0);
out.insert(k.clone(), json!(acc));
}
_ => {
@@ -2967,29 +3102,13 @@ fn merge_bucket_agg(old: &Value, new: &Value) -> Option<Value> {
Some(Value::Object(out))
}
-/// Collect the top-level keys of a JSON object into a HashSet.
-fn obj_keys_as_set(v: &Value) -> std::collections::HashSet<String> {
- v.as_object()
- .map(|o| o.keys().cloned().collect())
- .unwrap_or_default()
-}
-
-/// Recursively apply the synthetic-source dotted-key transform.
-/// At each level, the `props` value is the mapping properties object
-/// covering that nesting level (or `Value::Null` for unmapped levels).
-/// Recursion only descends into children whose parent IS mapped (i.e.
-/// the child has sub-properties in the mapping) — residual
-/// dotted-key leafs produced by an ignored_source split at an
-/// unmapped boundary stay literal verbatim, matching ES semantics.
-fn synthetic_transform_object(target: &mut serde_json::Map<String, Value>, props: &Value) {
- synthetic_transform_object_ext(target, props, false)
-}
-
-fn synthetic_transform_object_ext(target: &mut serde_json::Map<String, Value>, props: &Value, index_keep_arrays: bool) {
- synthetic_transform_object_ext2(target, props, index_keep_arrays, false, false)
-}
-
-fn synthetic_transform_object_ext2(target: &mut serde_json::Map<String, Value>, props: &Value, index_keep_arrays: bool, parent_dynamic_false: bool, inside_nested: bool) {
+fn synthetic_transform_object_ext2(
+ target: &mut serde_json::Map<String, Value>,
+ props: &Value,
+ index_keep_arrays: bool,
+ parent_dynamic_false: bool,
+ inside_nested: bool,
+) {
// Step 1: collect dotted keys at this level and re-insert via
// insert_synthetic_path (only transform when the target level has
// a real mapping — otherwise the dotted key is "ignored source"
@@ -2998,7 +3117,9 @@ fn synthetic_transform_object_ext2(target: &mut serde_json::Map<String, Value>,
if has_mapped_props {
let keys: Vec<String> = target.keys().cloned().collect();
for key in keys {
- if !key.contains('.') { continue; }
+ if !key.contains('.') {
+ continue;
+ }
let val = target.remove(&key).unwrap_or(Value::Null);
insert_synthetic_path(target, props, &key, val);
}
@@ -3030,7 +3151,9 @@ fn synthetic_transform_object_ext2(target: &mut serde_json::Map<String, Value>,
// the parent explicitly opts in (has properties AND is not
// dynamic-false).
if field_spec.is_none() {
- if parent_dynamic_false { continue; }
+ if parent_dynamic_false {
+ continue;
+ }
if !inside_nested {
// At a regular (non-nested) parent with no
// declared entry for this key, prefer the safe
@@ -3039,30 +3162,42 @@ fn synthetic_transform_object_ext2(target: &mut serde_json::Map<String, Value>,
// field is a dynamically-added object sibling and
// should flatten, matching "nested object next to
// regular").
- let parent_has_declared_siblings = props.as_object()
- .map(|o| !o.is_empty())
- .unwrap_or(false);
- if !parent_has_declared_siblings { continue; }
+ let parent_has_declared_siblings =
+ props.as_object().map(|o| !o.is_empty()).unwrap_or(false);
+ if !parent_has_declared_siblings {
+ continue;
+ }
}
}
- let (sub_props, sub_dynamic_false, keep_mode, is_nested, is_disabled) = match field_spec {
+ let (sub_props, sub_dynamic_false, keep_mode, is_nested, is_disabled) = match field_spec
+ {
Some(spec) => {
let ftype = spec.get("type").and_then(Value::as_str).unwrap_or("");
- let disabled = matches!(spec.get("enabled").and_then(Value::as_bool), Some(false));
- let keep = spec.get("synthetic_source_keep").and_then(Value::as_str)
+ let disabled =
+ matches!(spec.get("enabled").and_then(Value::as_bool), Some(false));
+ let keep = spec
+ .get("synthetic_source_keep")
+ .and_then(Value::as_str)
.map(String::from);
let sub_props = spec.get("properties").cloned().unwrap_or(Value::Null);
- let sub_df = spec.get("dynamic").map(|v| match v {
- Value::Bool(false) => true,
- Value::String(s) => s == "false" || s == "runtime",
- _ => false,
- }).unwrap_or(false);
+ let sub_df = spec
+ .get("dynamic")
+ .map(|v| match v {
+ Value::Bool(false) => true,
+ Value::String(s) => s == "false" || s == "runtime",
+ _ => false,
+ })
+ .unwrap_or(false);
(sub_props, sub_df, keep, ftype == "nested", disabled)
}
None => (Value::Null, false, None, false, false),
};
- if is_nested { continue; }
- if is_disabled { continue; }
+ if is_nested {
+ continue;
+ }
+ if is_disabled {
+ continue;
+ }
// Effective synthetic_source_keep: an explicit per-field
// `arrays|all|none` overrides the index-level default. keep:arrays|all
// preserves the source array shape (never column-flatten); keep:none
@@ -3075,14 +3210,26 @@ fn synthetic_transform_object_ext2(target: &mut serde_json::Map<String, Value>,
Some("none") => false,
_ => index_keep_arrays,
};
- if effective_keep_arrays { continue; }
+ if effective_keep_arrays {
+ continue;
+ }
let should_flatten = matches!(
target.get(&name),
Some(Value::Array(arr)) if !arr.is_empty() && arr.iter().all(|v| v.is_object())
);
- if !should_flatten { continue; }
+ if !should_flatten {
+ continue;
+ }
if let Some(Value::Array(arr)) = target.remove(&name) {
- target.insert(name, synthetic_flatten_object_array(&arr, &sub_props, sub_dynamic_false, index_keep_arrays));
+ target.insert(
+ name,
+ synthetic_flatten_object_array(
+ &arr,
+ &sub_props,
+ sub_dynamic_false,
+ index_keep_arrays,
+ ),
+ );
}
}
}
@@ -3098,7 +3245,9 @@ fn synthetic_transform_object_ext2(target: &mut serde_json::Map<String, Value>,
let names: Vec<String> = target.keys().cloned().collect();
for name in names {
let child_spec = props.as_object().and_then(|o| o.get(&name));
- let child_type = child_spec.and_then(|hp| hp.get("type")).and_then(Value::as_str);
+ let child_type = child_spec
+ .and_then(|hp| hp.get("type"))
+ .and_then(Value::as_str);
let child_is_nested = child_type == Some("nested");
// Flattened fields reconstruct as a single-level dotted-key map
// (`{host:{name:x}}` -> `{"host.name":x}`), not a re-nested object.
@@ -3129,12 +3278,15 @@ fn synthetic_transform_object_ext2(target: &mut serde_json::Map<String, Value>,
// Skip recursion unless we have sub-properties OR this is a
// nested parent (nested without declared sub-properties still
// needs per-element synthetic reconstruction for dynamic children).
- if child_props_raw.is_none() && !child_is_nested { continue; }
+ if child_props_raw.is_none() && !child_is_nested {
+ continue;
+ }
let child_props = child_props_raw.unwrap_or(Value::Null);
let child_keep = child_spec
.and_then(|hp| hp.get("synthetic_source_keep"))
.and_then(Value::as_str);
- let child_keep_arrays = index_keep_arrays || matches!(child_keep, Some("arrays") | Some("all"));
+ let child_keep_arrays =
+ index_keep_arrays || matches!(child_keep, Some("arrays") | Some("all"));
let child_dynamic_false = child_spec
.and_then(|hp| hp.get("dynamic"))
.map(|v| match v {
@@ -3145,11 +3297,23 @@ fn synthetic_transform_object_ext2(target: &mut serde_json::Map<String, Value>,
.unwrap_or(false);
if let Some(child_val) = target.get_mut(&name) {
match child_val {
- Value::Object(child_obj) => synthetic_transform_object_ext2(child_obj, &child_props, child_keep_arrays, child_dynamic_false, child_is_nested),
+ Value::Object(child_obj) => synthetic_transform_object_ext2(
+ child_obj,
+ &child_props,
+ child_keep_arrays,
+ child_dynamic_false,
+ child_is_nested,
+ ),
Value::Array(arr) => {
for el in arr.iter_mut() {
if let Value::Object(co) = el {
- synthetic_transform_object_ext2(co, &child_props, child_keep_arrays, child_dynamic_false, child_is_nested);
+ synthetic_transform_object_ext2(
+ co,
+ &child_props,
+ child_keep_arrays,
+ child_dynamic_false,
+ child_is_nested,
+ );
}
}
}
@@ -3168,13 +3332,17 @@ fn synthetic_transform_object_ext2(target: &mut serde_json::Map<String, Value>,
// Under `dynamic: false`, unmapped keys are ignored-source —
// their primitive-array leaves keep the original source order
// rather than being re-sorted via doc_values.
- if field_spec.is_none() && parent_dynamic_false { continue; }
+ if field_spec.is_none() && parent_dynamic_false {
+ continue;
+ }
let is_text = field_spec
.and_then(|hp| hp.get("type"))
.and_then(Value::as_str)
.map(|t| t == "text" || t == "match_only_text" || t == "annotated_text")
.unwrap_or(false);
- if is_text { continue; }
+ if is_text {
+ continue;
+ }
// Per-field synthetic_source_keep overrides index default:
// none → force sort (doc_values behaviour)
// arrays → preserve source order of the array
@@ -3187,12 +3355,18 @@ fn synthetic_transform_object_ext2(target: &mut serde_json::Map<String, Value>,
Some("arrays") | Some("all") => continue,
Some("none") => {} // force sort below
_ => {
- if index_keep_arrays { continue; }
+ if index_keep_arrays {
+ continue;
+ }
}
}
if let Some(Value::Array(arr)) = target.get_mut(&name) {
- let all_primitive = arr.iter().all(|v| matches!(v, Value::Number(_) | Value::String(_) | Value::Bool(_)));
- if !all_primitive || arr.len() < 2 { continue; }
+ let all_primitive = arr
+ .iter()
+ .all(|v| matches!(v, Value::Number(_) | Value::String(_) | Value::Bool(_)));
+ if !all_primitive || arr.len() < 2 {
+ continue;
+ }
arr.sort_by(|a, b| match (a, b) {
(Value::Number(x), Value::Number(y)) => {
let xf = x.as_f64().unwrap_or(0.0);
@@ -3227,7 +3401,11 @@ fn flatten_synthetic_dotted(prefix: &str, value: &Value, out: &mut serde_json::M
match value {
Value::Object(o) => {
for (k, v) in o {
- let key = if prefix.is_empty() { k.clone() } else { format!("{prefix}.{k}") };
+ let key = if prefix.is_empty() {
+ k.clone()
+ } else {
+ format!("{prefix}.{k}")
+ };
flatten_synthetic_dotted(&key, v, out);
}
}
@@ -3237,9 +3415,15 @@ fn flatten_synthetic_dotted(prefix: &str, value: &Value, out: &mut serde_json::M
}
}
-fn synthetic_flatten_object_array(arr: &[Value], props: &Value, parent_dynamic_false: bool, index_keep_arrays: bool) -> Value {
+fn synthetic_flatten_object_array(
+ arr: &[Value],
+ props: &Value,
+ parent_dynamic_false: bool,
+ index_keep_arrays: bool,
+) -> Value {
let mut keys_in_order: Vec<String> = Vec::new();
- let mut groups: std::collections::HashMap<String, Vec<Value>> = std::collections::HashMap::new();
+ let mut groups: std::collections::HashMap<String, Vec<Value>> =
+ std::collections::HashMap::new();
for el in arr {
let Value::Object(obj) = el else { continue };
for (k, v) in obj {
@@ -3249,7 +3433,9 @@ fn synthetic_flatten_object_array(arr: &[Value], props: &Value, parent_dynamic_f
let entry = groups.entry(k.clone()).or_default();
match v {
Value::Array(inner) => {
- for iv in inner { entry.push(iv.clone()); }
+ for iv in inner {
+ entry.push(iv.clone());
+ }
}
_ => entry.push(v.clone()),
}
@@ -3290,7 +3476,15 @@ fn synthetic_flatten_object_array(arr: &[Value], props: &Value, parent_dynamic_f
}
let all_objects = !values.is_empty() && values.iter().all(|v| v.is_object());
if all_objects {
- out.insert(k, synthetic_flatten_object_array(&values, &child_props, child_dynamic_false, index_keep_arrays));
+ out.insert(
+ k,
+ synthetic_flatten_object_array(
+ &values,
+ &child_props,
+ child_dynamic_false,
+ index_keep_arrays,
+ ),
+ );
continue;
}
// Unmapped leaves under a `dynamic: false` parent are
@@ -3300,9 +3494,10 @@ fn synthetic_flatten_object_array(arr: &[Value], props: &Value, parent_dynamic_f
out.insert(k, Value::Array(values));
continue;
}
- let all_primitive = !values.is_empty() && values.iter().all(|v| {
- matches!(v, Value::Number(_) | Value::String(_) | Value::Bool(_))
- });
+ let all_primitive = !values.is_empty()
+ && values
+ .iter()
+ .all(|v| matches!(v, Value::Number(_) | Value::String(_) | Value::Bool(_)));
if !all_primitive {
out.insert(k, Value::Array(values));
continue;
@@ -3341,7 +3536,9 @@ fn insert_synthetic_path(
// Walk segments and collect the "descent path" (mapped segments to
// descend into) and the "tail" (the final key and optional value key).
let segments: Vec<&str> = key.split('.').collect();
- if segments.is_empty() { return; }
+ if segments.is_empty() {
+ return;
+ }
let mut descent: Vec<String> = Vec::new();
let mut current_props: Value = props.clone();
let mut cursor = 0usize;
@@ -3385,7 +3582,8 @@ fn insert_synthetic_path(
// Navigate to parent, creating nested objects as needed.
let mut cur: &mut serde_json::Map<String, Value> = target;
for p in &parent_path {
- let entry = cur.entry(p.clone())
+ let entry = cur
+ .entry(p.clone())
.or_insert_with(|| Value::Object(serde_json::Map::new()));
if !entry.is_object() {
let old = entry.take();
@@ -3424,7 +3622,10 @@ fn default_size() -> usize {
}
/// Build a `SearchRequest` from the ES body, forwarding all relevant options.
-fn build_search_request(body: &EsSearchBody, aggs_value: Option<Value>) -> Result<xerj_query::ast::SearchRequest, xerj_common::XerjError> {
+fn build_search_request(
+ body: &EsSearchBody,
+ aggs_value: Option<Value>,
+) -> Result<xerj_query::ast::SearchRequest, xerj_common::XerjError> {
use xerj_query::ast::SourceFilter;
let query_val = body.query.clone().unwrap_or(json!({ "match_all": {} }));
@@ -3482,7 +3683,11 @@ fn build_search_request(body: &EsSearchBody, aggs_value: Option<Value>) -> Resul
.get("includes")
.or_else(|| obj.get("include"))
.and_then(|v| match v {
- Value::Array(a) => Some(a.iter().filter_map(|v| v.as_str().map(String::from)).collect()),
+ Value::Array(a) => Some(
+ a.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect(),
+ ),
Value::String(s) => Some(vec![s.clone()]),
_ => None,
})
@@ -3491,7 +3696,11 @@ fn build_search_request(body: &EsSearchBody, aggs_value: Option<Value>) -> Resul
.get("excludes")
.or_else(|| obj.get("exclude"))
.and_then(|v| match v {
- Value::Array(a) => Some(a.iter().filter_map(|v| v.as_str().map(String::from)).collect()),
+ Value::Array(a) => Some(
+ a.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect(),
+ ),
Value::String(s) => Some(vec![s.clone()]),
_ => None,
})
@@ -3621,13 +3830,29 @@ fn parse_rescore(val: &Value) -> Vec<xerj_query::ast::RescoreQuery> {
// ES wraps the actual script under TWO `script` levels — outer
// identifies the rescorer type, inner holds source + params.
if let Some(s_obj) = item.get("script").and_then(Value::as_object) {
- let inner = s_obj.get("script").and_then(Value::as_object).cloned()
+ let inner = s_obj
+ .get("script")
+ .and_then(Value::as_object)
+ .cloned()
.unwrap_or_else(serde_json::Map::new);
- let source = inner.get("source").and_then(Value::as_str).unwrap_or("").to_string();
+ let source = inner
+ .get("source")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
let params = inner.get("params").cloned().unwrap_or(json!({}));
- let q_w = s_obj.get("query_weight").and_then(Value::as_f64).unwrap_or(1.0) as f32;
- let r_w = s_obj.get("rescore_query_weight").and_then(Value::as_f64).unwrap_or(1.0) as f32;
- let mode = s_obj.get("score_mode").and_then(Value::as_str).map(String::from);
+ let q_w = s_obj
+ .get("query_weight")
+ .and_then(Value::as_f64)
+ .unwrap_or(1.0) as f32;
+ let r_w = s_obj
+ .get("rescore_query_weight")
+ .and_then(Value::as_f64)
+ .unwrap_or(1.0) as f32;
+ let mode = s_obj
+ .get("score_mode")
+ .and_then(Value::as_str)
+ .map(String::from);
if !source.is_empty() {
script_opt = Some(ScriptRescore {
source,
@@ -3655,16 +3880,23 @@ fn parse_rescore(val: &Value) -> Vec<xerj_query::ast::RescoreQuery> {
fn collect_hdr_percentile_aggs(aggs: &Value, out: &mut Vec<(String, String, Value)>) {
let Some(obj) = aggs.as_object() else { return };
for (name, body) in obj {
- let Some(body_obj) = body.as_object() else { continue };
+ let Some(body_obj) = body.as_object() else {
+ continue;
+ };
for (agg_type, spec) in body_obj {
- if matches!(agg_type.as_str(), "aggs" | "aggregations" | "meta") { continue; }
+ if matches!(agg_type.as_str(), "aggs" | "aggregations" | "meta") {
+ continue;
+ }
if agg_type == "percentiles" && spec.get("hdr").is_some() {
if let Some(field) = spec.get("field").and_then(Value::as_str) {
out.push((name.clone(), field.to_string(), spec.clone()));
}
}
}
- if let Some(subs) = body_obj.get("aggs").or_else(|| body_obj.get("aggregations")) {
+ if let Some(subs) = body_obj
+ .get("aggs")
+ .or_else(|| body_obj.get("aggregations"))
+ {
collect_hdr_percentile_aggs(subs, out);
}
}
@@ -3675,21 +3907,37 @@ fn collect_hdr_percentile_aggs(aggs: &Value, out: &mut Vec<(String, String, Valu
fn source_numeric_values(source: &Value, field: &str) -> Vec<f64> {
fn push_num(v: &Value, out: &mut Vec<f64>) {
match v {
- Value::Number(n) => { if let Some(f) = n.as_f64() { out.push(f); } }
- Value::String(s) => { if let Ok(f) = s.parse::<f64>() { out.push(f); } }
- Value::Array(a) => { for x in a { push_num(x, out); } }
+ Value::Number(n) => {
+ if let Some(f) = n.as_f64() {
+ out.push(f);
+ }
+ }
+ Value::String(s) => {
+ if let Ok(f) = s.parse::<f64>() {
+ out.push(f);
+ }
+ }
+ Value::Array(a) => {
+ for x in a {
+ push_num(x, out);
+ }
+ }
_ => {}
}
}
let v = source.get(field).cloned().or_else(|| {
if field.contains('.') {
- source.pointer(&format!("/{}", field.replace('.', "/"))).cloned()
+ source
+ .pointer(&format!("/{}", field.replace('.', "/")))
+ .cloned()
} else {
None
}
});
let mut out = Vec::new();
- if let Some(v) = v { push_num(&v, &mut out); }
+ if let Some(v) = v {
+ push_num(&v, &mut out);
+ }
out
}
@@ -3712,28 +3960,39 @@ fn hdr_percentiles_values(vals: &[f64], spec: &Value) -> Value {
let sub_bucket_count: u64 = {
let target = 2u64 * 10u64.pow(digits);
let mut p = 1u64;
- while p < target { p <<= 1; }
+ while p < target {
+ p <<= 1;
+ }
p
};
let half_sub_bucket = sub_bucket_count / 2;
let hdr_quantize = |v: f64| -> f64 {
- if v <= 0.0 || !v.is_finite() { return v; }
+ if v <= 0.0 || !v.is_finite() {
+ return v;
+ }
let bucket_exp = v.log2().floor() as i32;
- if bucket_exp < 5 { return v; }
+ if bucket_exp < 5 {
+ return v;
+ }
let unit_size = (1u64 << bucket_exp.min(62) as u32) as f64;
v + (unit_size - 1.0) / (half_sub_bucket as f64)
};
let mut nums: Vec<f64> = vals.to_vec();
nums.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let compute = |pct: f64| -> Option<f64> {
- if nums.is_empty() { return None; }
+ if nums.is_empty() {
+ return None;
+ }
let n = nums.len();
let count_at = ((((pct / 100.0) * n as f64) + 0.5) as i64).max(1) as usize;
let mut cumulative = 0usize;
let mut pick = nums[n - 1];
for &v in nums.iter() {
cumulative += 1;
- if cumulative >= count_at { pick = v; break; }
+ if cumulative >= count_at {
+ pick = v;
+ break;
+ }
}
Some(hdr_quantize(pick))
};
@@ -3758,7 +4017,9 @@ fn hdr_percentiles_values(vals: &[f64], spec: &Value) -> Value {
.and_then(serde_json::Number::from_f64)
.map(Value::Number)
.unwrap_or(Value::Null);
- let key_num = serde_json::Number::from_f64(pct).map(Value::Number).unwrap_or(Value::Null);
+ let key_num = serde_json::Number::from_f64(pct)
+ .map(Value::Number)
+ .unwrap_or(Value::Null);
json!({ "key": key_num, "value": val })
})
.collect();
@@ -3772,9 +4033,7 @@ fn is_lone_doc_sort(v: &Value) -> bool {
match v {
Value::String(s) => s == "_doc",
Value::Array(a) if a.len() == 1 => is_lone_doc_sort(&a[0]),
- Value::Object(o) if o.len() == 1 => {
- o.keys().next().map(|k| k == "_doc").unwrap_or(false)
- }
+ Value::Object(o) if o.len() == 1 => o.keys().next().map(|k| k == "_doc").unwrap_or(false),
_ => false,
}
}
@@ -3785,7 +4044,7 @@ fn is_lone_doc_sort(v: &Value) -> bool {
/// - `"_score"` / `"_doc"` (string)
/// - `[{"field": "asc"}, {"field": {"order": "desc", "missing": "_last"}}]`
fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {
- use xerj_query::sort::{SortField, SortOrder, SortMode, SortMissing};
+ use xerj_query::sort::{SortField, SortMissing, SortMode, SortOrder};
let mut fields: Vec<SortField> = Vec::new();
// ES accepts sort in multiple shapes:
@@ -3847,18 +4106,39 @@ fn parse_sort(sort_val: &Value) -> Vec<xerj_query::sort::SortField> {
_ => SortMissing::Value(v.clone()),
})
.unwrap_or_default();
- let format = opts
- .get("format")
- .and_then(Value::as_str)
- .map(String::from);
+ let format =
+ opts.get("format").and_then(Value::as_str).map(String::from);
(order, mode, missing, format)
}
- _ => (SortOrder::Asc, SortMode::default(), SortMissing::default(), None),
+ _ => (
+ SortOrder::Asc,
+ SortMode::default(),
+ SortMissing::default(),
+ None,
+ ),
};
let sf = match field_name.as_str() {
- "_score" => SortField { field: "_score".to_string(), order, mode, missing, format },
- "_doc" => SortField { field: "_doc".to_string(), order, mode, missing, format },
- other => SortField { field: other.to_string(), order, mode, missing, format },
+ "_score" => SortField {
+ field: "_score".to_string(),
+ order,
+ mode,
+ missing,
+ format,
+ },
+ "_doc" => SortField {
+ field: "_doc".to_string(),
+ order,
+ mode,
+ missing,
+ format,
+ },
+ other => SortField {
+ field: other.to_string(),
+ order,
+ mode,
+ missing,
+ format,
+ },
};
fields.push(sf);
}
@@ -3895,12 +4175,16 @@ fn parse_sort_mode(s: &str) -> xerj_query::sort::SortMode {
/// spec; the singular is a convenience for our internal AST). When
/// the array form is used we take the first element.
fn parse_highlight(hl_val: &Value) -> Option<xerj_query::ast::HighlightRequest> {
- use xerj_query::ast::{HighlightRequest, HighlightFieldOptions};
+ use xerj_query::ast::{HighlightFieldOptions, HighlightRequest};
let obj = hl_val.as_object()?;
let fields_val = obj.get("fields")?.as_object()?;
- fn pull_tag(obj: &serde_json::Map<String, Value>, plural: &str, singular: &str) -> Option<String> {
+ fn pull_tag(
+ obj: &serde_json::Map<String, Value>,
+ plural: &str,
+ singular: &str,
+ ) -> Option<String> {
obj.get(plural)
.and_then(|v| v.as_array())
.and_then(|a| a.first())
@@ -3915,8 +4199,14 @@ fn parse_highlight(hl_val: &Value) -> Option<xerj_query::ast::HighlightRequest>
HighlightFieldOptions {
pre_tag: pull_tag(o, "pre_tags", "pre_tag"),
post_tag: pull_tag(o, "post_tags", "post_tag"),
- fragment_size: o.get("fragment_size").and_then(Value::as_u64).map(|n| n as usize),
- number_of_fragments: o.get("number_of_fragments").and_then(Value::as_u64).map(|n| n as usize),
+ fragment_size: o
+ .get("fragment_size")
+ .and_then(Value::as_u64)
+ .map(|n| n as usize),
+ number_of_fragments: o
+ .get("number_of_fragments")
+ .and_then(Value::as_u64)
+ .map(|n| n as usize),
}
} else {
HighlightFieldOptions::default()
@@ -3928,8 +4218,14 @@ fn parse_highlight(hl_val: &Value) -> Option<xerj_query::ast::HighlightRequest>
fields,
pre_tag: pull_tag(obj, "pre_tags", "pre_tag"),
post_tag: pull_tag(obj, "post_tags", "post_tag"),
- fragment_size: obj.get("fragment_size").and_then(Value::as_u64).map(|n| n as usize),
- number_of_fragments: obj.get("number_of_fragments").and_then(Value::as_u64).map(|n| n as usize),
+ fragment_size: obj
+ .get("fragment_size")
+ .and_then(Value::as_u64)
+ .map(|n| n as usize),
+ number_of_fragments: obj
+ .get("number_of_fragments")
+ .and_then(Value::as_u64)
+ .map(|n| n as usize),
})
}
@@ -4039,7 +4335,9 @@ pub async fn search(
// URL parameter or a body field). Merge into the body so the
// downstream search-request builder sees it uniformly.
if !body.explain {
- body.explain = params.explain.as_deref()
+ body.explain = params
+ .explain
+ .as_deref()
.map(|v| v == "true" || v == "1")
.unwrap_or(false);
}
@@ -4057,9 +4355,18 @@ pub async fn search(
};
if matches!(
name,
- "_seq_no" | "_source" | "_feature" | "_nested_path"
- | "_field_names" | "_version" | "_routing" | "_id"
- | "_ignored" | "_type" | "_parent" | "_index"
+ "_seq_no"
+ | "_source"
+ | "_feature"
+ | "_nested_path"
+ | "_field_names"
+ | "_version"
+ | "_routing"
+ | "_id"
+ | "_ignored"
+ | "_type"
+ | "_parent"
+ | "_index"
| "_primary_term"
) {
// ES allows fetching metadata fields with real doc_values
@@ -4067,7 +4374,10 @@ pub async fn search(
// _primary_term, _source, _field_names, _feature,
// _nested_path, _type, _parent are NOT fetchable — ES
// returns illegal_argument_exception 400 for those.
- if matches!(name, "_id" | "_index" | "_version" | "_ignored" | "_routing") {
+ if matches!(
+ name,
+ "_id" | "_index" | "_version" | "_ignored" | "_routing"
+ ) {
continue;
}
let reason = format!(
@@ -4087,7 +4397,8 @@ pub async fn search(
},
"status": 400,
})),
- ).into_response();
+ )
+ .into_response();
}
}
}
@@ -4169,7 +4480,8 @@ pub async fn search(
body.track_total_hits = Some(match tth_str.as_str() {
"false" => Value::Bool(false),
"true" => Value::Bool(true),
- other => other.parse::<u64>()
+ other => other
+ .parse::<u64>()
.map(|n| json!(n))
.unwrap_or(Value::Bool(true)),
});
@@ -4303,12 +4615,16 @@ pub async fn search(
// recorded indices from the PIT snapshot. The path-level `index`
// for PIT searches is typically `_search` (no index in the URL);
// the PIT context provides the real index list.
- let pit_context: Option<xerj_engine::engine::PitContext> = body.pit.as_ref()
+ let pit_context: Option<xerj_engine::engine::PitContext> = body
+ .pit
+ .as_ref()
.and_then(|p| p.get("id").and_then(Value::as_str))
.and_then(|id| state.engine.pits.get(id).map(|r| r.value().clone()));
let index_names: Vec<String> = if let Some(pit) = pit_context.as_ref() {
pit.indices.clone()
- } else { index_names };
+ } else {
+ index_names
+ };
// `match: {_index: pattern}` / `term: {_index: pattern}` filters the
// index_names list to just those matching the pattern. ES treats
@@ -4325,9 +4641,14 @@ pub async fn search(
let index_names: Vec<String> = if index_name_filter_patterns.is_empty() {
index_names
} else {
- index_names.into_iter().filter(|n| {
- index_name_filter_patterns.iter().any(|p| p == n || glob_match(p, n))
- }).collect()
+ index_names
+ .into_iter()
+ .filter(|n| {
+ index_name_filter_patterns
+ .iter()
+ .any(|p| p == n || glob_match(p, n))
+ })
+ .collect()
};
let index_names: Vec<&str> = index_names.iter().map(|s| s.as_str()).collect();
// ── Closed-index resolution (ES expand_wildcards / ignore_unavailable) ──
@@ -4407,23 +4728,23 @@ pub async fn search(
// brute-force match count over the whole index. Capture that cap here,
// before the knn query is folded into a `bool` and `body.size` is bumped.
// Hybrid knn (knn + a sibling `query`) keeps normal scoring/counting.
- let knn_total_cap: Option<usize> = body
- .knn
- .as_ref()
- .filter(|_| body.query.is_none())
- .map(|knn_val| {
- let num_candidates = knn_val
- .get("num_candidates")
- .and_then(Value::as_u64)
- .map(|n| n as usize);
- let k = knn_val
- .get("k")
- .and_then(Value::as_u64)
- .map(|n| n as usize)
- .or(num_candidates)
- .unwrap_or(10);
- k.min(body.size)
- });
+ let knn_total_cap: Option<usize> =
+ body.knn
+ .as_ref()
+ .filter(|_| body.query.is_none())
+ .map(|knn_val| {
+ let num_candidates = knn_val
+ .get("num_candidates")
+ .and_then(Value::as_u64)
+ .map(|n| n as usize);
+ let k = knn_val
+ .get("k")
+ .and_then(Value::as_u64)
+ .map(|n| n as usize)
+ .or(num_candidates)
+ .unwrap_or(10);
+ k.min(body.size)
+ });
// If top-level "knn" is present, synthesise a Knn query and merge with any "query".
let effective_body: EsSearchBody = if let Some(ref knn_val) = body.knn {
@@ -4462,12 +4783,25 @@ pub async fn search(
// stored in settings at create / reopen time.
if index_names.len() == 1 {
let idx_name = index_names[0];
- let (sort_field, sort_order, explicit) = state.engine.index_settings.get(idx_name)
- .map(|r| {
- let s = r.value();
- let f = s.get("__xy_index_sort_field").and_then(Value::as_str).map(str::to_string);
- let o = s.get("__xy_index_sort_order").and_then(Value::as_str).unwrap_or("desc").to_string();
- let ex = s.get("__xy_index_sort_explicit").and_then(Value::as_bool).unwrap_or(false);
+ let (sort_field, sort_order, explicit) = state
+ .engine
+ .index_settings
+ .get(idx_name)
+ .map(|r| {
+ let s = r.value();
+ let f = s
+ .get("__xy_index_sort_field")
+ .and_then(Value::as_str)
+ .map(str::to_string);
+ let o = s
+ .get("__xy_index_sort_order")
+ .and_then(Value::as_str)
+ .unwrap_or("desc")
+ .to_string();
+ let ex = s
+ .get("__xy_index_sort_explicit")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
(f, o, ex)
})
.unwrap_or((None, "desc".to_string(), false));
@@ -4509,8 +4843,7 @@ pub async fn search(
.unwrap_or(false);
let apply = (explicit || implicit_ok)
&& (body.sort.is_none()
- || (explicit
- && body.sort.as_ref().map(is_lone_doc_sort).unwrap_or(false)));
+ || (explicit && body.sort.as_ref().map(is_lone_doc_sort).unwrap_or(false)));
if apply {
body.sort = Some(json!([{ f: { "order": sort_order } }]));
}
@@ -4531,18 +4864,30 @@ pub async fn search(
// passthrough priority so higher priority wins. A concrete root
// field with the same name as a passthrough sub-field always
// shadows the passthrough alias (per ES rules).
- let mut pass_map: std::collections::HashMap<String, (String, i64)> = std::collections::HashMap::new();
- let mut concrete_roots: std::collections::HashSet<String> = std::collections::HashSet::new();
+ let mut pass_map: std::collections::HashMap<String, (String, i64)> =
+ std::collections::HashMap::new();
+ let mut concrete_roots: std::collections::HashSet<String> =
+ std::collections::HashSet::new();
for ix in &index_names {
- let Some(m) = state.engine.index_mappings.get(*ix) else { continue };
+ let Some(m) = state.engine.index_mappings.get(*ix) else {
+ continue;
+ };
let mapping = m.clone();
- let props = mapping.get("mappings").and_then(|mm| mm.get("properties")).or_else(|| mapping.get("properties"));
- let Some(pobj) = props.and_then(Value::as_object) else { continue };
+ let props = mapping
+ .get("mappings")
+ .and_then(|mm| mm.get("properties"))
+ .or_else(|| mapping.get("properties"));
+ let Some(pobj) = props.and_then(Value::as_object) else {
+ continue;
+ };
// First pass: record every concrete root-level field.
for (name, spec) in pobj {
let pt = spec.get("type").and_then(Value::as_str);
let is_passthrough_root = pt == Some("passthrough")
- || spec.get("passthrough").and_then(|p| p.get("priority")).is_some();
+ || spec
+ .get("passthrough")
+ .and_then(|p| p.get("priority"))
+ .is_some();
if !is_passthrough_root {
concrete_roots.insert(name.clone());
}
@@ -4551,15 +4896,28 @@ pub async fn search(
for (parent_name, parent_spec) in pobj {
let pt = parent_spec.get("type").and_then(Value::as_str);
let is_passthrough = pt == Some("passthrough");
- let priority = parent_spec.get("passthrough").and_then(|p| p.get("priority")).and_then(Value::as_i64)
- .or_else(|| if is_passthrough { parent_spec.get("priority").and_then(Value::as_i64) } else { None });
+ let priority = parent_spec
+ .get("passthrough")
+ .and_then(|p| p.get("priority"))
+ .and_then(Value::as_i64)
+ .or_else(|| {
+ if is_passthrough {
+ parent_spec.get("priority").and_then(Value::as_i64)
+ } else {
+ None
+ }
+ });
let Some(prio) = priority else { continue };
let sub_props = parent_spec.get("properties").and_then(Value::as_object);
let Some(sp) = sub_props else { continue };
for (sub_name, _spec) in sp {
let parent_path = parent_name.clone();
- let entry = pass_map.entry(sub_name.clone()).or_insert((parent_path.clone(), prio));
- if entry.1 < prio { *entry = (parent_path, prio); }
+ let entry = pass_map
+ .entry(sub_name.clone())
+ .or_insert((parent_path.clone(), prio));
+ if entry.1 < prio {
+ *entry = (parent_path, prio);
+ }
}
}
}
@@ -4568,10 +4926,25 @@ pub async fn search(
pass_map.remove(n);
}
if !pass_map.is_empty() {
- fn rewrite_query(q: &mut Value, pass: &std::collections::HashMap<String, (String, i64)>) {
+ fn rewrite_query(
+ q: &mut Value,
+ pass: &std::collections::HashMap<String, (String, i64)>,
+ ) {
match q {
Value::Object(obj) => {
- for clause_key in ["term", "terms", "match", "match_phrase", "range", "prefix", "wildcard", "fuzzy", "regexp", "exists", "match_phrase_prefix"] {
+ for clause_key in [
+ "term",
+ "terms",
+ "match",
+ "match_phrase",
+ "range",
+ "prefix",
+ "wildcard",
+ "fuzzy",
+ "regexp",
+ "exists",
+ "match_phrase_prefix",
+ ] {
if let Some(Value::Object(inner)) = obj.get_mut(clause_key) {
let keys: Vec<String> = inner.keys().cloned().collect();
for k in keys {
@@ -4585,9 +4958,15 @@ pub async fn search(
}
}
}
- for (_, child) in obj.iter_mut() { rewrite_query(child, pass); }
+ for (_, child) in obj.iter_mut() {
+ rewrite_query(child, pass);
+ }
+ }
+ Value::Array(arr) => {
+ for item in arr.iter_mut() {
+ rewrite_query(item, pass);
+ }
}
- Value::Array(arr) => { for item in arr.iter_mut() { rewrite_query(item, pass); } }
_ => {}
}
}
@@ -4596,7 +4975,10 @@ pub async fn search(
}
// Also rewrite sort fields that reference passthrough roots.
if let Some(sort) = body.sort.as_mut() {
- fn rewrite_sort(v: &mut Value, pass: &std::collections::HashMap<String, (String, i64)>) {
+ fn rewrite_sort(
+ v: &mut Value,
+ pass: &std::collections::HashMap<String, (String, i64)>,
+ ) {
match v {
Value::String(s) => {
if let Some((parent, _)) = pass.get(s.as_str()) {
@@ -4613,7 +4995,11 @@ pub async fn search(
}
}
}
- Value::Array(arr) => { for item in arr.iter_mut() { rewrite_sort(item, pass); } }
+ Value::Array(arr) => {
+ for item in arr.iter_mut() {
+ rewrite_sort(item, pass);
+ }
+ }
_ => {}
}
}
@@ -4631,10 +5017,13 @@ pub async fn search(
let mut alias_filters: Vec<Value> = Vec::new();
for name in &index_names {
for entry in state.engine.aliases.iter() {
- if entry.key() != name { continue; }
+ if entry.key() != name {
+ continue;
+ }
for backing in entry.value().iter() {
if let Some(meta) = state.engine.index_alias_metadata.get(backing) {
- if let Some(filter) = meta.get(*name).and_then(|v| v.get("filter")).cloned() {
+ if let Some(filter) = meta.get(*name).and_then(|v| v.get("filter")).cloned()
+ {
alias_filters.push(filter);
}
}
@@ -4642,7 +5031,10 @@ pub async fn search(
}
}
if !alias_filters.is_empty() {
- let original = body.query.clone().unwrap_or_else(|| json!({"match_all": {}}));
+ let original = body
+ .query
+ .clone()
+ .unwrap_or_else(|| json!({"match_all": {}}));
// ES semantics: alias filters are `filter` clauses (unscored,
// pre-filter applied before the main query). This matters for
// knn queries where `bool.filter` acts as a pre-filter but
@@ -4677,15 +5069,25 @@ pub async fn search(
}
let keys: Vec<String> = obj.keys().cloned().collect();
for key in keys {
- if let Some(child) = obj.get_mut(&key) { inject_bg_filter(child, filter); }
+ if let Some(child) = obj.get_mut(&key) {
+ inject_bg_filter(child, filter);
+ }
+ }
+ }
+ Value::Array(arr) => {
+ for item in arr {
+ inject_bg_filter(item, filter);
}
}
- Value::Array(arr) => { for item in arr { inject_bg_filter(item, filter); } }
_ => {}
}
}
- if let Some(aggs) = body.aggs.as_mut() { inject_bg_filter(aggs, &combined); }
- if let Some(aggs) = body.aggregations.as_mut() { inject_bg_filter(aggs, &combined); }
+ if let Some(aggs) = body.aggs.as_mut() {
+ inject_bg_filter(aggs, &combined);
+ }
+ if let Some(aggs) = body.aggregations.as_mut() {
+ inject_bg_filter(aggs, &combined);
+ }
}
}
@@ -4713,17 +5115,27 @@ pub async fn search(
// `match string_field: foo` on the canonical filters_bucket test
// also matches docs with `string_field: "foo bar"`.
{
- let mut keyword_fields: std::collections::HashSet<String> = std::collections::HashSet::new();
- let mut split_kw_fields: std::collections::HashSet<String> = std::collections::HashSet::new();
+ let mut keyword_fields: std::collections::HashSet<String> =
+ std::collections::HashSet::new();
+ let mut split_kw_fields: std::collections::HashSet<String> =
+ std::collections::HashSet::new();
for ix in &index_names {
if let Some(m) = state.engine.index_mappings.get(*ix) {
let mapping = m.clone();
- let props = mapping.get("mappings").and_then(|mm| mm.get("properties")).or_else(|| mapping.get("properties")).cloned();
+ let props = mapping
+ .get("mappings")
+ .and_then(|mm| mm.get("properties"))
+ .or_else(|| mapping.get("properties"))
+ .cloned();
if let Some(Value::Object(props)) = props {
for (name, spec) in props {
if spec.get("type").and_then(Value::as_str) == Some("keyword") {
keyword_fields.insert(name.clone());
- if spec.get("split_queries_on_whitespace").and_then(Value::as_bool).unwrap_or(false) {
+ if spec
+ .get("split_queries_on_whitespace")
+ .and_then(Value::as_bool)
+ .unwrap_or(false)
+ {
split_kw_fields.insert(name);
}
}
@@ -4732,7 +5144,11 @@ pub async fn search(
}
}
if !keyword_fields.is_empty() {
- fn rewrite(v: &mut Value, kw: &std::collections::HashSet<String>, split: &std::collections::HashSet<String>) {
+ fn rewrite(
+ v: &mut Value,
+ kw: &std::collections::HashSet<String>,
+ split: &std::collections::HashSet<String>,
+ ) {
match v {
Value::Object(obj) => {
let mut new_node: Option<(String, Value)> = None;
@@ -4740,7 +5156,9 @@ pub async fn search(
if let Some((field, raw)) = m_val.iter().next() {
if kw.contains(field) {
let value = match raw {
- Value::Object(inner) => inner.get("query").cloned().unwrap_or(raw.clone()),
+ Value::Object(inner) => {
+ inner.get("query").cloned().unwrap_or(raw.clone())
+ }
_ => raw.clone(),
};
let s_owned = match &value {
@@ -4754,16 +5172,27 @@ pub async fn search(
let toks: Vec<&str> = s.split_whitespace().collect();
if toks.len() > 1 {
// Multi-token query → terms (OR over tokens).
- let arr: Vec<Value> = toks.iter().map(|t| Value::String(t.to_string())).collect();
+ let arr: Vec<Value> = toks
+ .iter()
+ .map(|t| Value::String(t.to_string()))
+ .collect();
let mut terms = serde_json::Map::new();
terms.insert(field.clone(), Value::Array(arr));
- new_node = Some(("terms".to_string(), Value::Object(terms)));
+ new_node = Some((
+ "terms".to_string(),
+ Value::Object(terms),
+ ));
} else {
// Single token → term.
let mut term = serde_json::Map::new();
- let v = if toks.is_empty() { Value::String(String::new()) } else { Value::String(toks[0].to_string()) };
+ let v = if toks.is_empty() {
+ Value::String(String::new())
+ } else {
+ Value::String(toks[0].to_string())
+ };
term.insert(field.clone(), v);
- new_node = Some(("term".to_string(), Value::Object(term)));
+ new_node =
+ Some(("term".to_string(), Value::Object(term)));
}
}
} else {
@@ -4780,15 +5209,29 @@ pub async fn search(
obj.insert(tag, body);
}
let keys: Vec<String> = obj.keys().cloned().collect();
- for k in keys { if let Some(c) = obj.get_mut(&k) { rewrite(c, kw, split); } }
+ for k in keys {
+ if let Some(c) = obj.get_mut(&k) {
+ rewrite(c, kw, split);
+ }
+ }
+ }
+ Value::Array(arr) => {
+ for item in arr {
+ rewrite(item, kw, split);
+ }
}
- Value::Array(arr) => { for item in arr { rewrite(item, kw, split); } }
_ => {}
}
}
- if let Some(q) = body.query.as_mut() { rewrite(q, &keyword_fields, &split_kw_fields); }
- if let Some(aggs) = body.aggs.as_mut() { rewrite(aggs, &keyword_fields, &split_kw_fields); }
- if let Some(aggs) = body.aggregations.as_mut() { rewrite(aggs, &keyword_fields, &split_kw_fields); }
+ if let Some(q) = body.query.as_mut() {
+ rewrite(q, &keyword_fields, &split_kw_fields);
+ }
+ if let Some(aggs) = body.aggs.as_mut() {
+ rewrite(aggs, &keyword_fields, &split_kw_fields);
+ }
+ if let Some(aggs) = body.aggregations.as_mut() {
+ rewrite(aggs, &keyword_fields, &split_kw_fields);
+ }
}
}
@@ -4801,7 +5244,11 @@ pub async fn search(
for ix in &index_names {
if let Some(m) = state.engine.index_mappings.get(*ix) {
let mapping = m.clone();
- let props = mapping.get("mappings").and_then(|mm| mm.get("properties")).or_else(|| mapping.get("properties")).cloned();
+ let props = mapping
+ .get("mappings")
+ .and_then(|mm| mm.get("properties"))
+ .or_else(|| mapping.get("properties"))
+ .cloned();
if let Some(Value::Object(props)) = props {
for (name, spec) in props {
if spec.get("type").and_then(Value::as_str) == Some("float") {
@@ -4815,27 +5262,55 @@ pub async fn search(
fn annotate(v: &mut Value, floats: &std::collections::HashSet<String>) {
match v {
Value::Object(obj) => {
- if let Some(ms) = obj.get_mut("matrix_stats").and_then(|x| x.as_object_mut()) {
+ if let Some(ms) =
+ obj.get_mut("matrix_stats").and_then(|x| x.as_object_mut())
+ {
if !ms.contains_key("__xy_f32_fields__") {
- let fields = ms.get("fields").and_then(Value::as_array).cloned().unwrap_or_default();
- let f32_list: Vec<Value> = fields.into_iter().filter_map(|v| {
- let s = v.as_str()?.to_string();
- if floats.contains(&s) { Some(Value::String(s)) } else { None }
- }).collect();
+ let fields = ms
+ .get("fields")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default();
+ let f32_list: Vec<Value> = fields
+ .into_iter()
+ .filter_map(|v| {
+ let s = v.as_str()?.to_string();
+ if floats.contains(&s) {
+ Some(Value::String(s))
+ } else {
+ None
+ }
+ })
+ .collect();
if !f32_list.is_empty() {
- ms.insert("__xy_f32_fields__".to_string(), Value::Array(f32_list));
+ ms.insert(
+ "__xy_f32_fields__".to_string(),
+ Value::Array(f32_list),
+ );
}
}
}
let keys: Vec<String> = obj.keys().cloned().collect();
- for k in keys { if let Some(c) = obj.get_mut(&k) { annotate(c, floats); } }
+ for k in keys {
+ if let Some(c) = obj.get_mut(&k) {
+ annotate(c, floats);
+ }
+ }
+ }
+ Value::Array(arr) => {
+ for item in arr {
+ annotate(item, floats);
+ }
}
- Value::Array(arr) => { for item in arr { annotate(item, floats); } }
_ => {}
}
}
- if let Some(aggs) = body.aggs.as_mut() { annotate(aggs, &all_floats); }
- if let Some(aggs) = body.aggregations.as_mut() { annotate(aggs, &all_floats); }
+ if let Some(aggs) = body.aggs.as_mut() {
+ annotate(aggs, &all_floats);
+ }
+ if let Some(aggs) = body.aggregations.as_mut() {
+ annotate(aggs, &all_floats);
+ }
}
}
@@ -4861,19 +5336,32 @@ pub async fn search(
let lookup_format = move |field: &str| -> Option<String> {
let fspec = props.get(field)?;
let ftype = fspec.get("type").and_then(Value::as_str)?;
- if !matches!(ftype, "date" | "date_nanos") { return None; }
- fspec.get("format").and_then(Value::as_str).map(String::from)
+ if !matches!(ftype, "date" | "date_nanos") {
+ return None;
+ }
+ fspec
+ .get("format")
+ .and_then(Value::as_str)
+ .map(String::from)
};
fn inject_format(v: &mut Value, lookup: &dyn Fn(&str) -> Option<String>) {
match v {
Value::Object(obj) => {
for key in ["date_histogram", "date_range", "range"].iter() {
- if let Some(body) = obj.get_mut(*key).and_then(|x| x.as_object_mut()) {
+ if let Some(body) =
+ obj.get_mut(*key).and_then(|x| x.as_object_mut())
+ {
if !body.contains_key("format") {
- let field = body.get("field").and_then(Value::as_str).map(String::from);
+ let field = body
+ .get("field")
+ .and_then(Value::as_str)
+ .map(String::from);
if let Some(f) = field {
if let Some(fmt) = lookup(&f) {
- body.insert("format".to_string(), Value::String(fmt));
+ body.insert(
+ "format".to_string(),
+ Value::String(fmt),
+ );
}
}
}
@@ -4881,15 +5369,25 @@ pub async fn search(
}
let keys: Vec<String> = obj.keys().cloned().collect();
for k in keys {
- if let Some(child) = obj.get_mut(&k) { inject_format(child, lookup); }
+ if let Some(child) = obj.get_mut(&k) {
+ inject_format(child, lookup);
+ }
+ }
+ }
+ Value::Array(arr) => {
+ for item in arr {
+ inject_format(item, lookup);
}
}
- Value::Array(arr) => { for item in arr { inject_format(item, lookup); } }
_ => {}
}
}
- if let Some(aggs) = body.aggs.as_mut() { inject_format(aggs, &lookup_format); }
- if let Some(aggs) = body.aggregations.as_mut() { inject_format(aggs, &lookup_format); }
+ if let Some(aggs) = body.aggs.as_mut() {
+ inject_format(aggs, &lookup_format);
+ }
+ if let Some(aggs) = body.aggregations.as_mut() {
+ inject_format(aggs, &lookup_format);
+ }
}
}
}
@@ -4915,7 +5413,10 @@ pub async fn search(
// `fields` into that top_hits params map.
let user_names: Vec<String> = obj.keys().cloned().collect();
for user_name in &user_names {
- let Some(agg_body) = obj.get_mut(user_name).and_then(|v| v.as_object_mut()) else { continue };
+ let Some(agg_body) = obj.get_mut(user_name).and_then(|v| v.as_object_mut())
+ else {
+ continue;
+ };
if let Some(Value::Object(th_params)) = agg_body.get_mut("top_hits") {
if !th_params.contains_key("fields") {
th_params.insert("fields".to_string(), fields.clone());
@@ -4931,11 +5432,17 @@ pub async fn search(
}
}
if let Value::Array(arr) = v {
- for item in arr { inject_top_hits_fields(item, fields); }
+ for item in arr {
+ inject_top_hits_fields(item, fields);
+ }
}
}
- if let Some(aggs) = body.aggs.as_mut() { inject_top_hits_fields(aggs, &top_fields); }
- if let Some(aggs) = body.aggregations.as_mut() { inject_top_hits_fields(aggs, &top_fields); }
+ if let Some(aggs) = body.aggs.as_mut() {
+ inject_top_hits_fields(aggs, &top_fields);
+ }
+ if let Some(aggs) = body.aggregations.as_mut() {
+ inject_top_hits_fields(aggs, &top_fields);
+ }
}
// Propagate `runtime_mappings` into every top_hits sub-agg's
@@ -4948,7 +5455,10 @@ pub async fn search(
if let Value::Object(obj) = v {
let user_names: Vec<String> = obj.keys().cloned().collect();
for user_name in &user_names {
- let Some(agg_body) = obj.get_mut(user_name).and_then(|v| v.as_object_mut()) else { continue };
+ let Some(agg_body) = obj.get_mut(user_name).and_then(|v| v.as_object_mut())
+ else {
+ continue;
+ };
if let Some(Value::Object(th_params)) = agg_body.get_mut("top_hits") {
if !th_params.contains_key("runtime_mappings") {
th_params.insert("runtime_mappings".to_string(), rm.clone());
@@ -4961,10 +5471,18 @@ pub async fn search(
}
}
}
- if let Value::Array(arr) = v { for item in arr { inject_top_hits_runtime(item, rm); } }
+ if let Value::Array(arr) = v {
+ for item in arr {
+ inject_top_hits_runtime(item, rm);
+ }
+ }
+ }
+ if let Some(aggs) = body.aggs.as_mut() {
+ inject_top_hits_runtime(aggs, &rm);
+ }
+ if let Some(aggs) = body.aggregations.as_mut() {
+ inject_top_hits_runtime(aggs, &rm);
}
- if let Some(aggs) = body.aggs.as_mut() { inject_top_hits_runtime(aggs, &rm); }
- if let Some(aggs) = body.aggregations.as_mut() { inject_top_hits_runtime(aggs, &rm); }
}
// When any participating index has `_source.enabled: false` in its
@@ -4972,11 +5490,16 @@ pub async fn search(
// the top_hits response doesn't contain the internal source clone
// we still keep around for fields/highlight/etc.
let source_disabled_somewhere = index_names.iter().any(|ix| {
- state.engine.index_mappings.get(*ix)
- .and_then(|m| m.get("mappings")
- .and_then(|mm| mm.get("_source"))
- .or_else(|| m.get("_source"))
- .cloned())
+ state
+ .engine
+ .index_mappings
+ .get(*ix)
+ .and_then(|m| {
+ m.get("mappings")
+ .and_then(|mm| mm.get("_source"))
+ .or_else(|| m.get("_source"))
+ .cloned()
+ })
.and_then(|src| src.get("enabled").and_then(Value::as_bool))
.map(|b| !b)
.unwrap_or(false)
@@ -4986,7 +5509,10 @@ pub async fn search(
if let Value::Object(obj) = v {
let user_names: Vec<String> = obj.keys().cloned().collect();
for user_name in &user_names {
- let Some(agg_body) = obj.get_mut(user_name).and_then(|v| v.as_object_mut()) else { continue };
+ let Some(agg_body) = obj.get_mut(user_name).and_then(|v| v.as_object_mut())
+ else {
+ continue;
+ };
if let Some(Value::Object(th_params)) = agg_body.get_mut("top_hits") {
if !th_params.contains_key("_source") {
th_params.insert("_source".to_string(), Value::Bool(false));
@@ -5000,8 +5526,12 @@ pub async fn search(
}
}
}
- if let Some(aggs) = body.aggs.as_mut() { inject_top_hits_no_source(aggs); }
- if let Some(aggs) = body.aggregations.as_mut() { inject_top_hits_no_source(aggs); }
+ if let Some(aggs) = body.aggs.as_mut() {
+ inject_top_hits_no_source(aggs);
+ }
+ if let Some(aggs) = body.aggregations.as_mut() {
+ inject_top_hits_no_source(aggs);
+ }
}
// Resolve stored script references inside scripted_metric scripts.
@@ -5020,10 +5550,7 @@ pub async fn search(
// (the field is materially unqueryable). Do this against each
// participating index's mapping, applying the `match_none`
// substitution when the field is unqueryable in *every* index.
- let participating_indices: Vec<String> = index_names
- .iter()
- .map(|s| s.to_string())
- .collect();
+ let participating_indices: Vec<String> = index_names.iter().map(|s| s.to_string()).collect();
if let Some(q) = body.query.as_mut() {
rewrite_unqueryable_exists(q, &state, &participating_indices);
}
@@ -5038,10 +5565,18 @@ pub async fn search(
let mut m = std::collections::HashMap::new();
for ix in &participating_indices {
if let Some(mapping) = state.engine.index_mappings.get(ix.as_str()) {
- fn collect_aliases(props: &Value, prefix: &str, out: &mut std::collections::HashMap<String, String>) {
+ fn collect_aliases(
+ props: &Value,
+ prefix: &str,
+ out: &mut std::collections::HashMap<String, String>,
+ ) {
let Some(obj) = props.as_object() else { return };
for (name, spec) in obj {
- let full = if prefix.is_empty() { name.clone() } else { format!("{prefix}.{name}") };
+ let full = if prefix.is_empty() {
+ name.clone()
+ } else {
+ format!("{prefix}.{name}")
+ };
if spec.get("type").and_then(Value::as_str) == Some("alias") {
if let Some(path) = spec.get("path").and_then(Value::as_str) {
out.entry(full.clone()).or_insert_with(|| path.to_string());
@@ -5052,7 +5587,9 @@ pub async fn search(
}
}
}
- let props = mapping.pointer("/mappings/properties").or_else(|| mapping.pointer("/properties"));
+ let props = mapping
+ .pointer("/mappings/properties")
+ .or_else(|| mapping.pointer("/properties"));
if let Some(p) = props {
collect_aliases(p, "", &mut m);
}
@@ -5062,10 +5599,16 @@ pub async fn search(
};
if !alias_map.is_empty() {
// Rewrite field names in the body's interesting shapes.
- fn rewrite_field_name(s: &str, aliases: &std::collections::HashMap<String, String>) -> String {
+ fn rewrite_field_name(
+ s: &str,
+ aliases: &std::collections::HashMap<String, String>,
+ ) -> String {
aliases.get(s).cloned().unwrap_or_else(|| s.to_string())
}
- fn rewrite_leaf_field_values(v: &mut Value, aliases: &std::collections::HashMap<String, String>) {
+ fn rewrite_leaf_field_values(
+ v: &mut Value,
+ aliases: &std::collections::HashMap<String, String>,
+ ) {
match v {
Value::Object(obj) => {
// Special handling: for `term/range/match/exists/prefix/etc.`
@@ -5087,18 +5630,30 @@ pub async fn search(
*f = Value::String(rewrite_field_name(s, aliases));
}
}
- for (_, child) in obj.iter_mut() { rewrite_leaf_field_values(child, aliases); }
+ for (_, child) in obj.iter_mut() {
+ rewrite_leaf_field_values(child, aliases);
+ }
}
Value::Array(arr) => {
- for item in arr { rewrite_leaf_field_values(item, aliases); }
+ for item in arr {
+ rewrite_leaf_field_values(item, aliases);
+ }
}
_ => {}
}
}
- if let Some(q) = body.query.as_mut() { rewrite_leaf_field_values(q, &alias_map); }
- if let Some(s) = body.sort.as_mut() { rewrite_leaf_field_values(s, &alias_map); }
- if let Some(a) = body.aggs.as_mut() { rewrite_leaf_field_values(a, &alias_map); }
- if let Some(a) = body.aggregations.as_mut() { rewrite_leaf_field_values(a, &alias_map); }
+ if let Some(q) = body.query.as_mut() {
+ rewrite_leaf_field_values(q, &alias_map);
+ }
+ if let Some(s) = body.sort.as_mut() {
+ rewrite_leaf_field_values(s, &alias_map);
+ }
+ if let Some(a) = body.aggs.as_mut() {
+ rewrite_leaf_field_values(a, &alias_map);
+ }
+ if let Some(a) = body.aggregations.as_mut() {
+ rewrite_leaf_field_values(a, &alias_map);
+ }
// Note: we intentionally do NOT rewrite `body.fields` or
// `body.collapse` — both are per-index-dependent. The response
// layer applies alias resolution at fetch time (`body.fields`)
@@ -5111,15 +5666,28 @@ pub async fn search(
// reject queries/sorts that touch `_seq_no` with ES's specific
// error messages.
let any_disabled_seqno = participating_indices.iter().any(|ix| {
- state.engine.index_settings.get(ix).map(|v| {
- let s = v.clone();
- let as_bool = |val: &Value| -> bool {
- val.as_bool().unwrap_or_else(|| val.as_str().map(|x| x == "true").unwrap_or(false))
- };
- s.pointer("/index/disable_sequence_numbers").map(as_bool).unwrap_or(false)
- || s.get("index").and_then(|i| i.get("index.disable_sequence_numbers")).map(as_bool).unwrap_or(false)
- || s.get("index.disable_sequence_numbers").map(as_bool).unwrap_or(false)
- }).unwrap_or(false)
+ state
+ .engine
+ .index_settings
+ .get(ix)
+ .map(|v| {
+ let s = v.clone();
+ let as_bool = |val: &Value| -> bool {
+ val.as_bool()
+ .unwrap_or_else(|| val.as_str().map(|x| x == "true").unwrap_or(false))
+ };
+ s.pointer("/index/disable_sequence_numbers")
+ .map(as_bool)
+ .unwrap_or(false)
+ || s.get("index")
+ .and_then(|i| i.get("index.disable_sequence_numbers"))
+ .map(as_bool)
+ .unwrap_or(false)
+ || s.get("index.disable_sequence_numbers")
+ .map(as_bool)
+ .unwrap_or(false)
+ })
+ .unwrap_or(false)
});
if any_disabled_seqno {
fn mentions_seq_no(v: &Value) -> bool {
@@ -5128,7 +5696,9 @@ pub async fn search(
// `term: {_seq_no: ...}` / `range: {_seq_no: ...}`
for leaf in ["term", "range", "terms", "match"] {
if let Some(body) = obj.get(leaf).and_then(|x| x.as_object()) {
- if body.contains_key("_seq_no") { return true; }
+ if body.contains_key("_seq_no") {
+ return true;
+ }
}
}
obj.values().any(mentions_seq_no)
@@ -5140,7 +5710,8 @@ pub async fn search(
if let Some(q) = body.query.as_ref() {
if mentions_seq_no(q) {
let reason = "failed to create query: Cannot query field [_seq_no]: _seq_no cannot be queried when [index.disable_sequence_numbers] is [true]";
- return ApiError::new(xerj_common::XerjError::invalid_query(reason)).into_response();
+ return ApiError::new(xerj_common::XerjError::invalid_query(reason))
+ .into_response();
}
}
// Sort by `_seq_no` → illegal_argument_exception (NOT query_shard).
@@ -5182,10 +5753,18 @@ pub async fn search(
.get(ix)
.map(|v| v.clone())
.and_then(|s| {
- let as_int = |v: &Value| v.as_u64().or_else(|| v.as_str().and_then(|s| s.parse().ok()));
- s.pointer("/index/max_result_window").and_then(|v| as_int(v))
- .or_else(|| s.get("index").and_then(|i| i.get("index.max_result_window")).and_then(|v| as_int(v)))
- .or_else(|| s.get("index.max_result_window").and_then(|v| as_int(v)))
+ let as_int = |v: &Value| {
+ v.as_u64()
+ .or_else(|| v.as_str().and_then(|s| s.parse().ok()))
+ };
+ s.pointer("/index/max_result_window")
+ .and_then(as_int)
+ .or_else(|| {
+ s.get("index")
+ .and_then(|i| i.get("index.max_result_window"))
+ .and_then(as_int)
+ })
+ .or_else(|| s.get("index.max_result_window").and_then(as_int))
})
.map(|v| v as usize)
.unwrap_or(10_000);
@@ -5193,10 +5772,16 @@ pub async fn search(
match q {
Value::Object(obj) => {
let mut m = 0;
- if let Some(ids) = obj.get("ids").and_then(|v| v.get("values")).and_then(Value::as_array) {
+ if let Some(ids) = obj
+ .get("ids")
+ .and_then(|v| v.get("values"))
+ .and_then(Value::as_array)
+ {
m = m.max(ids.len());
}
- for (_, v) in obj { m = m.max(max_ids_in_json(v)); }
+ for (_, v) in obj {
+ m = m.max(max_ids_in_json(v));
+ }
m
}
Value::Array(arr) => arr.iter().map(max_ids_in_json).max().unwrap_or(0),
@@ -5209,7 +5794,8 @@ pub async fn search(
"failed to create query: Too many ids specified, allowed max result window is [{}]",
max_w
);
- return ApiError::new(xerj_common::XerjError::invalid_query(reason)).into_response();
+ return ApiError::new(xerj_common::XerjError::invalid_query(reason))
+ .into_response();
}
}
}
@@ -5218,9 +5804,8 @@ pub async fn search(
// record hit/miss counters per participating index unless the
// caller explicitly sets `request_cache=false`. Only the
// `indices.stats · request_cache` numbers read these counters.
- let rc_enabled = body.size == 0
- && body.aggs.is_some()
- && params.request_cache.as_deref() != Some("false");
+ let rc_enabled =
+ body.size == 0 && body.aggs.is_some() && params.request_cache.as_deref() != Some("false");
if rc_enabled {
let body_hash = {
use std::collections::hash_map::DefaultHasher;
@@ -5228,10 +5813,26 @@ pub async fn search(
// EsSearchBody doesn't impl Serialize; reconstruct a canonical
// representation from the fields we care about for cache-key
// uniqueness (query + aggs + source filter + size).
- let q = body.query.as_ref().map(|v| v.to_string()).unwrap_or_default();
- let a = body.aggs.as_ref().map(|v| v.to_string()).unwrap_or_default();
- let ag = body.aggregations.as_ref().map(|v| v.to_string()).unwrap_or_default();
- let s = body.source.as_ref().map(|v| v.to_string()).unwrap_or_default();
+ let q = body
+ .query
+ .as_ref()
+ .map(|v| v.to_string())
+ .unwrap_or_default();
+ let a = body
+ .aggs
+ .as_ref()
+ .map(|v| v.to_string())
+ .unwrap_or_default();
+ let ag = body
+ .aggregations
+ .as_ref()
+ .map(|v| v.to_string())
+ .unwrap_or_default();
+ let s = body
+ .source
+ .as_ref()
+ .map(|v| v.to_string())
+ .unwrap_or_default();
let body_str = format!("{}|{}|{}|{}|{}|{}", q, a, ag, s, body.size, body.from);
let mut h = DefaultHasher::new();
body_str.hash(&mut h);
@@ -5263,9 +5864,13 @@ pub async fn search(
fn collect_value_types(v: &Value, out: &mut Vec<(String, String)>) {
let Some(obj) = v.as_object() else { return };
for (_name, body) in obj.iter() {
- let Some(body_obj) = body.as_object() else { continue };
+ let Some(body_obj) = body.as_object() else {
+ continue;
+ };
for (agg_type, spec) in body_obj.iter() {
- if matches!(agg_type.as_str(), "aggs" | "aggregations" | "meta") { continue; }
+ if matches!(agg_type.as_str(), "aggs" | "aggregations" | "meta") {
+ continue;
+ }
if let Some(spec_obj) = spec.as_object() {
if let (Some(field), Some(vt)) = (
spec_obj.get("field").and_then(Value::as_str),
@@ -5276,7 +5881,10 @@ pub async fn search(
}
}
// Recurse into sub-aggs
- if let Some(subs) = body_obj.get("aggs").or_else(|| body_obj.get("aggregations")) {
+ if let Some(subs) = body_obj
+ .get("aggs")
+ .or_else(|| body_obj.get("aggregations"))
+ {
collect_value_types(subs, out);
}
}
@@ -5286,18 +5894,36 @@ pub async fn search(
if !vts.is_empty() {
for idx_name in index_names.iter() {
let name_str: &str = idx_name;
- let Some(mapping_ref) = state.engine.index_mappings.get(name_str) else { continue };
+ let Some(mapping_ref) = state.engine.index_mappings.get(name_str) else {
+ continue;
+ };
let mapping = mapping_ref.value().clone();
drop(mapping_ref);
for (field, vt) in &vts {
- let field_type = mapping.pointer(&format!("/mappings/properties/{}/type", field))
+ let field_type = mapping
+ .pointer(&format!("/mappings/properties/{}/type", field))
.or_else(|| mapping.pointer(&format!("/properties/{}/type", field)))
.and_then(Value::as_str)
.unwrap_or("");
let incompatible = match (vt.as_str(), field_type) {
("ip", ft) if !ft.is_empty() && ft != "ip" => true,
("long" | "integer" | "short" | "byte" | "double" | "float", ft)
- if !ft.is_empty() && !matches!(ft, "long" | "integer" | "short" | "byte" | "double" | "float" | "half_float" | "scaled_float" | "unsigned_long") => true,
+ if !ft.is_empty()
+ && !matches!(
+ ft,
+ "long"
+ | "integer"
+ | "short"
+ | "byte"
+ | "double"
+ | "float"
+ | "half_float"
+ | "scaled_float"
+ | "unsigned_long"
+ ) =>
+ {
+ true
+ }
_ => false,
};
if incompatible {
@@ -5344,7 +5970,11 @@ pub async fn search(
.unwrap_or(search_selector_has_wildcard);
for idx_name in &index_names {
- state.metrics.queries_by_index.with_label_values(&[idx_name]).inc();
+ state
+ .metrics
+ .queries_by_index
+ .with_label_values(&[idx_name])
+ .inc();
let idx = match state.engine.get_index(idx_name) {
Ok(i) => i,
@@ -5365,7 +5995,11 @@ pub async fn search(
// Without this bump, size:1 + PIT would see a single hit get
// dropped and return 0 results.
if pit_context.is_some() {
- req_clone.size = req_clone.size.saturating_add(req_clone.from).saturating_mul(4).max(100);
+ req_clone.size = req_clone
+ .size
+ .saturating_add(req_clone.from)
+ .saturating_mul(4)
+ .max(100);
req_clone.from = 0;
}
// Per-index alias rewrite for collapse.field: if the declared
@@ -5374,7 +6008,9 @@ pub async fn search(
// store the field directly still collapse correctly.
if let Some(cs) = req_clone.collapse.as_mut() {
if let Some(mapping) = state.engine.index_mappings.get(*idx_name) {
- let props = mapping.pointer("/mappings/properties").or_else(|| mapping.pointer("/properties"));
+ let props = mapping
+ .pointer("/mappings/properties")
+ .or_else(|| mapping.pointer("/properties"));
if let Some(p) = props.and_then(Value::as_object) {
if let Some(field_spec) = p.get(&cs.field).and_then(Value::as_object) {
if field_spec.get("type").and_then(Value::as_str) == Some("alias") {
@@ -5386,9 +6022,7 @@ pub async fn search(
}
}
}
- let search_result = tokio::task::spawn(async move {
- idx.search(&req_clone).await
- }).await;
+ let search_result = tokio::task::spawn(async move { idx.search(&req_clone).await }).await;
match search_result {
Err(join_err) => {
@@ -5408,15 +6042,16 @@ pub async fn search(
total_relation = "gte".to_string();
}
if let Some(ms) = result.max_score {
- merged_population_max = Some(
- merged_population_max.map_or(ms as f64, |m: f64| m.max(ms as f64)),
- );
+ merged_population_max =
+ Some(merged_population_max.map_or(ms as f64, |m: f64| m.max(ms as f64)));
}
if let Some(new_aggs) = result.aggs {
match &mut merged_aggs {
None => merged_aggs = Some(new_aggs),
Some(existing) => {
- if let (Some(ex_obj), Some(new_obj)) = (existing.as_object_mut(), new_aggs.as_object()) {
+ if let (Some(ex_obj), Some(new_obj)) =
+ (existing.as_object_mut(), new_aggs.as_object())
+ {
for (k, v) in new_obj {
// Metric-agg aware merge: when both shards
// returned a metric result with internal
@@ -5440,8 +6075,16 @@ pub async fn search(
}
let should_replace = match (ex_obj.get(k), v) {
(Some(old), new) => {
- let old_empty = old.get("buckets").and_then(|b| b.as_array()).map(|a| a.is_empty()).unwrap_or(false);
- let new_has_data = new.get("buckets").and_then(|b| b.as_array()).map(|a| !a.is_empty()).unwrap_or(false);
+ let old_empty = old
+ .get("buckets")
+ .and_then(|b| b.as_array())
+ .map(|a| a.is_empty())
+ .unwrap_or(false);
+ let new_has_data = new
+ .get("buckets")
+ .and_then(|b| b.as_array())
+ .map(|a| !a.is_empty())
+ .unwrap_or(false);
old_empty && new_has_data
}
(None, _) => true,
@@ -5468,7 +6111,7 @@ pub async fn search(
// picks ONE interval across the global min/max; mirror that here by
// scanning the merged bucket keys, re-selecting the interval, and
// rebucketing at the coarser grid.
- if let Some(ref aggs_req) = body.aggs.as_ref().or(body.aggregations.as_ref()) {
+ if let Some(aggs_req) = body.aggs.as_ref().or(body.aggregations.as_ref()) {
if let Some(m) = merged_aggs.as_mut() {
rebucket_auto_date_histograms(m, aggs_req);
}
@@ -5480,26 +6123,45 @@ pub async fn search(
// number_of_shards; slice partitioning uses `shard_for(_id) % max` with
// ES's `number_of_shards` (from index settings) for consistency.
if let Some(slice_v) = body.slice.as_ref() {
- let slice_id = slice_v.get("id").and_then(|v| v.as_i64().or_else(|| v.as_str().and_then(|s| s.parse::<i64>().ok())));
- let slice_max = slice_v.get("max").and_then(|v| v.as_i64().or_else(|| v.as_str().and_then(|s| s.parse::<i64>().ok())));
- let slice_field = slice_v.get("field").and_then(|v| v.as_str()).unwrap_or("_id").to_string();
+ let slice_id = slice_v.get("id").and_then(|v| {
+ v.as_i64()
+ .or_else(|| v.as_str().and_then(|s| s.parse::<i64>().ok()))
+ });
+ let slice_max = slice_v.get("max").and_then(|v| {
+ v.as_i64()
+ .or_else(|| v.as_str().and_then(|s| s.parse::<i64>().ok()))
+ });
+ let slice_field = slice_v
+ .get("field")
+ .and_then(|v| v.as_str())
+ .unwrap_or("_id")
+ .to_string();
if let (Some(id), Some(max)) = (slice_id, slice_max) {
if max > 0 && id >= 0 && id < max {
// Compute number_of_shards from the first participating index's settings.
- let shards: u32 = index_names.first()
+ let shards: u32 = index_names
+ .first()
.and_then(|n| {
- let nn: &str = &**n;
- state.engine.index_settings.get(nn).map(|r| r.value().clone())
+ let nn: &str = n;
+ state
+ .engine
+ .index_settings
+ .get(nn)
+ .map(|r| r.value().clone())
})
.and_then(|s| {
s.pointer("/index/number_of_shards")
.or_else(|| s.pointer("/number_of_shards"))
- .and_then(|n| n.as_i64().or_else(|| n.as_str().and_then(|s| s.parse::<i64>().ok())))
+ .and_then(|n| {
+ n.as_i64()
+ .or_else(|| n.as_str().and_then(|s| s.parse::<i64>().ok()))
+ })
.map(|n| n as u32)
})
.unwrap_or(1);
let slice_by_id = slice_field == "_id";
- let mut kept: Vec<(String, xerj_query::executor::Hit)> = Vec::with_capacity(merged_hits.len());
+ let mut kept: Vec<(String, xerj_query::executor::Hit)> =
+ Vec::with_capacity(merged_hits.len());
for (idx_name, h) in merged_hits.drain(..) {
let key = if slice_by_id {
h.id.clone()
@@ -5512,7 +6174,9 @@ pub async fn search(
other => other.to_string(),
})
.unwrap_or_default()
- } else { String::new() };
+ } else {
+ String::new()
+ };
let shard = murmur3_routing_shard(&key, shards);
let in_slice = if (max as u32) <= shards {
(shard % (max as u32)) as i64 == id
@@ -5521,7 +6185,11 @@ pub async fn search(
// by id hash. Shard S gets slices [S*max/shards .. (S+1)*max/shards).
let slices_per_shard = (max as u32) / shards;
let start = shard * slices_per_shard;
- let end = if shard == shards - 1 { max as u32 } else { (shard + 1) * slices_per_shard };
+ let end = if shard == shards - 1 {
+ max as u32
+ } else {
+ (shard + 1) * slices_per_shard
+ };
if (id as u32) >= start && (id as u32) < end {
// Sub-slice by id hash within the shard's slice range.
let mut hasher = std::collections::hash_map::DefaultHasher::new();
@@ -5564,8 +6232,15 @@ pub async fn search(
let mut kept: Vec<(String, xerj_query::executor::Hit)> = Vec::new();
let mut filtered_out_in_page = 0u64;
for (idx_name, h) in merged_hits.drain(..) {
- let max_seq = pit.index_max_seq.get(&idx_name).copied().unwrap_or(u64::MAX);
- let hit_seq = state.engine.get_index(&idx_name).ok()
+ let max_seq = pit
+ .index_max_seq
+ .get(&idx_name)
+ .copied()
+ .unwrap_or(u64::MAX);
+ let hit_seq = state
+ .engine
+ .get_index(&idx_name)
+ .ok()
.and_then(|idx| idx.lookup_seq_no(&h.id))
.unwrap_or(0);
if hit_seq <= max_seq {
@@ -5681,16 +6356,21 @@ pub async fn search(
.and_then(|p| p.get(&sf.field))
.and_then(|fp| fp.get("type"))
.and_then(Value::as_str);
- let want_max = matches!(
- sf.missing,
- xerj_query::sort::SortMissing::Last
- );
+ let want_max = matches!(sf.missing, xerj_query::sort::SortMissing::Last);
let sentinel: Option<i64> = match ftype {
Some("integer") | Some("short") | Some("byte") => {
- if want_max { Some(i32::MAX as i64) } else { Some(i32::MIN as i64) }
+ if want_max {
+ Some(i32::MAX as i64)
+ } else {
+ Some(i32::MIN as i64)
+ }
}
Some("long") | Some("unsigned_long") => {
- if want_max { Some(i64::MAX) } else { Some(i64::MIN) }
+ if want_max {
+ Some(i64::MAX)
+ } else {
+ Some(i64::MIN)
+ }
}
_ => None,
};
@@ -5706,7 +6386,9 @@ pub async fn search(
if search_req.sort.is_empty() {
// Default: by score descending.
merged_hits.sort_by(|(_, a), (_, b)| {
- b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)
+ b.score
+ .partial_cmp(&a.score)
+ .unwrap_or(std::cmp::Ordering::Equal)
});
} else {
merged_hits.sort_by(|(_, a), (_, b)| {
@@ -5726,22 +6408,25 @@ pub async fn search(
let field = &cf.field;
let mut first_by_key: BTreeMap<String, usize> = BTreeMap::new();
let mut seen_keys: Vec<String> = Vec::new();
- let mut key_of = |src: &Value| -> String {
+ let key_of = |src: &Value| -> String {
// Try the declared field first; if not present (the
// caller queried a per-index alias target), fall back
// to a top-level alias-name scan via literal.
- let v = src.get(field)
- .or_else(|| {
- // Try each other known alias for this collapse
- // field by searching the src for numeric-ish
- // values. Simpler: iterate source keys that
- // aren't meta and aren't the declared field.
- src.as_object().and_then(|o| {
- o.iter().find_map(|(k, v)| {
- if k == field || k.starts_with('_') { None } else { Some(v) }
- })
+ let v = src.get(field).or_else(|| {
+ // Try each other known alias for this collapse
+ // field by searching the src for numeric-ish
+ // values. Simpler: iterate source keys that
+ // aren't meta and aren't the declared field.
+ src.as_object().and_then(|o| {
+ o.iter().find_map(|(k, v)| {
+ if k == field || k.starts_with('_') {
+ None
+ } else {
+ Some(v)
+ }
})
- });
+ })
+ });
match v {
Some(Value::String(s)) => s.clone(),
Some(Value::Number(n)) => n.to_string(),
@@ -5757,18 +6442,20 @@ pub async fn search(
// the floor — visible as `inner_hits.sub_hits.hits.total`
// off by N when an alias resolves the collapse field to
// a different stored field on a sibling index.
- let mut extra_members_by_key: BTreeMap<String, Vec<Value>> =
- BTreeMap::new();
+ let mut extra_members_by_key: BTreeMap<String, Vec<Value>> = BTreeMap::new();
for (i, (_, h)) in merged_hits.iter().enumerate() {
let k = key_of(&h.source);
- if k.is_empty() { continue; }
+ if k.is_empty() {
+ continue;
+ }
if !first_by_key.contains_key(&k) {
first_by_key.insert(k.clone(), i);
seen_keys.push(k);
} else {
// Not the leader — fold this hit's group members
// into the bucket so the leader can absorb them.
- if let Some(group) = h.source
+ if let Some(group) = h
+ .source
.as_object()
.and_then(|o| o.get("__xy_collapse_group__"))
.and_then(|v| v.as_array())
@@ -5786,7 +6473,8 @@ pub async fn search(
let unique_keys = first_by_key.len();
if unique_keys > 0 && unique_keys < merged_hits.len() {
let mut kept: Vec<(String, xerj_query::executor::Hit)> = Vec::new();
- let mut inserted: std::collections::HashSet<String> = std::collections::HashSet::new();
+ let mut inserted: std::collections::HashSet<String> =
+ std::collections::HashSet::new();
for (_k, &idx) in first_by_key.iter() {
let (idx_name, h) = &merged_hits[idx];
let kk = key_of(&h.source);
@@ -5796,13 +6484,14 @@ pub async fn search(
if let Some(extras) = extra_members_by_key.get(&kk) {
if !extras.is_empty() {
if let Some(obj) = leader.source.as_object_mut() {
- let combined: Vec<Value> = match obj.remove("__xy_collapse_group__") {
- Some(Value::Array(mut a)) => {
- a.extend(extras.iter().cloned());
- a
- }
- _ => extras.clone(),
- };
+ let combined: Vec<Value> =
+ match obj.remove("__xy_collapse_group__") {
+ Some(Value::Array(mut a)) => {
+ a.extend(extras.iter().cloned());
+ a
+ }
+ _ => extras.clone(),
+ };
obj.insert(
"__xy_collapse_group__".to_string(),
Value::Array(combined),
@@ -5818,7 +6507,9 @@ pub async fn search(
// against search_req.sort.
if search_req.sort.is_empty() {
kept.sort_by(|(_, a), (_, b)| {
- b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)
+ b.score
+ .partial_cmp(&a.score)
+ .unwrap_or(std::cmp::Ordering::Equal)
});
} else {
kept.sort_by(|(_, a), (_, b)| {
@@ -5883,10 +6574,12 @@ pub async fn search(
let before = merged_hits.len();
let mut negative_present = false;
merged_hits.retain(|(_, h)| {
- let neg = fields.iter().any(|f| {
- source_numeric_values(&h.source, f).iter().any(|v| *v < 0.0)
- });
- if neg { negative_present = true; }
+ let neg = fields
+ .iter()
+ .any(|f| source_numeric_values(&h.source, f).iter().any(|v| *v < 0.0));
+ if neg {
+ negative_present = true;
+ }
!neg
});
let excluded = (before - merged_hits.len()) as u64;
@@ -5901,7 +6594,8 @@ pub async fn search(
.filter(|v| *v >= 0.0)
.collect();
if let Some(agg_obj) = magg.get_mut(name).and_then(|v| v.as_object_mut()) {
- agg_obj.insert("values".to_string(), hdr_percentiles_values(&vals, spec));
+ agg_obj
+ .insert("values".to_string(), hdr_percentiles_values(&vals, spec));
}
}
hdr_shard_failure = Some(json!({
@@ -5950,8 +6644,8 @@ pub async fn search(
|| search_req.sort.iter().all(|s| s.is_score())
|| body.track_scores.unwrap_or(false);
let max_score = if sort_tracks_scores {
- let explicit_field_sort = !search_req.sort.is_empty()
- && search_req.sort.iter().any(|s| !s.is_score());
+ let explicit_field_sort =
+ !search_req.sort.is_empty() && search_req.sort.iter().any(|s| !s.is_score());
if explicit_field_sort {
// Prefer the per-index pre-collapse population max; fall back to the
// (possibly collapsed/paged) merged hit set when unavailable.
@@ -5960,7 +6654,10 @@ pub async fn search(
.iter()
.map(|(_, h)| h.score as f64)
.fold(None, |acc: Option<f64>, s| {
- Some(match acc { Some(m) if m >= s => m, _ => s })
+ Some(match acc {
+ Some(m) if m >= s => m,
+ _ => s,
+ })
})
})
} else {
@@ -5984,11 +6681,8 @@ pub async fn search(
// Build script_fields null map if requested.
let script_fields_map: Option<HashMap<String, Value>> =
search_req.script_fields.as_ref().and_then(|sf| {
- sf.as_object().map(|obj| {
- obj.keys()
- .map(|k| (k.clone(), Value::Null))
- .collect()
- })
+ sf.as_object()
+ .map(|obj| obj.keys().map(|k| (k.clone(), Value::Null)).collect())
});
let requested_fields = &search_req.fields;
@@ -5998,32 +6692,39 @@ pub async fn search(
// like `{"field": "date", "format": "yyyy-MM-dd"}` (7.11+). When the
// body is absent, falls back to search_req.fields (plain strings).
let field_specs: Vec<(String, Option<String>, bool)> = match &body.fields {
- Some(Value::Array(arr)) => arr.iter().filter_map(|v| match v {
- Value::String(s) => Some((s.clone(), None, false)),
- Value::Object(o) => {
- let name = o.get("field").and_then(Value::as_str)?.to_string();
- let fmt = o.get("format").and_then(Value::as_str).map(str::to_string);
- let inc = o.get("include_unmapped").and_then(Value::as_bool).unwrap_or(false);
- Some((name, fmt, inc))
- }
- _ => None,
- }).collect(),
+ Some(Value::Array(arr)) => arr
+ .iter()
+ .filter_map(|v| match v {
+ Value::String(s) => Some((s.clone(), None, false)),
+ Value::Object(o) => {
+ let name = o.get("field").and_then(Value::as_str)?.to_string();
+ let fmt = o.get("format").and_then(Value::as_str).map(str::to_string);
+ let inc = o
+ .get("include_unmapped")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
+ Some((name, fmt, inc))
+ }
+ _ => None,
+ })
+ .collect(),
Some(Value::String(s)) => vec![(s.clone(), None, false)],
- _ => requested_fields.iter().map(|s| (s.clone(), None, false)).collect(),
+ _ => requested_fields
+ .iter()
+ .map(|s| (s.clone(), None, false))
+ .collect(),
};
// Parse stored_fields from body.
let (suppress_source_for_stored, stored_meta_fields) = body
.stored_fields
.as_ref()
- .map(|sf| parse_stored_fields(sf))
+ .map(parse_stored_fields)
.unwrap_or((false, vec![]));
// When parse returns the `__none__` sentinel, suppress `_id` on every
// hit in the final response. Drop the sentinel from the requested-
// meta list so it doesn't leak into the output `fields` map.
- let suppress_meta_fields = stored_meta_fields
- .iter()
- .any(|f| f == "__none__");
+ let suppress_meta_fields = stored_meta_fields.iter().any(|f| f == "__none__");
let stored_meta_fields: Vec<String> = stored_meta_fields
.into_iter()
.filter(|f| f != "__none__")
@@ -6055,7 +6756,8 @@ pub async fn search(
if let (Some(path), Some(ih)) = (path, ih) {
// Use path as bucket key unless spec
// provides a `name`.
- let bucket_name = ih.as_object()
+ let bucket_name = ih
+ .as_object()
.and_then(|o| o.get("name"))
.and_then(Value::as_str)
.unwrap_or(path)
@@ -6073,7 +6775,9 @@ pub async fn search(
if let Some(b) = val.as_object() {
for sub_key in ["must", "should", "filter", "must_not"] {
if let Some(arr) = b.get(sub_key).and_then(Value::as_array) {
- for item in arr { collect_nested_inner_hits(item, out); }
+ for item in arr {
+ collect_nested_inner_hits(item, out);
+ }
}
}
}
@@ -6087,7 +6791,9 @@ pub async fn search(
if !extracted.is_empty() {
// Merge extracted with existing (top-level wins for name clashes).
if let Some(existing_obj) = inner_hits_config.as_object_mut() {
- for (k, v) in extracted { existing_obj.entry(k).or_insert(v); }
+ for (k, v) in extracted {
+ existing_obj.entry(k).or_insert(v);
+ }
} else {
inner_hits_config = Value::Object(extracted);
}
@@ -7550,6 +8256,10 @@ pub async fn search(
.iter()
.find_map(|(orig, target)| if target == &cf.field { Some(orig.clone()) } else { None })
.unwrap_or_else(|| cf.field.clone());
+ // `response_key` is read again inside this block (and only
+ // conditionally inserted), so the Entry API would tangle the
+ // borrow / force an extra clone — keep the explicit check.
+ #[allow(clippy::map_entry)]
if !fmap.contains_key(&response_key) {
// Look up the value from `_source` under any of:
// 1. the literal collapse field (covers indices
@@ -8063,9 +8773,11 @@ pub async fn search(
}
fields.dedup();
- let mut all_terms: std::collections::HashMap<String, Vec<(String, usize)>> = std::collections::HashMap::new();
+ let mut all_terms: std::collections::HashMap<String, Vec<(String, usize)>> =
+ std::collections::HashMap::new();
for field in &fields {
- let mut combined: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
+ let mut combined: std::collections::HashMap<String, usize> =
+ std::collections::HashMap::new();
for idx_name in &index_names {
if let Ok(idx) = state.engine.get_index(idx_name) {
for (term, freq) in idx.get_all_terms_for_field(field).await {
@@ -8080,7 +8792,9 @@ pub async fn search(
// Also collect doc sources for backward-compatible completion suggester.
let all_docs: Vec<Value> = {
- let suggest_req = match xerj_query::parse_request(&json!({"query": {"match_all": {}}, "size": 1000, "from": 0})) {
+ let suggest_req = match xerj_query::parse_request(
+ &json!({"query": {"match_all": {}}, "size": 1000, "from": 0}),
+ ) {
Ok(r) => r,
Err(_) => xerj_query::parse_request(&json!({"size": 1000})).unwrap_or_default(),
};
@@ -8098,7 +8812,11 @@ pub async fn search(
}
all
};
- Some(process_suggest_with_terms(suggest_body, &all_docs, &index_terms))
+ Some(process_suggest_with_terms(
+ suggest_body,
+ &all_docs,
+ &index_terms,
+ ))
} else {
None
};
@@ -8264,9 +8982,11 @@ pub async fn search(
let bpath = settings
.get("persistent")
.and_then(|p| p.get("search.aggs.rewrite_to_filter_by_filter"))
- .or_else(|| settings
- .get("transient")
- .and_then(|t| t.get("search.aggs.rewrite_to_filter_by_filter")));
+ .or_else(|| {
+ settings
+ .get("transient")
+ .and_then(|t| t.get("search.aggs.rewrite_to_filter_by_filter"))
+ });
match bpath {
Some(Value::Bool(b)) => *b,
Some(Value::String(s)) => s != "false",
@@ -8294,9 +9014,9 @@ pub async fn search(
.clone()
.unwrap_or_else(|| json!({"match_all": {}}));
let mut fg_docs: Vec<Value> = Vec::new();
- if let Ok(fg_req) = xerj_query::parse_request(
- &json!({"query": fg_query, "size": 10000, "from": 0}),
- ) {
+ if let Ok(fg_req) =
+ xerj_query::parse_request(&json!({"query": fg_query, "size": 10000, "from": 0}))
+ {
for idx_name in &index_names {
if let Ok(idx) = state.engine.get_index(idx_name) {
if let Ok(result) = idx.search(&fg_req).await {
@@ -8316,9 +9036,13 @@ pub async fn search(
}
}
let aggs_profile = build_aggregation_profile_full(
- search_req.aggs.as_ref().or(body.aggs.as_ref()).or(body.aggregations.as_ref()),
+ search_req
+ .aggs
+ .as_ref()
+ .or(body.aggs.as_ref())
+ .or(body.aggregations.as_ref()),
took_ms,
- total_count as u64,
+ total_count,
response_body.get("aggregations"),
terms_use_filter_path,
&sig_text_debug,
@@ -8332,7 +9056,11 @@ pub async fn search(
let stored_fields_debug: Vec<Value> = if suppress_meta_fields {
vec![]
} else {
- vec![Value::String("_id".into()), Value::String("_routing".into()), Value::String("_source".into())]
+ vec![
+ Value::String("_id".into()),
+ Value::String("_routing".into()),
+ Value::String("_source".into()),
+ ]
};
// ES emits a list of FetchSubPhase children inside `fetch`. The
// canonical default set is {FetchFieldsPhase, FetchSourcePhase,
@@ -8346,7 +9074,9 @@ pub async fn search(
fn has_inner_hits_in(v: &Value) -> bool {
match v {
Value::Object(o) => {
- if o.contains_key("inner_hits") { return true; }
+ if o.contains_key("inner_hits") {
+ return true;
+ }
o.values().any(has_inner_hits_in)
}
Value::Array(a) => a.iter().any(has_inner_hits_in),
@@ -8361,7 +9091,12 @@ pub async fn search(
let phases: &[&str] = match (source_disabled, inner_hits_phase) {
(true, true) => &["FetchFieldsPhase", "InnerHitsPhase", "StoredFieldsPhase"],
(true, false) => &["FetchFieldsPhase", "StoredFieldsPhase"],
- (false, true) => &["FetchFieldsPhase", "FetchSourcePhase", "InnerHitsPhase", "StoredFieldsPhase"],
+ (false, true) => &[
+ "FetchFieldsPhase",
+ "FetchSourcePhase",
+ "InnerHitsPhase",
+ "StoredFieldsPhase",
+ ],
(false, false) => &["FetchFieldsPhase", "FetchSourcePhase", "StoredFieldsPhase"],
};
phases
@@ -8403,15 +9138,21 @@ pub async fn search(
fetch_map.insert("type".into(), Value::String("fetch".into()));
fetch_map.insert("description".into(), Value::String("".into()));
fetch_map.insert("time_in_nanos".into(), json!(fetch_ns));
- fetch_map.insert("breakdown".into(), json!({
- "load_stored_fields": fetch_ns,
- "load_stored_fields_count": 1u64,
- "load_source": fetch_ns,
- "load_source_count": 1u64,
- "next_reader": 1u64,
- "next_reader_count": 1u64,
- }));
- fetch_map.insert("debug".into(), json!({"stored_fields": stored_fields_debug}));
+ fetch_map.insert(
+ "breakdown".into(),
+ json!({
+ "load_stored_fields": fetch_ns,
+ "load_stored_fields_count": 1u64,
+ "load_source": fetch_ns,
+ "load_source_count": 1u64,
+ "next_reader": 1u64,
+ "next_reader_count": 1u64,
+ }),
+ );
+ fetch_map.insert(
+ "debug".into(),
+ json!({"stored_fields": stored_fields_debug}),
+ );
if !fetch_children.is_empty() {
fetch_map.insert("children".into(), Value::Array(fetch_children));
}
@@ -8426,7 +9167,8 @@ pub async fn search(
let has_knn = original_knn.is_some();
let dfs_block = if is_dfs || has_knn {
let knn_array: Vec<Value> = if has_knn {
- let knn_field = original_knn.as_ref()
+ let knn_field = original_knn
+ .as_ref()
.and_then(|k| k.get("field"))
.and_then(Value::as_str)
.unwrap_or("");
@@ -8468,41 +9210,54 @@ pub async fn search(
}],
"vector_operations_count": total_hits.max(1),
})]
- } else { Vec::new() };
+ } else {
+ Vec::new()
+ };
let mut dfs_obj = serde_json::Map::new();
// The `dfs.statistics` sub-block (only emitted for explicit
// dfs_query_then_fetch — knn-only DFS doesn't include it).
if is_dfs {
- dfs_obj.insert("statistics".to_string(), json!({
- "type": "statistics",
- "description": "collect term statistics",
- "time_in_nanos": 1u64,
- "breakdown": {
- "collection_statistics": 0u64,
- "collection_statistics_count": 0u64,
- "create_weight": 0u64,
- "create_weight_count": 0u64,
- "rewrite": 0u64,
- "rewrite_count": 0u64,
- "term_statistics": 0u64,
- "term_statistics_count": 0u64,
- }
- }));
+ dfs_obj.insert(
+ "statistics".to_string(),
+ json!({
+ "type": "statistics",
+ "description": "collect term statistics",
+ "time_in_nanos": 1u64,
+ "breakdown": {
+ "collection_statistics": 0u64,
+ "collection_statistics_count": 0u64,
+ "create_weight": 0u64,
+ "create_weight_count": 0u64,
+ "rewrite": 0u64,
+ "rewrite_count": 0u64,
+ "term_statistics": 0u64,
+ "term_statistics_count": 0u64,
+ }
+ }),
+ );
}
if !knn_array.is_empty() {
dfs_obj.insert("knn".to_string(), Value::Array(knn_array));
}
Some(Value::Object(dfs_obj))
- } else { None };
+ } else {
+ None
+ };
// Number of shards to emit in the profile. Multi-shard indexes
// expose N primary profiles so `profile.shards.1.dfs.*` works.
- let profile_shards_n: u64 = state.engine.index_settings.get(profile_index)
+ let profile_shards_n: u64 = state
+ .engine
+ .index_settings
+ .get(profile_index)
.and_then(|r| {
let s = r.value();
s.pointer("/index/number_of_shards")
.or_else(|| s.get("number_of_shards"))
.or_else(|| s.get("index.number_of_shards"))
- .and_then(|n| n.as_i64().or_else(|| n.as_str().and_then(|s| s.parse::<i64>().ok())))
+ .and_then(|n| {
+ n.as_i64()
+ .or_else(|| n.as_str().and_then(|s| s.parse::<i64>().ok()))
+ })
.map(|n| n as u64)
})
.unwrap_or(1);
@@ -8582,7 +9337,10 @@ pub async fn search(
// Pick the first backing index for the scroll context (multi-index
// scrolls are rare and we keep the raw index spec so `_index` on
// each hit remains authoritative when paging).
- let scroll_index = index_names.first().map(|s| s.to_string()).unwrap_or_default();
+ let scroll_index = index_names
+ .first()
+ .map(|s| s.to_string())
+ .unwrap_or_default();
let scroll_id = Uuid::new_v4().to_string();
let ctx = xerj_engine::engine::ScrollContext {
index: scroll_index,
@@ -8629,31 +9387,67 @@ pub async fn search(
if let Some(hits) = response_body.pointer_mut("/hits/hits") {
if let Some(arr) = hits.as_array_mut() {
for hit in arr.iter_mut() {
- let ix = hit.get("_index").and_then(Value::as_str).unwrap_or("").to_string();
- let Some(mapping) = state.engine.index_mappings.get(&ix).map(|v| v.clone()) else { continue };
+ let ix = hit
+ .get("_index")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
+ let Some(mapping) = state.engine.index_mappings.get(&ix).map(|v| v.clone()) else {
+ continue;
+ };
let settings = state.engine.index_settings.get(&ix).map(|v| v.clone());
- let synthetic = settings.as_ref().map(|s| {
- let is_synth = |v: &Value| v.as_str() == Some("synthetic");
- s.pointer("/index/mapping/source.mode").map(is_synth).unwrap_or(false)
- || s.pointer("/index/mapping.source.mode").map(is_synth).unwrap_or(false)
- || s.get("index").and_then(|i| i.get("mapping.source.mode")).map(is_synth).unwrap_or(false)
- || s.get("index.mapping.source.mode").map(is_synth).unwrap_or(false)
- || s.get("mapping.source.mode").map(is_synth).unwrap_or(false)
- }).unwrap_or(false) || mapping
- .pointer("/_source/mode").and_then(Value::as_str) == Some("synthetic")
- || mapping.pointer("/mappings/_source/mode").and_then(Value::as_str) == Some("synthetic");
- if !synthetic { continue; }
+ let synthetic = settings
+ .as_ref()
+ .map(|s| {
+ let is_synth = |v: &Value| v.as_str() == Some("synthetic");
+ s.pointer("/index/mapping/source.mode")
+ .map(is_synth)
+ .unwrap_or(false)
+ || s.pointer("/index/mapping.source.mode")
+ .map(is_synth)
+ .unwrap_or(false)
+ || s.get("index")
+ .and_then(|i| i.get("mapping.source.mode"))
+ .map(is_synth)
+ .unwrap_or(false)
+ || s.get("index.mapping.source.mode")
+ .map(is_synth)
+ .unwrap_or(false)
+ || s.get("mapping.source.mode").map(is_synth).unwrap_or(false)
+ })
+ .unwrap_or(false)
+ || mapping.pointer("/_source/mode").and_then(Value::as_str)
+ == Some("synthetic")
+ || mapping
+ .pointer("/mappings/_source/mode")
+ .and_then(Value::as_str)
+ == Some("synthetic");
+ if !synthetic {
+ continue;
+ }
let Some(props) = mapping
.pointer("/mappings/properties")
.or_else(|| mapping.pointer("/properties"))
- .and_then(Value::as_object).cloned()
- else { continue };
+ .and_then(Value::as_object)
+ .cloned()
+ else {
+ continue;
+ };
// Resolve the index-level `index.mapping.ignore_above`
// default; per-field `ignore_above` still wins.
let default_ignore_above: Option<usize> = settings.as_ref().and_then(|s| {
- let as_u = |v: &Value| v.as_u64().or_else(|| v.as_str().and_then(|x| x.parse().ok())).map(|n| n as usize);
- s.pointer("/index/mapping/ignore_above").and_then(as_u)
- .or_else(|| s.get("index").and_then(|i| i.get("mapping.ignore_above")).and_then(as_u))
+ let as_u = |v: &Value| {
+ v.as_u64()
+ .or_else(|| v.as_str().and_then(|x| x.parse().ok()))
+ .map(|n| n as usize)
+ };
+ s.pointer("/index/mapping/ignore_above")
+ .and_then(as_u)
+ .or_else(|| {
+ s.get("index")
+ .and_then(|i| i.get("mapping.ignore_above"))
+ .and_then(as_u)
+ })
.or_else(|| s.get("index.mapping.ignore_above").and_then(as_u))
});
if let Some(src) = hit.get_mut("_source").and_then(|v| v.as_object_mut()) {
@@ -8664,7 +9458,9 @@ pub async fn search(
let mut ignored: Vec<Value> = Vec::new();
for x in arr.drain(..) {
match &x {
- Value::String(s) if s.chars().count() > max => ignored.push(x),
+ Value::String(s) if s.chars().count() > max => {
+ ignored.push(x)
+ }
_ => kept.push(x),
}
}
@@ -8676,7 +9472,9 @@ pub async fn search(
*arr = kept;
}
Value::Object(o) => {
- for (_, val) in o { reorder(val, max); }
+ for (_, val) in o {
+ reorder(val, max);
+ }
}
_ => {}
}
@@ -8694,7 +9492,9 @@ pub async fn search(
if let Some(Value::Array(arr)) = src.get_mut(field) {
let mut wrap = Value::Array(std::mem::take(arr));
reorder(&mut wrap, max);
- if let Value::Array(rs) = wrap { *arr = rs; }
+ if let Value::Array(rs) = wrap {
+ *arr = rs;
+ }
}
}
"flattened" => {
@@ -8716,18 +9516,38 @@ pub async fn search(
if let Some(hits) = response_body.pointer_mut("/hits/hits") {
if let Some(arr) = hits.as_array_mut() {
for hit in arr.iter_mut() {
- let ix = hit.get("_index").and_then(Value::as_str).unwrap_or("").to_string();
- let disabled = state.engine.index_settings.get(&ix).map(|v| {
- let s = v.clone();
- let as_bool = |val: &Value| val.as_bool().unwrap_or_else(|| val.as_str().map(|x| x == "true").unwrap_or(false));
- s.pointer("/index/disable_sequence_numbers").map(as_bool).unwrap_or(false)
- || s.get("index").and_then(|i| i.get("index.disable_sequence_numbers")).map(as_bool).unwrap_or(false)
- || s.get("index.disable_sequence_numbers").map(as_bool).unwrap_or(false)
- }).unwrap_or(false);
- if disabled {
- if let Some(obj) = hit.as_object_mut() {
- obj.insert("_seq_no".into(), json!(-2i64));
- obj.insert("_primary_term".into(), json!(0));
+ let ix = hit
+ .get("_index")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
+ let disabled = state
+ .engine
+ .index_settings
+ .get(&ix)
+ .map(|v| {
+ let s = v.clone();
+ let as_bool = |val: &Value| {
+ val.as_bool().unwrap_or_else(|| {
+ val.as_str().map(|x| x == "true").unwrap_or(false)
+ })
+ };
+ s.pointer("/index/disable_sequence_numbers")
+ .map(as_bool)
+ .unwrap_or(false)
+ || s.get("index")
+ .and_then(|i| i.get("index.disable_sequence_numbers"))
+ .map(as_bool)
+ .unwrap_or(false)
+ || s.get("index.disable_sequence_numbers")
+ .map(as_bool)
+ .unwrap_or(false)
+ })
+ .unwrap_or(false);
+ if disabled {
+ if let Some(obj) = hit.as_object_mut() {
+ obj.insert("_seq_no".into(), json!(-2i64));
+ obj.insert("_primary_term".into(), json!(0));
}
}
}
@@ -8764,11 +9584,17 @@ fn expand_dotted_keys(v: Value) -> Value {
// Merge if the key already exists (dotted-key siblings).
match out.get_mut(&k) {
Some(existing) if existing.is_object() && val.is_object() => {
- if let (Some(e), Some(v_obj)) = (existing.as_object_mut(), val.as_object()) {
- for (kk, vv) in v_obj { e.insert(kk.clone(), vv.clone()); }
+ if let (Some(e), Some(v_obj)) =
+ (existing.as_object_mut(), val.as_object())
+ {
+ for (kk, vv) in v_obj {
+ e.insert(kk.clone(), vv.clone());
+ }
}
}
- _ => { out.insert(k, val); }
+ _ => {
+ out.insert(k, val);
+ }
}
}
}
@@ -8778,12 +9604,10 @@ fn expand_dotted_keys(v: Value) -> Value {
}
}
-fn insert_nested(
- target: &mut serde_json::Map<String, Value>,
- segs: &[&str],
- val: Value,
-) {
- if segs.is_empty() { return; }
+fn insert_nested(target: &mut serde_json::Map<String, Value>, segs: &[&str], val: Value) {
+ if segs.is_empty() {
+ return;
+ }
if segs.len() == 1 {
target.insert(segs[0].to_string(), val);
return;
@@ -8829,7 +9653,8 @@ fn strip_type_tags(aggs: Value) -> Value {
match aggs {
Value::Object(mut obj) => {
obj.remove("__type__");
- let cleaned: serde_json::Map<String, Value> = obj.into_iter()
+ let cleaned: serde_json::Map<String, Value> = obj
+ .into_iter()
.map(|(k, v)| (k, strip_type_tags(v)))
.collect();
Value::Object(cleaned)
@@ -8879,10 +9704,7 @@ fn rewrite_agg_with_typed_keys(val: Value) -> Value {
// Prefix every nested agg result.
for (k, v) in nested_aggs {
let prefix = typed_key_prefix(&v);
- retained.insert(
- format!("{}#{}", prefix, k),
- rewrite_agg_with_typed_keys(v),
- );
+ retained.insert(format!("{}#{}", prefix, k), rewrite_agg_with_typed_keys(v));
}
if let Some(buckets_val) = buckets {
@@ -8890,9 +9712,9 @@ fn rewrite_agg_with_typed_keys(val: Value) -> Value {
}
Value::Object(retained)
}
- Value::Array(arr) => Value::Array(
- arr.into_iter().map(rewrite_agg_with_typed_keys).collect(),
- ),
+ Value::Array(arr) => {
+ Value::Array(arr.into_iter().map(rewrite_agg_with_typed_keys).collect())
+ }
other => other,
}
}
@@ -8923,10 +9745,7 @@ fn rewrite_buckets(buckets: Value) -> Value {
}
for (k, v) in nested {
let prefix = typed_key_prefix(&v);
- out.insert(
- format!("{}#{}", prefix, k),
- rewrite_agg_with_typed_keys(v),
- );
+ out.insert(format!("{}#{}", prefix, k), rewrite_agg_with_typed_keys(v));
}
Value::Object(out)
} else {
@@ -8964,8 +9783,12 @@ fn typed_key_prefix(val: &Value) -> String {
"terms" => {
if let Some(Value::Array(buckets)) = obj.get("buckets") {
if let Some(first) = buckets.first() {
- if first.get("key").and_then(Value::as_i64).is_some() { return "lterms".into(); }
- if first.get("key").and_then(Value::as_f64).is_some() { return "dterms".into(); }
+ if first.get("key").and_then(Value::as_i64).is_some() {
+ return "lterms".into();
+ }
+ if first.get("key").and_then(Value::as_f64).is_some() {
+ return "dterms".into();
+ }
}
}
"sterms".into()
@@ -8978,14 +9801,22 @@ fn typed_key_prefix(val: &Value) -> String {
if obj.contains_key("buckets") {
if let Some(Value::Array(buckets)) = obj.get("buckets") {
if let Some(first) = buckets.first() {
- if first.get("key_as_string").is_some() { return "date_histogram".into(); }
- if first.get("key").and_then(Value::as_f64).is_some() { return "histogram".into(); }
+ if first.get("key_as_string").is_some() {
+ return "date_histogram".into();
+ }
+ if first.get("key").and_then(Value::as_f64).is_some() {
+ return "histogram".into();
+ }
}
}
return "sterms".into();
}
- if obj.contains_key("values") { return "tdigest_percentiles".into(); }
- if obj.contains_key("value") { return "avg".into(); }
+ if obj.contains_key("values") {
+ return "tdigest_percentiles".into();
+ }
+ if obj.contains_key("value") {
+ return "avg".into();
+ }
}
"value".into()
}
@@ -9007,13 +9838,19 @@ fn apply_filter_path(value: Value, paths: &[&str]) -> Value {
let mut excludes: Vec<Vec<String>> = Vec::new();
for p in paths {
let p = p.trim();
- if p.is_empty() { continue; }
+ if p.is_empty() {
+ continue;
+ }
let (segments, is_neg) = if let Some(rest) = p.strip_prefix('-') {
(rest.split('.').map(String::from).collect::<Vec<_>>(), true)
} else {
(p.split('.').map(String::from).collect::<Vec<_>>(), false)
};
- if is_neg { excludes.push(segments); } else { includes.push(segments); }
+ if is_neg {
+ excludes.push(segments);
+ } else {
+ includes.push(segments);
+ }
}
// Starting point: empty (only-includes) or full tree (only-excludes).
let mut result = if includes.is_empty() {
@@ -9036,7 +9873,9 @@ fn match_pattern<'a>(pattern: &'a [String], key: &str) -> Vec<Option<&'a [String
// Returns outcomes for consuming one path segment (`key`):
// - `None` → terminal (the pattern ends here, include whole subtree)
// - `Some(suffix)` → still pattern remaining to match below this key
- if pattern.is_empty() { return Vec::new(); }
+ if pattern.is_empty() {
+ return Vec::new();
+ }
let first = &pattern[0];
let rest = &pattern[1..];
let mut out = Vec::new();
@@ -9060,22 +9899,34 @@ fn match_pattern<'a>(pattern: &'a [String], key: &str) -> Vec<Option<&'a [String
/// Match a single path segment against a glob pattern (`*` wildcard
/// within the segment, as distinct from `**` which spans segments).
fn segment_matches(pattern: &str, key: &str) -> bool {
- if pattern == "*" || pattern == key { return true; }
- if !pattern.contains('*') { return false; }
+ if pattern == "*" || pattern == key {
+ return true;
+ }
+ if !pattern.contains('*') {
+ return false;
+ }
// Simple `*`-glob within a single segment.
let mut k = key;
let parts: Vec<&str> = pattern.split('*').collect();
let len = parts.len();
for (i, part) in parts.iter().enumerate() {
if i == 0 {
- if !k.starts_with(part) { return false; }
+ if !k.starts_with(part) {
+ return false;
+ }
k = &k[part.len()..];
} else if i == len - 1 {
- if !k.ends_with(part) { return false; }
- if part.len() > k.len() { return false; }
+ if !k.ends_with(part) {
+ return false;
+ }
+ if part.len() > k.len() {
+ return false;
+ }
} else if !part.is_empty() {
match k.find(part) {
- Some(idx) => { k = &k[idx + part.len()..]; }
+ Some(idx) => {
+ k = &k[idx + part.len()..];
+ }
None => return false,
}
}
@@ -9095,11 +9946,17 @@ fn filter_include(value: &Value, patterns: &[Vec<String>]) -> Value {
for pat in patterns {
for outcome in match_pattern(pat, key) {
match outcome {
- None => { include_all = true; }
- Some(suffix) => { sub_patterns.push(suffix.to_vec()); }
+ None => {
+ include_all = true;
+ }
+ Some(suffix) => {
+ sub_patterns.push(suffix.to_vec());
+ }
}
}
- if include_all { break; }
+ if include_all {
+ break;
+ }
}
if include_all {
result.insert(key.clone(), val.clone());
@@ -9118,14 +9975,12 @@ fn filter_include(value: &Value, patterns: &[Vec<String>]) -> Value {
}
Value::Object(result)
}
- Value::Array(arr) => {
- Value::Array(
- arr.iter()
- .map(|v| filter_include(v, patterns))
- .filter(|v| !is_empty_collection(v))
- .collect(),
- )
- }
+ Value::Array(arr) => Value::Array(
+ arr.iter()
+ .map(|v| filter_include(v, patterns))
+ .filter(|v| !is_empty_collection(v))
+ .collect(),
+ ),
other => other.clone(),
}
}
@@ -9140,11 +9995,17 @@ fn filter_exclude(value: Value, patterns: &[Vec<String>]) -> Value {
for pat in patterns {
for outcome in match_pattern(pat, &key) {
match outcome {
- None => { drop_key = true; }
- Some(suffix) => { sub_patterns.push(suffix.to_vec()); }
+ None => {
+ drop_key = true;
+ }
+ Some(suffix) => {
+ sub_patterns.push(suffix.to_vec());
+ }
}
}
- if drop_key { break; }
+ if drop_key {
+ break;
+ }
}
if drop_key {
continue;
@@ -9157,13 +10018,11 @@ fn filter_exclude(value: Value, patterns: &[Vec<String>]) -> Value {
}
Value::Object(result)
}
- Value::Array(arr) => {
- Value::Array(
- arr.into_iter()
- .map(|v| filter_exclude(v, patterns))
- .collect(),
- )
- }
+ Value::Array(arr) => Value::Array(
+ arr.into_iter()
+ .map(|v| filter_exclude(v, patterns))
+ .collect(),
+ ),
other => other,
}
}
@@ -9270,7 +10129,9 @@ pub async fn global_bulk(
process_bulk_body(&state, None, &body, started, opts).await
}
-fn bulk_opts_from_query(qp: &std::collections::HashMap<String, String>) -> xerj_engine::bulk::BulkOpts {
+fn bulk_opts_from_query(
+ qp: &std::collections::HashMap<String, String>,
+) -> xerj_engine::bulk::BulkOpts {
// `?_source=true` / `?_source=field` / `?_source_includes=f1,f2` /
// `?_source_excludes=f3` — translate the ES URL-shape to the
// search body `_source` filter shape the engine consumes.
@@ -9281,11 +10142,17 @@ fn bulk_opts_from_query(qp: &std::collections::HashMap<String, String>) -> xerj_
if inc.is_some() || exc.is_some() {
let mut obj = serde_json::Map::new();
if let Some(v) = inc {
- let arr: Vec<Value> = v.split(',').map(|s| Value::String(s.trim().to_string())).collect();
+ let arr: Vec<Value> = v
+ .split(',')
+ .map(|s| Value::String(s.trim().to_string()))
+ .collect();
obj.insert("includes".to_string(), Value::Array(arr));
}
if let Some(v) = exc {
- let arr: Vec<Value> = v.split(',').map(|s| Value::String(s.trim().to_string())).collect();
+ let arr: Vec<Value> = v
+ .split(',')
+ .map(|s| Value::String(s.trim().to_string()))
+ .collect();
obj.insert("excludes".to_string(), Value::Array(arr));
}
Some(Value::Object(obj))
@@ -9294,7 +10161,10 @@ fn bulk_opts_from_query(qp: &std::collections::HashMap<String, String>) -> xerj_
"true" => Some(Value::Bool(true)),
"false" => Some(Value::Bool(false)),
other => {
- let arr: Vec<Value> = other.split(',').map(|s| Value::String(s.trim().to_string())).collect();
+ let arr: Vec<Value> = other
+ .split(',')
+ .map(|s| Value::String(s.trim().to_string()))
+ .collect();
Some(Value::Array(arr))
}
}
@@ -9362,7 +10232,9 @@ async fn process_bulk_body(
&rewritten_body
};
- let result = xerj_engine::bulk::process_bulk_with_opts(&state.engine, default_index, text_ref, opts).await;
+ let result =
+ xerj_engine::bulk::process_bulk_with_opts(&state.engine, default_index, text_ref, opts)
+ .await;
let took_ms = result.took_ms;
let errors = result.errors;
@@ -9371,15 +10243,16 @@ async fn process_bulk_body(
// urllib) also back off. If only some items are 429 we still return 200
// per ES bulk semantics, and the client is expected to read per-item
// statuses.
- let all_backpressure = !result.items.is_empty()
- && result.items.iter().all(|i| i.status == 429);
+ let all_backpressure = !result.items.is_empty() && result.items.iter().all(|i| i.status == 429);
let mut items: Vec<EsBulkItem> = Vec::with_capacity(result.items.len());
for item in result.items {
- let get = item.get_source.clone().map(|s| serde_json::json!({
- "found": true,
- "_source": s,
- }));
+ let get = item.get_source.clone().map(|s| {
+ serde_json::json!({
+ "found": true,
+ "_source": s,
+ })
+ });
let item_result = EsBulkItemResult {
index: item.index,
id: item.id,
@@ -9393,9 +10266,8 @@ async fn process_bulk_body(
error: item.error.map(|e| {
// Map common per-item error phrases to their ES exception
// type names so clients can match on `error.type`.
- let error_type = if e.starts_with("if _id is specified") {
- "illegal_argument_exception"
- } else if e.contains("invalid document JSON")
+ let error_type = if e.starts_with("if _id is specified")
+ || e.contains("invalid document JSON")
|| e.contains("missing document body")
|| e.contains("unknown action type")
|| e.contains("no write index is defined")
@@ -9406,9 +10278,7 @@ async fn process_bulk_body(
"version_conflict_engine_exception"
} else if e.contains("index not found") || e.starts_with("no such index") {
"index_not_found_exception"
- } else if e.contains("dynamic template")
- || e.contains("failed to parse field")
- {
+ } else if e.contains("dynamic template") || e.contains("failed to parse field") {
"document_parsing_exception"
} else {
"engine_exception"
@@ -9433,7 +10303,11 @@ async fn process_bulk_body(
items.push(action);
}
- let resp = EsBulkResponse { took: took_ms, errors, items };
+ let resp = EsBulkResponse {
+ took: took_ms,
+ errors,
+ items,
+ };
// Fast path (hot bulk-ingest): the Value round-trip below exists ONLY to
// patch `_seq_no`/`_primary_term` sentinels for indices that declare
@@ -9449,12 +10323,16 @@ async fn process_bulk_body(
val.as_bool()
.unwrap_or_else(|| val.as_str().map(|x| x == "true").unwrap_or(false))
};
- s.pointer("/index/disable_sequence_numbers").map(as_bool).unwrap_or(false)
+ s.pointer("/index/disable_sequence_numbers")
+ .map(as_bool)
+ .unwrap_or(false)
|| s.get("index")
.and_then(|i| i.get("index.disable_sequence_numbers"))
.map(as_bool)
.unwrap_or(false)
- || s.get("index.disable_sequence_numbers").map(as_bool).unwrap_or(false)
+ || s.get("index.disable_sequence_numbers")
+ .map(as_bool)
+ .unwrap_or(false)
});
if !seq_disabled_anywhere {
let mut r = Json(&resp).into_response();
@@ -9474,20 +10352,42 @@ async fn process_bulk_body(
// the struct-level u64 types.
let mut resp_val = serde_json::to_value(&resp).unwrap_or(Value::Null);
{
- let items_v = resp_val.pointer_mut("/items").and_then(|v| v.as_array_mut());
+ let items_v = resp_val
+ .pointer_mut("/items")
+ .and_then(|v| v.as_array_mut());
if let Some(items_arr) = items_v {
for item in items_arr.iter_mut() {
if let Some(obj) = item.as_object_mut() {
for action_key in ["index", "create", "update", "delete"] {
if let Some(result) = obj.get_mut(action_key) {
- let ix = result.get("_index").and_then(Value::as_str).unwrap_or("").to_string();
- let disabled = state.engine.index_settings.get(&ix).map(|v| {
- let s = v.clone();
- let as_bool = |val: &Value| val.as_bool().unwrap_or_else(|| val.as_str().map(|x| x == "true").unwrap_or(false));
- s.pointer("/index/disable_sequence_numbers").map(as_bool).unwrap_or(false)
- || s.get("index").and_then(|i| i.get("index.disable_sequence_numbers")).map(as_bool).unwrap_or(false)
- || s.get("index.disable_sequence_numbers").map(as_bool).unwrap_or(false)
- }).unwrap_or(false);
+ let ix = result
+ .get("_index")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
+ let disabled = state
+ .engine
+ .index_settings
+ .get(&ix)
+ .map(|v| {
+ let s = v.clone();
+ let as_bool = |val: &Value| {
+ val.as_bool().unwrap_or_else(|| {
+ val.as_str().map(|x| x == "true").unwrap_or(false)
+ })
+ };
+ s.pointer("/index/disable_sequence_numbers")
+ .map(as_bool)
+ .unwrap_or(false)
+ || s.get("index")
+ .and_then(|i| i.get("index.disable_sequence_numbers"))
+ .map(as_bool)
+ .unwrap_or(false)
+ || s.get("index.disable_sequence_numbers")
+ .map(as_bool)
+ .unwrap_or(false)
+ })
+ .unwrap_or(false);
if disabled {
if let Some(ro) = result.as_object_mut() {
ro.insert("_seq_no".into(), json!(-2i64));
@@ -9736,7 +10636,10 @@ fn resolve_date_math_expr(expr: &str) -> String {
// Split on an inner `{` for optional format: `now/d{yyyy.MM.dd}`.
let (math_expr, fmt) = if let Some(inner_brace) = date_part.find('{') {
let inner_end = date_part.rfind('}').unwrap_or(date_part.len());
- (&date_part[..inner_brace], &date_part[inner_brace + 1..inner_end])
+ (
+ &date_part[..inner_brace],
+ &date_part[inner_brace + 1..inner_end],
+ )
} else {
(date_part, "yyyy.MM.dd")
};
@@ -9756,10 +10659,9 @@ fn resolve_date_math_expr(expr: &str) -> String {
// Accept `yyyy-MM-dd`, `yyyy-MM-ddTHH:mm:ss`, and full RFC3339.
if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(anchor_str) {
dt.with_timezone(&chrono::Utc)
- } else if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(
- anchor_str,
- "%Y-%m-%dT%H:%M:%S",
- ) {
+ } else if let Ok(ndt) =
+ chrono::NaiveDateTime::parse_from_str(anchor_str, "%Y-%m-%dT%H:%M:%S")
+ {
ndt.and_utc()
} else if let Ok(nd) = chrono::NaiveDate::parse_from_str(anchor_str, "%Y-%m-%d") {
nd.and_hms_opt(0, 0, 0)
@@ -9811,7 +10713,7 @@ fn apply_date_math_tail(
"y" => Duration::days(n * 365),
_ => Duration::zero(),
};
- dt = dt + offset;
+ dt += offset;
rest = r3;
} else if let Some(r) = rest.strip_prefix('/') {
let (unit, r2) = if !r.is_empty() {
@@ -9828,10 +10730,7 @@ fn apply_date_math_tail(
dt
}
-fn round_date_down(
- dt: chrono::DateTime<chrono::Utc>,
- unit: &str,
-) -> chrono::DateTime<chrono::Utc> {
+fn round_date_down(dt: chrono::DateTime<chrono::Utc>, unit: &str) -> chrono::DateTime<chrono::Utc> {
use chrono::{Datelike, TimeZone, Timelike};
match unit {
"d" => chrono::Utc
@@ -9859,7 +10758,10 @@ fn round_date_down(
}
/// Parse and evaluate a `now`-based date math expression.
-fn resolve_now_expr(expr: &str, base: chrono::DateTime<chrono::Utc>) -> chrono::DateTime<chrono::Utc> {
+fn resolve_now_expr(
+ expr: &str,
+ base: chrono::DateTime<chrono::Utc>,
+) -> chrono::DateTime<chrono::Utc> {
use chrono::Duration;
// Strip leading "now".
@@ -9873,7 +10775,10 @@ fn resolve_now_expr(expr: &str, base: chrono::DateTime<chrono::Utc>) -> chrono::
let sign = if rest.starts_with('+') { 1i64 } else { -1i64 };
let rest = &rest[1..];
// Parse number.
- let (num_str, rest) = rest.split_at(rest.find(|c: char| !c.is_ascii_digit()).unwrap_or(rest.len()));
+ let (num_str, rest) = rest.split_at(
+ rest.find(|c: char| !c.is_ascii_digit())
+ .unwrap_or(rest.len()),
+ );
let n: i64 = num_str.parse().unwrap_or(0) * sign;
// Parse unit.
let (unit, rest) = if !rest.is_empty() {
@@ -9902,25 +10807,23 @@ fn resolve_now_expr(expr: &str, base: chrono::DateTime<chrono::Utc>) -> chrono::
dt = match unit {
'd' => {
// Round down to day.
- chrono::Utc.with_ymd_and_hms(dt.year(), dt.month(), dt.day(), 0, 0, 0)
- .single()
- .unwrap_or(dt)
- }
- 'h' => {
- chrono::Utc.with_ymd_and_hms(dt.year(), dt.month(), dt.day(), dt.hour(), 0, 0)
- .single()
- .unwrap_or(dt)
- }
- 'M' => {
- chrono::Utc.with_ymd_and_hms(dt.year(), dt.month(), 1, 0, 0, 0)
- .single()
- .unwrap_or(dt)
- }
- 'y' => {
- chrono::Utc.with_ymd_and_hms(dt.year(), 1, 1, 0, 0, 0)
+ chrono::Utc
+ .with_ymd_and_hms(dt.year(), dt.month(), dt.day(), 0, 0, 0)
.single()
.unwrap_or(dt)
}
+ 'h' => chrono::Utc
+ .with_ymd_and_hms(dt.year(), dt.month(), dt.day(), dt.hour(), 0, 0)
+ .single()
+ .unwrap_or(dt),
+ 'M' => chrono::Utc
+ .with_ymd_and_hms(dt.year(), dt.month(), 1, 0, 0, 0)
+ .single()
+ .unwrap_or(dt),
+ 'y' => chrono::Utc
+ .with_ymd_and_hms(dt.year(), 1, 1, 0, 0, 0)
+ .single()
+ .unwrap_or(dt),
_ => dt,
};
}
@@ -10002,7 +10905,10 @@ pub async fn update_doc(
Ok(resp) => {
state.metrics.record_doc_indexed(&index);
let er = crate::responses::EsDocResponse::updated(
- &index, &resp.id, resp.version, resp.seq_no,
+ &index,
+ &resp.id,
+ resp.version,
+ resp.seq_no,
);
Json(er).into_response()
}
@@ -10015,13 +10921,21 @@ pub async fn update_doc(
// not run against the upsert body (matches ES default,
// scripted_upsert=false).
let upsert_body = body.upsert.clone().or_else(|| {
- if body.doc_as_upsert { body.doc.clone() } else { None }
+ if body.doc_as_upsert {
+ body.doc.clone()
+ } else {
+ None
+ }
});
if let Some(up) = upsert_body {
return match idx.index_document(Some(id.clone()), up).await {
Ok(resp) => {
state.metrics.record_doc_indexed(&index);
- let er = crate::responses::EsDocResponse::created(&index, &resp.id, resp.seq_no);
+ let er = crate::responses::EsDocResponse::created(
+ &index,
+ &resp.id,
+ resp.seq_no,
+ );
Json(er).into_response()
}
Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
@@ -10034,15 +10948,18 @@ pub async fn update_doc(
}
}
- match idx.update_document_with_upsert(
- &id,
- body.doc,
- body.upsert,
- body.doc_as_upsert,
- ).await {
+ match idx
+ .update_document_with_upsert(&id, body.doc, body.upsert, body.doc_as_upsert)
+ .await
+ {
Ok(Some(resp)) => {
state.metrics.record_doc_indexed(&index);
- let er = crate::responses::EsDocResponse::updated(&index, &resp.id, resp.version, resp.seq_no);
+ let er = crate::responses::EsDocResponse::updated(
+ &index,
+ &resp.id,
+ resp.version,
+ resp.seq_no,
+ );
Json(er).into_response()
}
Ok(None) => {
@@ -10205,7 +11122,8 @@ fn painless_update_value(v: xerj_engine::painless::PainlessValue) -> Value {
/// is an expression that must be evaluated.
fn parse_rhs_literal(rhs: &str) -> Option<Value> {
let t = rhs.trim();
- if t.len() >= 2 && t.starts_with('\'') && t.ends_with('\'') && !t[1..t.len() - 1].contains('\'') {
+ if t.len() >= 2 && t.starts_with('\'') && t.ends_with('\'') && !t[1..t.len() - 1].contains('\'')
+ {
return Some(Value::String(t[1..t.len() - 1].to_string()));
}
serde_json::from_str::<Value>(t).ok()
@@ -10222,7 +11140,11 @@ fn eval_update_expr(expr: &str, source: &Value, params: &Value) -> Result<Value,
/// Apply a painless update script to `source` in place. Supports the common
/// `ctx._source.*` mutation forms: assignment (`=`), compound assignment
/// (`+= -= *= /=`), increment / decrement (`++ --`), and `remove(...)`.
-fn apply_painless_update(source: &mut Value, script_src: &str, params: &Value) -> Result<(), String> {
+fn apply_painless_update(
+ source: &mut Value,
+ script_src: &str,
+ params: &Value,
+) -> Result<(), String> {
for raw in split_update_statements(script_src) {
let stmt = raw.trim();
if stmt.is_empty() {
@@ -10301,9 +11223,11 @@ fn apply_one_update_stmt(source: &mut Value, stmt: &str, params: &Value) -> Resu
if opr == "++" || opr == "--" {
let cur = get_source_path(source, &path).as_f64().unwrap_or(0.0);
let nv = if opr == "++" { cur + 1.0 } else { cur - 1.0 };
- set_source_path(source, &path, painless_update_value(
- xerj_engine::painless::PainlessValue::Number(nv),
- ));
+ set_source_path(
+ source,
+ &path,
+ painless_update_value(xerj_engine::painless::PainlessValue::Number(nv)),
+ );
return Ok(());
}
@@ -10412,7 +11336,9 @@ pub async fn count_docs(
.unwrap_or(false);
let mut total: u64 = 0;
for ix_name in &wanted {
- let Ok(idx) = state.engine.get_index(ix_name) else { continue };
+ let Ok(idx) = state.engine.get_index(ix_name) else {
+ continue;
+ };
if has_query {
let query_val = body
.as_ref()
@@ -10612,7 +11538,11 @@ pub async fn mget_index(
.and_then(Value::as_str)
.unwrap_or(&index)
.to_string();
- let id = d.get("_id").and_then(Value::as_str).unwrap_or("").to_string();
+ let id = d
+ .get("_id")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
entries.push((i, id));
}
}
@@ -10679,12 +11609,13 @@ pub async fn cat_health(State(state): State<AppState>) -> impl IntoResponse {
let now = Utc::now();
let ts = now.format("%H:%M:%S").to_string();
let shards = health.index_count as u32;
- let body = format!(
- "{epoch} {ts} xerj green 1 1 {shards} {shards} 0 0 0 0 - 100.0%\n"
- );
+ let body = format!("{epoch} {ts} xerj green 1 1 {shards} {shards} 0 0 0 0 - 100.0%\n");
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -10719,7 +11650,10 @@ pub async fn cat_nodes(State(state): State<AppState>) -> impl IntoResponse {
);
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -10894,7 +11828,8 @@ fn tokenize_for_analyze(input: &str, analyzer: &str) -> Vec<Value> {
let bytes = input.as_bytes();
let mut i = 0usize;
while i <= input.len() {
- let is_alnum = i < input.len() && (bytes[i].is_ascii_alphanumeric() || bytes[i] > 127);
+ let is_alnum =
+ i < input.len() && (bytes[i].is_ascii_alphanumeric() || bytes[i] > 127);
if is_alnum && !in_tok {
start = i;
in_tok = true;
@@ -10942,33 +11877,41 @@ pub async fn index_disk_usage(
Err(_) => continue,
};
all_size += size;
- indices.insert(name.clone(), json!({
- "store_size": format!("{}b", size),
- "store_size_in_bytes": size,
- "all_fields": {
- "total": format!("{}b", size),
- "total_in_bytes": size,
- "inverted_index": { "total": "0b", "total_in_bytes": 0 },
- "stored_fields": format!("{}b", size),
- "stored_fields_in_bytes": size,
- "doc_values": "0b",
- "doc_values_in_bytes": 0,
- "points": "0b",
- "points_in_bytes": 0,
- "norms": "0b",
- "norms_in_bytes": 0,
- "term_vectors": "0b",
- "term_vectors_in_bytes": 0,
- "knn_vectors": "0b",
- "knn_vectors_in_bytes": 0,
- },
- "fields": {}
- }));
+ indices.insert(
+ name.clone(),
+ json!({
+ "store_size": format!("{}b", size),
+ "store_size_in_bytes": size,
+ "all_fields": {
+ "total": format!("{}b", size),
+ "total_in_bytes": size,
+ "inverted_index": { "total": "0b", "total_in_bytes": 0 },
+ "stored_fields": format!("{}b", size),
+ "stored_fields_in_bytes": size,
+ "doc_values": "0b",
+ "doc_values_in_bytes": 0,
+ "points": "0b",
+ "points_in_bytes": 0,
+ "norms": "0b",
+ "norms_in_bytes": 0,
+ "term_vectors": "0b",
+ "term_vectors_in_bytes": 0,
+ "knn_vectors": "0b",
+ "knn_vectors_in_bytes": 0,
+ },
+ "fields": {}
+ }),
+ );
}
let mut out = serde_json::Map::new();
- out.insert("_shards".to_string(), json!({"total":1,"successful":1,"failed":0}));
+ out.insert(
+ "_shards".to_string(),
+ json!({"total":1,"successful":1,"failed":0}),
+ );
let _ = all_size;
- for (k, v) in indices { out.insert(k, v); }
+ for (k, v) in indices {
+ out.insert(k, v);
+ }
Json(Value::Object(out)).into_response()
}
@@ -11007,7 +11950,12 @@ pub async fn index_stats(
let (seg_count, merging) = state
.engine
.get_index(name)
- .map(|idx| (idx.store_snapshot().segments.len(), idx.is_merge_in_progress()))
+ .map(|idx| {
+ (
+ idx.store_snapshot().segments.len(),
+ idx.is_merge_in_progress(),
+ )
+ })
.unwrap_or((0, false));
let primaries = json!({
"docs": { "count": doc_count, "deleted": 0 },
@@ -11020,10 +11968,13 @@ pub async fn index_stats(
},
"segments": { "count": seg_count, "memory_in_bytes": 0 },
});
- all_indices.insert(name.clone(), json!({
- "primaries": primaries,
- "total": primaries,
- }));
+ all_indices.insert(
+ name.clone(),
+ json!({
+ "primaries": primaries,
+ "total": primaries,
+ }),
+ );
}
// Build _all by summing per-index stats. dense_vector merges by
// summing every numeric leaf and dropping per-index `fielddata`.
@@ -11039,13 +11990,34 @@ pub async fn index_stats(
let dv = per_index_dense_vector_stats(&state, name);
let off = dv.get("off_heap").and_then(|v| v.as_object());
if let Some(o) = off {
- all_dv_size += o.get("total_size_bytes").and_then(Value::as_u64).unwrap_or(0);
- all_dv_vec += o.get("total_vec_size_bytes").and_then(Value::as_u64).unwrap_or(0);
- all_dv_veb += o.get("total_veb_size_bytes").and_then(Value::as_u64).unwrap_or(0);
- all_dv_veq += o.get("total_veq_size_bytes").and_then(Value::as_u64).unwrap_or(0);
- all_dv_vex += o.get("total_vex_size_bytes").and_then(Value::as_u64).unwrap_or(0);
- all_dv_cenivf += o.get("total_cenivf_size_bytes").and_then(Value::as_u64).unwrap_or(0);
- all_dv_clivf += o.get("total_clivf_size_bytes").and_then(Value::as_u64).unwrap_or(0);
+ all_dv_size += o
+ .get("total_size_bytes")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
+ all_dv_vec += o
+ .get("total_vec_size_bytes")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
+ all_dv_veb += o
+ .get("total_veb_size_bytes")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
+ all_dv_veq += o
+ .get("total_veq_size_bytes")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
+ all_dv_vex += o
+ .get("total_vex_size_bytes")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
+ all_dv_cenivf += o
+ .get("total_cenivf_size_bytes")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
+ all_dv_clivf += o
+ .get("total_clivf_size_bytes")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
}
all_dv_value_count += dv.get("value_count").and_then(Value::as_u64).unwrap_or(0);
}
@@ -11069,7 +12041,8 @@ pub async fn index_stats(
"_shards": { "total": 1, "successful": 1, "failed": 0 },
"_all": { "primaries": all_primaries, "total": all_primaries },
"indices": all_indices,
- })).into_response();
+ }))
+ .into_response();
}
let stats = match state.engine.index_stats(&index).await {
Ok(s) => s,
@@ -11086,25 +12059,35 @@ pub async fn index_stats(
.unwrap_or(0);
// Pull the live noop + request-cache counters from the index.
- let (noop_total, rc_hit, rc_miss, get_total, get_ms, get_exists, get_missing) = match state.engine.get_index(&index) {
- Ok(idx) => (
- idx.noop_update_total(),
- idx.request_cache_hit_count(),
- idx.request_cache_miss_count(),
- idx.metric_get_count.load(std::sync::atomic::Ordering::Relaxed),
- idx.metric_get_total_ms.load(std::sync::atomic::Ordering::Relaxed),
- idx.metric_get_exists_count.load(std::sync::atomic::Ordering::Relaxed),
- idx.metric_get_missing_count.load(std::sync::atomic::Ordering::Relaxed),
- ),
- Err(_) => (0, 0, 0, 0, 0, 0, 0),
- };
+ let (noop_total, rc_hit, rc_miss, get_total, get_ms, get_exists, get_missing) =
+ match state.engine.get_index(&index) {
+ Ok(idx) => (
+ idx.noop_update_total(),
+ idx.request_cache_hit_count(),
+ idx.request_cache_miss_count(),
+ idx.metric_get_count
+ .load(std::sync::atomic::Ordering::Relaxed),
+ idx.metric_get_total_ms
+ .load(std::sync::atomic::Ordering::Relaxed),
+ idx.metric_get_exists_count
+ .load(std::sync::atomic::Ordering::Relaxed),
+ idx.metric_get_missing_count
+ .load(std::sync::atomic::Ordering::Relaxed),
+ ),
+ Err(_) => (0, 0, 0, 0, 0, 0, 0),
+ };
let dense_vector_stats = per_index_dense_vector_stats(&state, &index);
// Merge status + real segment count so benchmarks can verify the
// index is merge-quiescent (merges.current == 0) before measuring.
let (seg_count, merges_current) = state
.engine
.get_index(&index)
- .map(|idx| (idx.store_snapshot().segments.len(), idx.is_merge_in_progress()))
+ .map(|idx| {
+ (
+ idx.store_snapshot().segments.len(),
+ idx.is_merge_in_progress(),
+ )
+ })
.unwrap_or((0, false));
let primaries = json!({
"docs": { "count": doc_count, "deleted": 0 },
@@ -11177,7 +12160,10 @@ pub async fn index_stats(
// dense_vector stats but keeps it on the per-index entries (per
// 220_dense_vector_node_bbq_disk_index_stats).
let mut all_primaries = primaries.clone();
- if let Some(dv) = all_primaries.pointer_mut("/dense_vector/off_heap").and_then(|v| v.as_object_mut()) {
+ if let Some(dv) = all_primaries
+ .pointer_mut("/dense_vector/off_heap")
+ .and_then(|v| v.as_object_mut())
+ {
dv.remove("fielddata");
}
Json(json!({
@@ -11385,10 +12371,16 @@ pub async fn put_index_template(
Path(name): Path<String>,
Json(body): Json<IndexTemplateBody>,
) -> impl IntoResponse {
- let settings = body.template.as_ref().and_then(|t| t.settings.clone())
+ let settings = body
+ .template
+ .as_ref()
+ .and_then(|t| t.settings.clone())
.or(body.settings.clone())
.unwrap_or(json!({}));
- let mappings = body.template.as_ref().and_then(|t| t.mappings.clone())
+ let mappings = body
+ .template
+ .as_ref()
+ .and_then(|t| t.mappings.clone())
.or(body.mappings.clone())
.unwrap_or(json!({}));
@@ -11411,17 +12403,20 @@ pub async fn get_index_template(
let mut templates = serde_json::Map::new();
for entry in state.engine.templates.iter() {
let t = entry.value();
- templates.insert(entry.key().clone(), json!({
- "name": entry.key().clone(),
- "index_template": {
- "index_patterns": t.index_patterns,
- "priority": t.priority,
- "template": {
- "settings": t.settings,
- "mappings": t.mappings,
+ templates.insert(
+ entry.key().clone(),
+ json!({
+ "name": entry.key().clone(),
+ "index_template": {
+ "index_patterns": t.index_patterns,
+ "priority": t.priority,
+ "template": {
+ "settings": t.settings,
+ "mappings": t.mappings,
+ }
}
- }
- }));
+ }),
+ );
}
let index_templates: Vec<Value> = templates.values().cloned().collect();
return Json(json!({ "index_templates": index_templates })).into_response();
@@ -11445,7 +12440,8 @@ pub async fn get_index_template(
Json(resp).into_response()
}
None => {
- let e = xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));
+ let e =
+ xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));
ApiError::new(e).into_response()
}
}
@@ -11563,7 +12559,8 @@ pub async fn search_with_scroll(
if params.scroll.is_some() {
let scroll_id = Uuid::new_v4().to_string();
// Extract just the hits from the pairs.
- let hits_only: Vec<xerj_query::executor::Hit> = all_hits.iter().map(|(_, h)| h.clone()).collect();
+ let hits_only: Vec<xerj_query::executor::Hit> =
+ all_hits.iter().map(|(_, h)| h.clone()).collect();
// Return first page.
let first_page: Vec<EsHit> = all_hits
@@ -11576,13 +12573,31 @@ pub async fn search_with_scroll(
version: Some(1),
seq_no: Some(0),
primary_term: Some(1),
- source: if h.source.is_null() { None } else { Some(h.source.clone()) },
+ source: if h.source.is_null() {
+ None
+ } else {
+ Some(h.source.clone())
+ },
fields: None,
- sort: if h.sort.is_empty() { None } else { Some(h.sort.clone()) },
+ sort: if h.sort.is_empty() {
+ None
+ } else {
+ Some(h.sort.clone())
+ },
highlight: h.highlight.clone(),
explanation: None,
inner_hits: None,
- matched_queries: if h.matched_queries.is_empty() { Value::Null } else { Value::Array(h.matched_queries.iter().cloned().map(Value::String).collect()) },
+ matched_queries: if h.matched_queries.is_empty() {
+ Value::Null
+ } else {
+ Value::Array(
+ h.matched_queries
+ .iter()
+ .cloned()
+ .map(Value::String)
+ .collect(),
+ )
+ },
ignored: None,
ignored_field_values: None,
})
@@ -11630,13 +12645,31 @@ pub async fn search_with_scroll(
version: Some(1),
seq_no: Some(0),
primary_term: Some(1),
- source: if h.source.is_null() { None } else { Some(h.source.clone()) },
+ source: if h.source.is_null() {
+ None
+ } else {
+ Some(h.source.clone())
+ },
fields: None,
- sort: if h.sort.is_empty() { None } else { Some(h.sort.clone()) },
+ sort: if h.sort.is_empty() {
+ None
+ } else {
+ Some(h.sort.clone())
+ },
highlight: h.highlight.clone(),
explanation: None,
inner_hits: None,
- matched_queries: if h.matched_queries.is_empty() { Value::Null } else { Value::Array(h.matched_queries.iter().cloned().map(Value::String).collect()) },
+ matched_queries: if h.matched_queries.is_empty() {
+ Value::Null
+ } else {
+ Value::Array(
+ h.matched_queries
+ .iter()
+ .cloned()
+ .map(Value::String)
+ .collect(),
+ )
+ },
ignored: None,
ignored_field_values: None,
})
@@ -11648,7 +12681,10 @@ pub async fn search_with_scroll(
timed_out: false,
shards: crate::responses::EsShards::search_success(),
hits: EsHits {
- total: EsHitsTotal { value: total_count, relation: "eq".to_string() },
+ total: EsHitsTotal {
+ value: total_count,
+ relation: "eq".to_string(),
+ },
max_score,
hits: page,
},
@@ -11698,7 +12734,9 @@ pub async fn next_scroll(
// Detect whether the initial search sorted by a non-score key;
// in that case `max_score` must be null on scroll pages too.
- let has_non_score_sort = ctx.hits.first()
+ let has_non_score_sort = ctx
+ .hits
+ .first()
.map(|h| !h.sort.is_empty())
.unwrap_or(false);
@@ -11714,13 +12752,31 @@ pub async fn next_scroll(
version: Some(1),
seq_no: Some(0),
primary_term: Some(1),
- source: if h.source.is_null() { None } else { Some(h.source.clone()) },
+ source: if h.source.is_null() {
+ None
+ } else {
+ Some(h.source.clone())
+ },
fields: None,
- sort: if h.sort.is_empty() { None } else { Some(h.sort.clone()) },
+ sort: if h.sort.is_empty() {
+ None
+ } else {
+ Some(h.sort.clone())
+ },
highlight: h.highlight.clone(),
explanation: None,
inner_hits: None,
- matched_queries: if h.matched_queries.is_empty() { Value::Null } else { Value::Array(h.matched_queries.iter().cloned().map(Value::String).collect()) },
+ matched_queries: if h.matched_queries.is_empty() {
+ Value::Null
+ } else {
+ Value::Array(
+ h.matched_queries
+ .iter()
+ .cloned()
+ .map(Value::String)
+ .collect(),
+ )
+ },
ignored: None,
ignored_field_values: None,
})
@@ -11767,7 +12823,9 @@ pub async fn next_scroll(
Json(resp).into_response()
}
None => {
- let e = xerj_common::XerjError::index_not_found(format!("No search context found for id [{scroll_id}]"));
+ let e = xerj_common::XerjError::index_not_found(format!(
+ "No search context found for id [{scroll_id}]"
+ ));
ApiError::new(e).into_response()
}
}
@@ -11891,7 +12949,11 @@ pub async fn reindex(
Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),
};
- let query_val = body.source.query.clone().unwrap_or(json!({ "match_all": {} }));
+ let query_val = body
+ .source
+ .query
+ .clone()
+ .unwrap_or(json!({ "match_all": {} }));
let page_size = body.source.size.min(10_000); // cap per-batch at 10k
let max_total = 100_000usize; // safety cap for reindex total
@@ -11933,10 +12995,22 @@ pub async fn reindex(
for hit in results.hits {
if !hit.source.is_null() {
// Check if doc already exists in dest to track created vs updated.
- let exists = dest_idx.get_document(&hit.id).await.ok().flatten().is_some();
- match dest_idx.index_document(Some(hit.id.clone()), hit.source).await {
+ let exists = dest_idx
+ .get_document(&hit.id)
+ .await
+ .ok()
+ .flatten()
+ .is_some();
+ match dest_idx
+ .index_document(Some(hit.id.clone()), hit.source)
+ .await
+ {
Ok(_) => {
- if exists { updated += 1; } else { created += 1; }
+ if exists {
+ updated += 1;
+ } else {
+ created += 1;
+ }
}
Err(e) => {
failures.push(json!({
@@ -11992,9 +13066,18 @@ pub async fn field_caps(
) -> impl IntoResponse {
// Support "*" as a wildcard for all indices.
let resolved_indices: Vec<String> = if index == "*" || index == "_all" {
- state.engine.list_indices().await.into_iter().map(|i| i.name).collect()
+ state
+ .engine
+ .list_indices()
+ .await
+ .into_iter()
+ .map(|i| i.name)
+ .collect()
} else {
- index.split(',').flat_map(|n| state.engine.resolve_alias(n.trim())).collect()
+ index
+ .split(',')
+ .flat_map(|n| state.engine.resolve_alias(n.trim()))
+ .collect()
};
let fields_filter = params.fields.as_deref().unwrap_or("*");
@@ -12149,7 +13232,8 @@ async fn msearch_impl(
// PIT override: when the per-item body has pit.id, resolve
// the PIT context and use its indices list + index_filter.
- let pit_context: Option<xerj_engine::engine::PitContext> = search_body_val.get("pit")
+ let pit_context: Option<xerj_engine::engine::PitContext> = search_body_val
+ .get("pit")
.and_then(|p| p.get("id"))
.and_then(Value::as_str)
.and_then(|id| state.engine.pits.get(id).map(|r| r.value().clone()));
@@ -12206,15 +13290,27 @@ async fn msearch_impl(
let index_names: Vec<String> = if let Some(pit) = pit_context.as_ref() {
pit.indices.clone()
} else if index_name == "*" || index_name == "_all" {
- state.engine.list_indices().await.into_iter().map(|i| i.name).collect()
+ state
+ .engine
+ .list_indices()
+ .await
+ .into_iter()
+ .map(|i| i.name)
+ .collect()
} else {
- index_name.split(',').flat_map(|n| state.engine.resolve_alias(n.trim())).collect()
+ index_name
+ .split(',')
+ .flat_map(|n| state.engine.resolve_alias(n.trim()))
+ .collect()
};
let index_names: Vec<String> = if idx_constraints.is_empty() {
index_names
} else {
- index_names.into_iter().filter(|n| idx_constraints.iter().any(|p| p == n || glob_match(p, n))).collect()
+ index_names
+ .into_iter()
+ .filter(|n| idx_constraints.iter().any(|p| p == n || glob_match(p, n)))
+ .collect()
};
let mut merged_hits: Vec<(String, xerj_query::executor::Hit)> = Vec::new();
@@ -12278,7 +13374,11 @@ async fn msearch_impl(
let hits: Vec<Value> = merged_hits
.into_iter()
.map(|(idx_name, h)| {
- let source = if h.source.is_null() { None } else { Some(h.source.clone()) };
+ let source = if h.source.is_null() {
+ None
+ } else {
+ Some(h.source.clone())
+ };
json!({
"_index": idx_name,
"_id": h.id,
@@ -12337,15 +13437,29 @@ fn suggest_edit_distance(a: &str, b: &str) -> usize {
let b: Vec<char> = b.chars().collect();
let m = a.len();
let n = b.len();
- if m == 0 { return n; }
- if n == 0 { return m; }
+ if m == 0 {
+ return n;
+ }
+ if n == 0 {
+ return m;
+ }
let mut dp = vec![vec![0usize; n + 1]; m + 1];
- for i in 0..=m { dp[i][0] = i; }
- for j in 0..=n { dp[0][j] = j; }
+ // `i`/`j` are both the index into the 2D `dp` grid and the value written;
+ // the range form is the clearest expression of this DP initialization.
+ #[allow(clippy::needless_range_loop)]
+ for i in 0..=m {
+ dp[i][0] = i;
+ }
+ #[allow(clippy::needless_range_loop)]
+ for j in 0..=n {
+ dp[0][j] = j;
+ }
for i in 1..=m {
for j in 1..=n {
let cost = if a[i - 1] == b[j - 1] { 0 } else { 1 };
- dp[i][j] = (dp[i - 1][j] + 1).min(dp[i][j - 1] + 1).min(dp[i - 1][j - 1] + cost);
+ dp[i][j] = (dp[i - 1][j] + 1)
+ .min(dp[i][j - 1] + 1)
+ .min(dp[i - 1][j - 1] + cost);
}
}
dp[m][n]
@@ -12368,7 +13482,10 @@ pub fn process_suggest(suggest_body: &Value, docs: &[Value]) -> Value {
// ── Completion suggester ──────────────────────────────────────────────
// {"prefix": "sea", "completion": {"field": "suggest"}}
if let Some(completion_opts) = suggest_def.get("completion") {
- let prefix = suggest_def.get("prefix").and_then(Value::as_str).unwrap_or("");
+ let prefix = suggest_def
+ .get("prefix")
+ .and_then(Value::as_str)
+ .unwrap_or("");
let field = completion_opts
.get("field")
.and_then(Value::as_str)
@@ -12400,18 +13517,24 @@ pub fn process_suggest(suggest_body: &Value, docs: &[Value]) -> Value {
.map(|(text, _freq)| json!({ "text": text, "_score": 1.0, "_source": {} }))
.collect();
- result.insert(suggest_name.clone(), Value::Array(vec![json!({
- "text": prefix,
- "offset": 0,
- "length": prefix.len(),
- "options": options,
- })]));
+ result.insert(
+ suggest_name.clone(),
+ Value::Array(vec![json!({
+ "text": prefix,
+ "offset": 0,
+ "length": prefix.len(),
+ "options": options,
+ })]),
+ );
continue;
}
// ── Term suggester ────────────────────────────────────────────────────
// {"text": "...", "term": {"field": "..."}}
- let text = suggest_def.get("text").and_then(Value::as_str).unwrap_or("");
+ let text = suggest_def
+ .get("text")
+ .and_then(Value::as_str)
+ .unwrap_or("");
let term_opts = suggest_def.get("term");
let field = term_opts
.and_then(|t| t.get("field"))
@@ -12448,7 +13571,9 @@ pub fn process_suggest(suggest_body: &Value, docs: &[Value]) -> Value {
.iter()
.filter(|(term, _)| {
// Always include prefix matches; also include edit-distance matches.
- if term.as_str() == input_lower { return false; } // skip exact match
+ if term.as_str() == input_lower {
+ return false;
+ } // skip exact match
let dist = suggest_edit_distance(term.as_str(), &input_lower);
dist <= max_edits
})
@@ -12517,7 +13642,10 @@ pub fn process_suggest_with_terms(
// ── Completion suggester ──────────────────────────────────────────────
// {"prefix": "sea", "completion": {"field": "suggest"}}
if let Some(completion_opts) = suggest_def.get("completion") {
- let prefix = suggest_def.get("prefix").and_then(Value::as_str).unwrap_or("");
+ let prefix = suggest_def
+ .get("prefix")
+ .and_then(Value::as_str)
+ .unwrap_or("");
let field = completion_opts
.get("field")
.and_then(Value::as_str)
@@ -12560,18 +13688,24 @@ pub fn process_suggest_with_terms(
.map(|(text, _freq)| json!({ "text": text, "_score": 1.0, "_source": {} }))
.collect();
- result.insert(suggest_name.clone(), Value::Array(vec![json!({
- "text": prefix,
- "offset": 0,
- "length": prefix.len(),
- "options": options,
- })]));
+ result.insert(
+ suggest_name.clone(),
+ Value::Array(vec![json!({
+ "text": prefix,
+ "offset": 0,
+ "length": prefix.len(),
+ "options": options,
+ })]),
+ );
continue;
}
// ── Term suggester ────────────────────────────────────────────────────
// {"text": "...", "term": {"field": "..."}}
- let text = suggest_def.get("text").and_then(Value::as_str).unwrap_or("");
+ let text = suggest_def
+ .get("text")
+ .and_then(Value::as_str)
+ .unwrap_or("");
let term_opts = suggest_def.get("term");
let field = term_opts
.and_then(|t| t.get("field"))
@@ -12613,7 +13747,9 @@ pub fn process_suggest_with_terms(
let mut matches: Vec<(String, u64, usize)> = term_counts
.iter()
.filter(|(term, _)| {
- if term.as_str() == input_lower { return false; } // skip exact match
+ if term.as_str() == input_lower {
+ return false;
+ } // skip exact match
let dist = suggest_edit_distance(term.as_str(), &input_lower);
dist <= max_edits
})
@@ -12729,8 +13865,10 @@ struct SigTextSpec {
fn collect_sig_text_specs(aggs: &Value, parent_field: Option<&str>, out: &mut Vec<SigTextSpec>) {
let Some(obj) = aggs.as_object() else { return };
for (name, spec) in obj {
- let Some(spec_obj) = spec.as_object() else { continue };
- let (agg_type, agg_cfg) = match spec_obj
+ let Some(spec_obj) = spec.as_object() else {
+ continue;
+ };
+ let (agg_type, agg_cfg) = match spec_obj
.iter()
.find(|(k, _)| !matches!(k.as_str(), "aggs" | "aggregations" | "meta"))
{
@@ -12750,12 +13888,16 @@ fn collect_sig_text_specs(aggs: &Value, parent_field: Option<&str>, out: &mut Ve
}
// A bucketing agg becomes the owning grouping for its descendants.
let child_parent = match agg_type {
- "terms" | "significant_terms" | "histogram" | "date_histogram" | "range" => {
- agg_cfg.get("field").and_then(Value::as_str).or(parent_field)
- }
+ "terms" | "significant_terms" | "histogram" | "date_histogram" | "range" => agg_cfg
+ .get("field")
+ .and_then(Value::as_str)
+ .or(parent_field),
_ => parent_field,
};
- if let Some(children) = spec_obj.get("aggs").or_else(|| spec_obj.get("aggregations")) {
+ if let Some(children) = spec_obj
+ .get("aggs")
+ .or_else(|| spec_obj.get("aggregations"))
+ {
collect_sig_text_specs(children, child_parent, out);
}
}
@@ -12835,17 +13977,6 @@ fn compute_sig_text_debug(spec: &SigTextSpec, fg_docs: &[Value]) -> Value {
})
}
-/// Build the `aggregations` block of the `profile.shards[0]` response.
-///
-/// Walks the aggregation request tree and emits one node per agg whose
-/// `type` field carries the ES Java class name. xerj doesn't own those
-/// classes; this mapping is a translation layer over our executor so the
-/// published profile shape matches ES.
-fn build_aggregation_profile(aggs: Option<&Value>, took_ms: u64) -> Vec<Value> {
- let empty = std::collections::HashMap::new();
- build_aggregation_profile_full(aggs, took_ms, 1, None, true, &empty)
-}
-
fn build_aggregation_profile_full(
aggs: Option<&Value>,
took_ms: u64,
@@ -12854,7 +13985,15 @@ fn build_aggregation_profile_full(
terms_use_filter_path: bool,
sig_text_debug: &std::collections::HashMap<String, Value>,
) -> Vec<Value> {
- build_aggregation_profile_full_at(aggs, took_ms, collect_count, results, terms_use_filter_path, false, sig_text_debug)
+ build_aggregation_profile_full_at(
+ aggs,
+ took_ms,
+ collect_count,
+ results,
+ terms_use_filter_path,
+ false,
+ sig_text_debug,
+ )
}
fn build_aggregation_profile_full_at(
@@ -12867,16 +14006,21 @@ fn build_aggregation_profile_full_at(
sig_text_debug: &std::collections::HashMap<String, Value>,
) -> Vec<Value> {
let Some(aggs) = aggs else { return Vec::new() };
- let Some(obj) = aggs.as_object() else { return Vec::new() };
+ let Some(obj) = aggs.as_object() else {
+ return Vec::new();
+ };
let mut out: Vec<Value> = Vec::new();
for (name, spec) in obj {
- let Some(spec_obj) = spec.as_object() else { continue };
+ let Some(spec_obj) = spec.as_object() else {
+ continue;
+ };
// First one-key entry (besides aggs/aggregations/meta) is the agg
// type. Sub-aggregations live under "aggs" or "aggregations".
- let (agg_type, agg_cfg) = match spec_obj.iter().find(|(k, _)| {
- !matches!(k.as_str(), "aggs" | "aggregations" | "meta")
- }) {
+ let (agg_type, agg_cfg) = match spec_obj
+ .iter()
+ .find(|(k, _)| !matches!(k.as_str(), "aggs" | "aggregations" | "meta"))
+ {
Some((t, c)) => (t.as_str(), c),
None => continue,
};
@@ -12893,15 +14037,15 @@ fn build_aggregation_profile_full_at(
.and_then(Value::as_array)
.map(|a| {
a.iter()
- .filter(|b| {
- b.get("doc_count").and_then(Value::as_u64).unwrap_or(0) > 0
- })
+ .filter(|b| b.get("doc_count").and_then(Value::as_u64).unwrap_or(0) > 0)
.count() as u64
});
// Sub-agg tree, if any. Capture the sub-agg names so the parent
// node's debug payload can list deferred aggregator names.
- let children_spec = spec_obj.get("aggs").or_else(|| spec_obj.get("aggregations"));
+ let children_spec = spec_obj
+ .get("aggs")
+ .or_else(|| spec_obj.get("aggregations"));
let sub_agg_names: Vec<String> = children_spec
.and_then(Value::as_object)
.map(|o| o.keys().cloned().collect())
@@ -12922,7 +14066,8 @@ fn build_aggregation_profile_full_at(
// number of docs a sub-agg collector observes for
// bucketed parents (including the FromFilters path,
// which visits each filter's docs sequentially).
- let summed: u64 = arr.iter()
+ let summed: u64 = arr
+ .iter()
.filter_map(|b| b.get("doc_count").and_then(Value::as_u64))
.sum();
if summed > 0 {
@@ -12951,15 +14096,13 @@ fn build_aggregation_profile_full_at(
// "am I a sub-agg?" flag so nested terms stay on the
// ordinals aggregator (filter-by-filter is a top-level
// optimization).
- let class_name = es_aggregator_class_name_ctx(agg_type, agg_cfg, terms_use_filter_path, is_sub_level);
+ let class_name =
+ es_aggregator_class_name_ctx(agg_type, agg_cfg, terms_use_filter_path, is_sub_level);
let mut node = serde_json::Map::new();
node.insert("type".to_string(), Value::String(class_name));
node.insert("description".to_string(), Value::String(name.clone()));
- node.insert(
- "time_in_nanos".to_string(),
- json!(took_ms * 1_000_000u64),
- );
+ node.insert("time_in_nanos".to_string(), json!(took_ms * 1_000_000u64));
// Breakdown numbers are in nanoseconds. Even a fast agg takes some
// wall-clock time to initialize; the `gt 0` assertions in the YAML
// tests reject a literal zero. Floor every phase at 1 ns so those
@@ -13024,7 +14167,8 @@ fn build_aggregation_profile_full_at(
// reports `collect_count: 0` on the parent and exposes a
// `debug.delegate` block listing the child filter queries
// (bucket keys, alphabetically sorted, formatted `field:term`).
- let is_from_filters = node.get("type")
+ let is_from_filters = node
+ .get("type")
.and_then(Value::as_str)
.map(|s| s.ends_with("FromFilters"))
.unwrap_or(false);
@@ -13039,9 +14183,7 @@ fn build_aggregation_profile_full_at(
if is_terms {
if let Some(Value::Object(dbg)) = node.get_mut("debug") {
if dbg.contains_key("segments_with_single_valued_ords") {
- let multi = bucket_count
- .map(|bc| bc > collect_count)
- .unwrap_or(false);
+ let multi = bucket_count.map(|bc| bc > collect_count).unwrap_or(false);
dbg.insert(
"segments_with_single_valued_ords".into(),
json!(if multi { 0u64 } else { 1u64 }),
@@ -13069,31 +14211,41 @@ fn build_aggregation_profile_full_at(
let mut bucket_keys: Vec<String> = this_result
.and_then(|r| r.get("buckets"))
.and_then(Value::as_array)
- .map(|a| a.iter().filter_map(|b| {
- b.get("key").and_then(|k| match k {
- Value::String(s) => Some(s.clone()),
- Value::Number(n) => Some(n.to_string()),
- Value::Bool(b) => Some(b.to_string()),
- _ => None,
- })
- }).collect())
+ .map(|a| {
+ a.iter()
+ .filter_map(|b| {
+ b.get("key").and_then(|k| match k {
+ Value::String(s) => Some(s.clone()),
+ Value::Number(n) => Some(n.to_string()),
+ Value::Bool(b) => Some(b.to_string()),
+ _ => None,
+ })
+ })
+ .collect()
+ })
.unwrap_or_default();
bucket_keys.sort();
- let filters: Vec<Value> = bucket_keys.into_iter().map(|k| {
- json!({
- "query": format!("{}:{}", field_name, k),
- "specialized_for": "term",
- "results_from_metadata": 0u64,
- "segments_counted_in_constant_time": 0u64,
+ let filters: Vec<Value> = bucket_keys
+ .into_iter()
+ .map(|k| {
+ json!({
+ "query": format!("{}:{}", field_name, k),
+ "specialized_for": "term",
+ "results_from_metadata": 0u64,
+ "segments_counted_in_constant_time": 0u64,
+ })
})
- }).collect();
+ .collect();
if let Some(Value::Object(dbg)) = node.get_mut("debug") {
dbg.insert("delegate".to_string(), json!("FilterByFilterAggregator"));
- dbg.insert("delegate_debug".to_string(), json!({
- "segments_counted_in_constant_time": 0u64,
- "segments_with_deleted_docs": 0u64,
- "filters": filters,
- }));
+ dbg.insert(
+ "delegate_debug".to_string(),
+ json!({
+ "segments_counted_in_constant_time": 0u64,
+ "segments_with_deleted_docs": 0u64,
+ "filters": filters,
+ }),
+ );
}
}
if !children.is_empty() {
@@ -13104,22 +14256,12 @@ fn build_aggregation_profile_full_at(
out
}
-/// Map an ES aggregation request type + its config to the Java aggregator
-/// class name that ES would surface in `profile.shards[].aggregations[].type`.
-///
-/// The picks match ES 8.x defaults: numeric terms use `NumericTermsAggregator`,
-/// keyword terms use the global-ordinals variant unless an `execution_hint`
-/// forces map mode, etc. YAML tests only ever assert on the leaf class name.
-/// Synthesize an ES-style `debug` block for a profiled aggregation node.
-/// Zeros for counters we can't measure; non-zero for collector-flavor
-/// fields that match the declared execution hint. Agg-specific keys
-/// (`surviving_buckets`, `empty_collectors_used` etc.) are emitted
-/// based on `agg_type` so the YAML tests find the keys they assert on.
-fn es_aggregator_debug(agg_type: &str, agg_cfg: &Value) -> Value {
- es_aggregator_debug_full(agg_type, agg_cfg, &[], 0)
-}
-
-fn es_aggregator_debug_full(agg_type: &str, agg_cfg: &Value, sub_agg_names: &[String], doc_count: u64) -> Value {
+fn es_aggregator_debug_full(
+ agg_type: &str,
+ agg_cfg: &Value,
+ sub_agg_names: &[String],
+ doc_count: u64,
+) -> Value {
let execution_hint = agg_cfg
.get("execution_hint")
.and_then(Value::as_str)
@@ -13137,8 +14279,10 @@ fn es_aggregator_debug_full(agg_type: &str, agg_cfg: &Value, sub_agg_names: &[St
.get("field")
.and_then(Value::as_str)
.map(|f| {
- matches!(f, "int" | "long" | "double" | "float" | "number" | "count" | "num")
- || f.contains("int")
+ matches!(
+ f,
+ "int" | "long" | "double" | "float" | "number" | "count" | "num"
+ ) || f.contains("int")
|| f.contains("long")
|| f.contains("double")
|| f.contains("float")
@@ -13167,7 +14311,11 @@ fn es_aggregator_debug_full(agg_type: &str, agg_cfg: &Value, sub_agg_names: &[St
// keyword field, map → "terms"
// keyword field, ord → "terms"
// `total_buckets` and `built_buckets` reflect the unique-key count.
- let strategy = if field_is_numeric { "long_terms" } else { "terms" };
+ let strategy = if field_is_numeric {
+ "long_terms"
+ } else {
+ "terms"
+ };
let mut debug = serde_json::Map::new();
debug.insert("total_buckets".into(), json!(doc_count));
debug.insert("built_buckets".into(), json!(doc_count));
@@ -13177,13 +14325,20 @@ fn es_aggregator_debug_full(agg_type: &str, agg_cfg: &Value, sub_agg_names: &[St
if collect_mode == "breadth_first" && !sub_agg_names.is_empty() {
debug.insert(
"deferred_aggregators".into(),
- Value::Array(sub_agg_names.iter().map(|s| Value::String(s.clone())).collect()),
+ Value::Array(
+ sub_agg_names
+ .iter()
+ .map(|s| Value::String(s.clone()))
+ .collect(),
+ ),
);
}
// For string keyword fields with global ordinals, ES emits a
// `collection_strategy` describing the ord-collection variant
// chosen. The default for "many" buckets is the remap path.
- if !field_is_numeric && (execution_hint == "global_ordinals" || execution_hint.is_empty()) {
+ if !field_is_numeric
+ && (execution_hint == "global_ordinals" || execution_hint.is_empty())
+ {
debug.insert(
"collection_strategy".into(),
Value::String("remap using many bucket ords".into()),
@@ -13229,14 +14384,6 @@ fn es_aggregator_debug_full(agg_type: &str, agg_cfg: &Value, sub_agg_names: &[St
}
}
-fn es_aggregator_class_name(
- agg_type: &str,
- agg_cfg: &Value,
- terms_use_filter_path: bool,
-) -> String {
- es_aggregator_class_name_ctx(agg_type, agg_cfg, terms_use_filter_path, false)
-}
-
fn es_aggregator_class_name_ctx(
agg_type: &str,
agg_cfg: &Value,
@@ -13311,7 +14458,12 @@ fn es_aggregator_class_name_ctx(
.get("filters")
.and_then(Value::as_object)
.map(|o| o.len())
- .or_else(|| agg_cfg.get("filters").and_then(Value::as_array).map(|a| a.len()))
+ .or_else(|| {
+ agg_cfg
+ .get("filters")
+ .and_then(Value::as_array)
+ .map(|a| a.len())
+ })
.unwrap_or(0);
if bucket_count <= 1 {
"FilterByFilterAggregator".to_string()
@@ -13327,7 +14479,7 @@ fn es_aggregator_class_name_ctx(
"diversified_sampler" => "DiversifiedBytesHashSamplerAggregator".to_string(),
"random_sampler" => "RandomSamplerAggregator".to_string(),
"composite" => "CompositeAggregator".to_string(),
- "geo_distance" | "geohash_grid" => "GeoHashGridAggregator".to_string(),
+ "geohash_grid" => "GeoHashGridAggregator".to_string(),
"geotile_grid" => "GeoTileGridAggregator".to_string(),
"geohex_grid" => "GeoHexGridAggregator".to_string(),
"time_series" => "TimeSeriesAggregator".to_string(),
@@ -13410,9 +14562,17 @@ pub async fn nodes_stats(State(state): State<AppState>) -> impl IntoResponse {
let rss_bytes = read_rss_bytes().unwrap_or(0);
let (mem_total, mem_avail) = read_meminfo().unwrap_or((rss_bytes * 4, rss_bytes * 3));
let mem_used = mem_total.saturating_sub(mem_avail);
- let used_pct = if mem_total > 0 { mem_used * 100 / mem_total } else { 0 };
+ let used_pct = if mem_total > 0 {
+ mem_used * 100 / mem_total
+ } else {
+ 0
+ };
let free_pct = 100u64.saturating_sub(used_pct);
- let heap_used_pct = if mem_total > 0 { rss_bytes * 100 / mem_total } else { 0 };
+ let heap_used_pct = if mem_total > 0 {
+ rss_bytes * 100 / mem_total
+ } else {
+ 0
+ };
let cpu_pct = sample_cpu_percent().await;
let node_id = state.engine.node_id.as_str();
@@ -13540,26 +14700,38 @@ fn per_index_dense_vector_stats(state: &AppState, index: &str) -> Value {
let mut fielddata = serde_json::Map::new();
if let Some(entry) = state.engine.index_mappings.get(index) {
let mapping = entry.clone();
- let props = mapping.get("mappings").and_then(|m| m.get("properties"))
+ let props = mapping
+ .get("mappings")
+ .and_then(|m| m.get("properties"))
.or_else(|| mapping.get("properties"));
if let Some(pobj) = props.and_then(Value::as_object) {
for (fname, fspec) in pobj {
- if fspec.get("type").and_then(Value::as_str) != Some("dense_vector") { continue }
+ if fspec.get("type").and_then(Value::as_str) != Some("dense_vector") {
+ continue;
+ }
let dim = fspec.get("dims").and_then(Value::as_u64).unwrap_or(0);
let bytes_per_vec = (dim * 4).max(64);
- let bbq = fspec.get("index_options").and_then(|io| io.get("type"))
- .and_then(Value::as_str).unwrap_or("");
+ let bbq = fspec
+ .get("index_options")
+ .and_then(|io| io.get("type"))
+ .and_then(Value::as_str)
+ .unwrap_or("");
let n = 1u64;
let mut field_obj = serde_json::Map::new();
- let (mut vec_b, mut veb_b, mut veq_b, mut vex_b, mut ceniv_b, mut cliv_b) = (0u64, 0u64, 0u64, 0u64, 0u64, 0u64);
+ let (mut vec_b, mut veb_b, veq_b, vex_b, mut ceniv_b, mut cliv_b) =
+ (0u64, 0u64, 0u64, 0u64, 0u64, 0u64);
match bbq {
"bbq_disk" => {
vec_b = n * bytes_per_vec;
ceniv_b = n * 8;
cliv_b = n * 8;
}
- "bbq_hnsw" | "bbq_flat" => { veb_b = n * (bytes_per_vec / 8 + 1); }
- _ => { vec_b = n * bytes_per_vec; }
+ "bbq_hnsw" | "bbq_flat" => {
+ veb_b = n * (bytes_per_vec / 8 + 1);
+ }
+ _ => {
+ vec_b = n * bytes_per_vec;
+ }
}
let total_b = vec_b + veb_b + veq_b + vex_b + ceniv_b + cliv_b;
field_obj.insert("vec_size_bytes".into(), json!(vec_b));
@@ -13607,27 +14779,38 @@ fn dense_vector_off_heap_stats(state: &AppState) -> Value {
let mut total_size = 0u64;
let mut total_vec = 0u64;
let mut total_veb = 0u64;
- let mut total_veq = 0u64;
- let mut total_vex = 0u64;
+ let total_veq = 0u64;
+ let total_vex = 0u64;
let mut total_cenivf = 0u64;
let mut total_clivf = 0u64;
for entry in state.engine.index_mappings.iter() {
let mapping = entry.value().clone();
- let props = mapping.get("mappings").and_then(|m| m.get("properties"))
+ let props = mapping
+ .get("mappings")
+ .and_then(|m| m.get("properties"))
.or_else(|| mapping.get("properties"));
- let Some(pobj) = props.and_then(Value::as_object) else { continue };
- let docs: u64 = state.engine.get_index(entry.key())
+ let Some(pobj) = props.and_then(Value::as_object) else {
+ continue;
+ };
+ let docs: u64 = state
+ .engine
+ .get_index(entry.key())
.ok()
- .map(|_| 1u64) // Just need >0 for tests; precise per-index doc count requires async
+ .map(|_| 1u64) // Just need >0 for tests; precise per-index doc count requires async
.unwrap_or(0);
for (_fname, fspec) in pobj {
let ftype = fspec.get("type").and_then(Value::as_str);
- if ftype != Some("dense_vector") { continue }
+ if ftype != Some("dense_vector") {
+ continue;
+ }
let dim = fspec.get("dims").and_then(Value::as_u64).unwrap_or(0);
let bytes_per_vec = (dim * 4).max(64);
- let bbq = fspec.get("index_options").and_then(|io| io.get("type"))
- .and_then(Value::as_str).unwrap_or("");
- let n = docs.max(1) as u64;
+ let bbq = fspec
+ .get("index_options")
+ .and_then(|io| io.get("type"))
+ .and_then(Value::as_str)
+ .unwrap_or("");
+ let n = docs.max(1);
match bbq {
"bbq_disk" => {
total_vec += n * bytes_per_vec;
@@ -13664,11 +14847,7 @@ fn read_rss_bytes() -> Option<u64> {
let status = std::fs::read_to_string("/proc/self/status").ok()?;
for line in status.lines() {
if line.starts_with("VmRSS:") {
- let kb: u64 = line
- .split_whitespace()
- .nth(1)?
- .parse()
- .ok()?;
+ let kb: u64 = line.split_whitespace().nth(1)?.parse().ok()?;
return Some(kb * 1024);
}
}
@@ -13756,7 +14935,13 @@ fn explain_query_node(q: &xerj_query::ast::QueryNode) -> (String, Vec<Value>) {
// Single-token match stays as the direct weight label.
let tokens: Vec<&str> = query.split_whitespace().collect();
if tokens.len() <= 1 {
- (format!("weight({}:{} in 0) [PerFieldSimilarity], result of:", field, query), vec![])
+ (
+ format!(
+ "weight({}:{} in 0) [PerFieldSimilarity], result of:",
+ field, query
+ ),
+ vec![],
+ )
} else {
let details: Vec<Value> = tokens.iter().map(|t| json!({
"value": 0.0,
@@ -13766,60 +14951,94 @@ fn explain_query_node(q: &xerj_query::ast::QueryNode) -> (String, Vec<Value>) {
("sum of:".to_string(), details)
}
}
- QueryNode::MatchPhrase { field, query, .. } => (
- format!("weight(phrase {}:{} in doc)", field, query),
- vec![],
- ),
+ QueryNode::MatchPhrase { field, query, .. } => {
+ (format!("weight(phrase {}:{} in doc)", field, query), vec![])
+ }
QueryNode::MultiMatch { query, fields, .. } => (
format!("weight({}:{} in doc)", fields.join("|"), query),
vec![],
),
- QueryNode::Term { field, value, .. } => (
- format!("weight({}:{} in doc)", field, value),
- vec![],
- ),
+ QueryNode::Term { field, value, .. } => {
+ (format!("weight({}:{} in doc)", field, value), vec![])
+ }
QueryNode::Terms { field, values, .. } => (
- format!("weight({}:({}) in doc)", field,
- values.iter().map(|v| v.to_string()).collect::<Vec<_>>().join(" | ")),
+ format!(
+ "weight({}:({}) in doc)",
+ field,
+ values
+ .iter()
+ .map(|v| v.to_string())
+ .collect::<Vec<_>>()
+ .join(" | ")
+ ),
vec![],
),
- QueryNode::Range { field, gte, gt, lte, lt, .. } => {
+ QueryNode::Range {
+ field,
+ gte,
+ gt,
+ lte,
+ lt,
+ ..
+ } => {
let mut parts = Vec::new();
- if let Some(v) = gte { parts.push(format!("{} >= {}", field, v)); }
- if let Some(v) = gt { parts.push(format!("{} > {}", field, v)); }
- if let Some(v) = lte { parts.push(format!("{} <= {}", field, v)); }
- if let Some(v) = lt { parts.push(format!("{} < {}", field, v)); }
+ if let Some(v) = gte {
+ parts.push(format!("{} >= {}", field, v));
+ }
+ if let Some(v) = gt {
+ parts.push(format!("{} > {}", field, v));
+ }
+ if let Some(v) = lte {
+ parts.push(format!("{} <= {}", field, v));
+ }
+ if let Some(v) = lt {
+ parts.push(format!("{} < {}", field, v));
+ }
(format!("range({})", parts.join(", ")), vec![])
}
- QueryNode::Prefix { field, value, .. } => (
- format!("weight({}:{} in doc)", field, value),
- vec![],
- ),
- QueryNode::Exists { field } => (
- format!("ConstantScore(exists(field={}))", field),
- vec![],
- ),
- QueryNode::Bool { must, should, filter, must_not, .. } => {
+ QueryNode::Prefix { field, value, .. } => {
+ (format!("weight({}:{} in doc)", field, value), vec![])
+ }
+ QueryNode::Exists { field } => (format!("ConstantScore(exists(field={}))", field), vec![]),
+ QueryNode::Bool {
+ must,
+ should,
+ filter,
+ must_not,
+ ..
+ } => {
let mut detail_children: Vec<Value> = Vec::new();
for c in must {
let (d, ch) = explain_query_node(c);
- detail_children.push(json!({ "value": 0, "description": format!("must: {}", d), "details": ch }));
+ detail_children.push(
+ json!({ "value": 0, "description": format!("must: {}", d), "details": ch }),
+ );
}
for c in should {
let (d, ch) = explain_query_node(c);
- detail_children.push(json!({ "value": 0, "description": format!("should: {}", d), "details": ch }));
+ detail_children.push(
+ json!({ "value": 0, "description": format!("should: {}", d), "details": ch }),
+ );
}
for c in filter {
let (d, ch) = explain_query_node(c);
- detail_children.push(json!({ "value": 0, "description": format!("filter: {}", d), "details": ch }));
+ detail_children.push(
+ json!({ "value": 0, "description": format!("filter: {}", d), "details": ch }),
+ );
}
for c in must_not {
let (d, ch) = explain_query_node(c);
- detail_children.push(json!({ "value": 0, "description": format!("must_not: {}", d), "details": ch }));
+ detail_children.push(
+ json!({ "value": 0, "description": format!("must_not: {}", d), "details": ch }),
+ );
}
("sum of:".to_string(), detail_children)
}
- QueryNode::Boosting { positive, negative, negative_boost } => {
+ QueryNode::Boosting {
+ positive,
+ negative,
+ negative_boost,
+ } => {
let (pd, pc) = explain_query_node(positive);
let (nd, nc) = explain_query_node(negative);
(
@@ -13830,11 +15049,17 @@ fn explain_query_node(q: &xerj_query::ast::QueryNode) -> (String, Vec<Value>) {
],
)
}
- QueryNode::DisMax { queries, tie_breaker } => {
- let children: Vec<Value> = queries.iter().map(|c| {
- let (d, ch) = explain_query_node(c);
- json!({ "value": 0, "description": d, "details": ch })
- }).collect();
+ QueryNode::DisMax {
+ queries,
+ tie_breaker,
+ } => {
+ let children: Vec<Value> = queries
+ .iter()
+ .map(|c| {
+ let (d, ch) = explain_query_node(c);
+ json!({ "value": 0, "description": d, "details": ch })
+ })
+ .collect();
(format!("max of, tie_breaker={}", tie_breaker), children)
}
QueryNode::Constant { score, query } => {
@@ -14032,7 +15257,10 @@ pub async fn cat_aliases(State(state): State<AppState>) -> impl IntoResponse {
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -14061,7 +15289,10 @@ pub async fn cat_count(
let body = format!("{epoch} {ts} {count}\n");
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -14084,9 +15315,7 @@ pub async fn cat_shards(State(state): State<AppState>) -> impl IntoResponse {
.unwrap_or(0);
lines.push(format!(
"{} 0 p STARTED {} {}b 127.0.0.1 xerj-node-1",
- info.name,
- info.doc_count,
- store_bytes,
+ info.name, info.doc_count, store_bytes,
));
}
let body = if lines.is_empty() {
@@ -14096,7 +15325,10 @@ pub async fn cat_shards(State(state): State<AppState>) -> impl IntoResponse {
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -14182,8 +15414,9 @@ pub async fn put_ingest_pipeline(
other => other.to_string(),
}
}
- let xerj_config = if let Some(processors) = body.get("processors").and_then(Value::as_array) {
- let stages: Vec<Value> = processors.iter().filter_map(|proc| {
+ let xerj_config =
+ if let Some(processors) = body.get("processors").and_then(Value::as_array) {
+ let stages: Vec<Value> = processors.iter().filter_map(|proc| {
let obj = proc.as_object()?;
let (proc_type, proc_config) = obj.iter().next()?;
let xerj_type = map_proc_name(proc_type.as_str());
@@ -14211,13 +15444,13 @@ pub async fn put_ingest_pipeline(
};
Some(json!({"type": xerj_type, "config": adapted_config}))
}).collect();
- json!({
- "description": body.get("description").and_then(Value::as_str).unwrap_or(""),
- "stages": stages
- })
- } else {
- body.clone()
- };
+ json!({
+ "description": body.get("description").and_then(Value::as_str).unwrap_or(""),
+ "stages": stages
+ })
+ } else {
+ body.clone()
+ };
if let Err(e) = state.engine.create_pipeline(&id, xerj_config) {
tracing::warn!(pipeline = %id, error = %e, "pipeline stored but failed to compile");
}
@@ -14294,10 +15527,7 @@ pub async fn simulate_inline_pipeline(
let pipeline_def = match body.pipeline {
Some(p) => p,
None => {
- return build_ingest_parse_error(
- "pipeline",
- "required property is missing",
- );
+ return build_ingest_parse_error("pipeline", "required property is missing");
}
};
@@ -14369,7 +15599,9 @@ pub async fn simulate_inline_pipeline(
let out_docs: Vec<Value> = body
.docs
.iter()
- .map(|d| run_inline_pipeline_on_doc(d, &processors, params.verbose.as_deref() == Some("true")))
+ .map(|d| {
+ run_inline_pipeline_on_doc(d, &processors, params.verbose.as_deref() == Some("true"))
+ })
.collect();
Json(json!({ "docs": out_docs })).into_response()
@@ -14379,10 +15611,21 @@ pub async fn simulate_inline_pipeline(
/// target pipeline's processors, detecting cycles and flagging
/// missing pipelines with a sentinel error processor.
fn expand_pipeline_processors(state: &AppState, processors: &[Value]) -> Vec<Value> {
- fn walk(state: &AppState, procs: &[Value], seen: &mut std::collections::HashSet<String>, out: &mut Vec<Value>) {
+ fn walk(
+ state: &AppState,
+ procs: &[Value],
+ seen: &mut std::collections::HashSet<String>,
+ out: &mut Vec<Value>,
+ ) {
for p in procs {
- let Some(obj) = p.as_object() else { out.push(p.clone()); continue };
- let Some((name, cfg)) = obj.iter().next() else { out.push(p.clone()); continue };
+ let Some(obj) = p.as_object() else {
+ out.push(p.clone());
+ continue;
+ };
+ let Some((name, cfg)) = obj.iter().next() else {
+ out.push(p.clone());
+ continue;
+ };
if name != "pipeline" {
out.push(p.clone());
continue;
@@ -14405,14 +15648,20 @@ fn expand_pipeline_processors(state: &AppState, processors: &[Value]) -> Vec<Val
let inner = if let Some(p) = pipe.get("processors").and_then(Value::as_array) {
p.clone()
} else if let Some(stages) = pipe.get("stages").and_then(Value::as_array) {
- stages.iter().filter_map(|st| {
- let obj = st.as_object()?;
- let ty = obj.get("type").and_then(Value::as_str)?;
- let cfg = obj.get("config").cloned().unwrap_or(Value::Object(serde_json::Map::new()));
- let mut m = serde_json::Map::new();
- m.insert(ty.to_string(), cfg);
- Some(Value::Object(m))
- }).collect()
+ stages
+ .iter()
+ .filter_map(|st| {
+ let obj = st.as_object()?;
+ let ty = obj.get("type").and_then(Value::as_str)?;
+ let cfg = obj
+ .get("config")
+ .cloned()
+ .unwrap_or(Value::Object(serde_json::Map::new()));
+ let mut m = serde_json::Map::new();
+ m.insert(ty.to_string(), cfg);
+ Some(Value::Object(m))
+ })
+ .collect()
} else {
Vec::new()
};
@@ -14423,9 +15672,9 @@ fn expand_pipeline_processors(state: &AppState, processors: &[Value]) -> Vec<Val
// the outermost cycle, not a nested wrapper chain.
let mut inner_expanded: Vec<Value> = Vec::new();
walk(state, &inner, seen, &mut inner_expanded);
- let has_cycle_or_missing = inner_expanded.iter().any(|v| {
- v.get("__xy_missing_pipeline__").is_some()
- });
+ let has_cycle_or_missing = inner_expanded
+ .iter()
+ .any(|v| v.get("__xy_missing_pipeline__").is_some());
if !has_cycle_or_missing {
out.push(json!({"__xy_pipeline_ref__": cfg.clone()}));
out.extend(inner_expanded);
@@ -14469,23 +15718,39 @@ fn find_missing_processor_property(name: &str, cfg: &Value) -> Option<(&'static
let has = |k: &str| cfg.get(k).is_some();
let missing = match name {
"remove" | "lowercase" | "uppercase" | "trim" => {
- if !has("field") { Some("field") } else { None }
+ if !has("field") {
+ Some("field")
+ } else {
+ None
+ }
}
// `set` and `append` require BOTH `field` and `value`. ES
// validates property-by-property, reporting `field` first
// when both are missing.
"set" | "append" => {
- if !has("field") { Some("field") }
- else if !has("value") && !has("copy_from") { Some("value") }
- else { None }
+ if !has("field") {
+ Some("field")
+ } else if !has("value") && !has("copy_from") {
+ Some("value")
+ } else {
+ None
+ }
}
"rename" => {
- if !has("field") { Some("field") }
- else if !has("target_field") { Some("target_field") }
- else { None }
+ if !has("field") {
+ Some("field")
+ } else if !has("target_field") {
+ Some("target_field")
+ } else {
+ None
+ }
}
"script" => {
- if !has("source") && !has("id") { Some("source") } else { None }
+ if !has("source") && !has("id") {
+ Some("source")
+ } else {
+ None
+ }
}
_ => None,
}?;
@@ -14524,14 +15789,21 @@ fn build_ingest_parse_error(property: &str, reason: &str) -> axum::response::Res
/// here — only `set`, `remove`, `rename`, `append`, `lowercase`, `uppercase`,
/// `trim`, and `script` — but the shape of the response is identical so tests
/// that exercise the happy path (set/remove) pass.
-fn run_inline_pipeline_on_doc(
- input: &Value,
- processors: &[Value],
- verbose: bool,
-) -> Value {
- let index = input.get("_index").and_then(Value::as_str).unwrap_or("_index").to_string();
- let id = input.get("_id").and_then(Value::as_str).unwrap_or("_id").to_string();
- let mut source: Value = input.get("_source").cloned().unwrap_or(Value::Object(serde_json::Map::new()));
+fn run_inline_pipeline_on_doc(input: &Value, processors: &[Value], verbose: bool) -> Value {
+ let index = input
+ .get("_index")
+ .and_then(Value::as_str)
+ .unwrap_or("_index")
+ .to_string();
+ let id = input
+ .get("_id")
+ .and_then(Value::as_str)
+ .unwrap_or("_id")
+ .to_string();
+ let mut source: Value = input
+ .get("_source")
+ .cloned()
+ .unwrap_or(Value::Object(serde_json::Map::new()));
let timestamp = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
@@ -14543,6 +15815,9 @@ fn run_inline_pipeline_on_doc(
// `on_failure_tag` lets nested on_failure chains stamp their own
// doc._ingest.on_failure_processor_tag so the YAML tests can
// inspect which processor triggered the recovery.
+ // Renderer needs every piece of per-step context; splitting the
+ // signature into a struct would not improve clarity here.
+ #[allow(clippy::too_many_arguments)]
fn render_entry(
name: &str,
cfg: &Value,
@@ -14556,7 +15831,10 @@ fn run_inline_pipeline_on_doc(
on_failure_tag: Option<&str>,
) -> Value {
let mut entry = serde_json::Map::new();
- entry.insert("processor_type".to_string(), Value::String(name.to_string()));
+ entry.insert(
+ "processor_type".to_string(),
+ Value::String(name.to_string()),
+ );
entry.insert("status".to_string(), Value::String(status.to_string()));
if let Some(tag) = cfg.get("tag").and_then(Value::as_str) {
if !tag.is_empty() {
@@ -14588,18 +15866,30 @@ fn run_inline_pipeline_on_doc(
};
if let Some(msg) = error_msg {
let et = etype_of(msg);
- entry.insert("error".to_string(), json!({
- "root_cause": [{"type": et, "reason": msg}],
- "type": et,
- "reason": msg,
- }));
+ entry.insert(
+ "error".to_string(),
+ json!({
+ "root_cause": [{"type": et, "reason": msg}],
+ "type": et,
+ "reason": msg,
+ }),
+ );
} else {
// Emit doc for success / error_ignored paths.
let mut ingest = serde_json::Map::new();
- ingest.insert("timestamp".to_string(), Value::String(timestamp.to_string()));
- ingest.insert("pipeline".to_string(), Value::String("_simulate_pipeline".into()));
+ ingest.insert(
+ "timestamp".to_string(),
+ Value::String(timestamp.to_string()),
+ );
+ ingest.insert(
+ "pipeline".to_string(),
+ Value::String("_simulate_pipeline".into()),
+ );
if let Some(tag) = on_failure_tag {
- ingest.insert("on_failure_processor_tag".to_string(), Value::String(tag.to_string()));
+ ingest.insert(
+ "on_failure_processor_tag".to_string(),
+ Value::String(tag.to_string()),
+ );
}
entry.insert(
"doc".to_string(),
@@ -14613,13 +15903,16 @@ fn run_inline_pipeline_on_doc(
}
if let Some(msg) = ignored_error {
let et = etype_of(msg);
- entry.insert("ignored_error".to_string(), json!({
- "error": {
- "root_cause": [{"type": et, "reason": msg}],
- "type": et,
- "reason": msg,
- }
- }));
+ entry.insert(
+ "ignored_error".to_string(),
+ json!({
+ "error": {
+ "root_cause": [{"type": et, "reason": msg}],
+ "type": et,
+ "reason": msg,
+ }
+ }),
+ );
}
Value::Object(entry)
}
@@ -14642,11 +15935,23 @@ fn run_inline_pipeline_on_doc(
parent_on_failure_tag: Option<&str>,
) -> Result<(), String> {
if name == "__xy_missing_pipeline__" {
- let reason = cfg.get("reason").and_then(Value::as_str).unwrap_or("").to_string();
+ let reason = cfg
+ .get("reason")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
if verbose {
results.push(render_entry(
- "pipeline", cfg, "error", Some(&reason), None,
- source, index, id, timestamp, parent_on_failure_tag,
+ "pipeline",
+ cfg,
+ "error",
+ Some(&reason),
+ None,
+ source,
+ index,
+ id,
+ timestamp,
+ parent_on_failure_tag,
));
}
return Err(reason);
@@ -14655,8 +15960,16 @@ fn run_inline_pipeline_on_doc(
if cond == Some("false") {
if verbose {
results.push(render_entry(
- name, cfg, "skipped", None, None,
- source, index, id, timestamp, parent_on_failure_tag,
+ name,
+ cfg,
+ "skipped",
+ None,
+ None,
+ source,
+ index,
+ id,
+ timestamp,
+ parent_on_failure_tag,
));
}
return Ok(());
@@ -14665,14 +15978,25 @@ fn run_inline_pipeline_on_doc(
Ok(()) => {
if verbose {
results.push(render_entry(
- name, cfg, "success", None, None,
- source, index, id, timestamp, parent_on_failure_tag,
+ name,
+ cfg,
+ "success",
+ None,
+ None,
+ source,
+ index,
+ id,
+ timestamp,
+ parent_on_failure_tag,
));
}
Ok(())
}
Err(e) => {
- let ignore_failure = cfg.get("ignore_failure").and_then(Value::as_bool).unwrap_or(false);
+ let ignore_failure = cfg
+ .get("ignore_failure")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
let on_failure = cfg.get("on_failure").and_then(Value::as_array).cloned();
if ignore_failure {
// ignore_failure takes precedence over on_failure:
@@ -14681,8 +16005,16 @@ fn run_inline_pipeline_on_doc(
// recovery-chain entries.
if verbose {
results.push(render_entry(
- name, cfg, "error_ignored", None, Some(&e),
- source, index, id, timestamp, parent_on_failure_tag,
+ name,
+ cfg,
+ "error_ignored",
+ None,
+ Some(&e),
+ source,
+ index,
+ id,
+ timestamp,
+ parent_on_failure_tag,
));
}
return Ok(());
@@ -14692,8 +16024,16 @@ fn run_inline_pipeline_on_doc(
// each on_failure step emits its own verbose
// entry (carrying on_failure_processor_tag).
results.push(render_entry(
- name, cfg, "error", Some(&e), None,
- source, index, id, timestamp, parent_on_failure_tag,
+ name,
+ cfg,
+ "error",
+ Some(&e),
+ None,
+ source,
+ index,
+ id,
+ timestamp,
+ parent_on_failure_tag,
));
}
if let Some(of_procs) = on_failure {
@@ -14704,13 +16044,17 @@ fn run_inline_pipeline_on_doc(
for of_spec in &of_procs {
if let Some(obj2) = of_spec.as_object() {
if let Some((of_name, of_cfg)) = obj2.iter().next() {
- if let Err(msg) = run_one(
- of_name, of_cfg, source, verbose, results,
- index, id, timestamp,
+ run_one(
+ of_name,
+ of_cfg,
+ source,
+ verbose,
+ results,
+ index,
+ id,
+ timestamp,
my_tag.as_deref(),
- ) {
- return Err(msg);
- }
+ )?;
}
}
}
@@ -14723,11 +16067,19 @@ fn run_inline_pipeline_on_doc(
}
for proc_spec in processors {
- let Some(obj) = proc_spec.as_object() else { continue };
- let Some((name, cfg)) = obj.iter().next() else { continue };
+ let Some(obj) = proc_spec.as_object() else {
+ continue;
+ };
+ let Some((name, cfg)) = obj.iter().next() else {
+ continue;
+ };
if !verbose && name == "__xy_missing_pipeline__" {
- let reason = cfg.get("reason").and_then(Value::as_str).unwrap_or("").to_string();
+ let reason = cfg
+ .get("reason")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
error = Some(("pipeline".to_string(), reason));
break;
}
@@ -14738,7 +16090,10 @@ fn run_inline_pipeline_on_doc(
if name == "__xy_pipeline_ref__" {
if verbose {
let mut entry = serde_json::Map::new();
- entry.insert("processor_type".to_string(), Value::String("pipeline".into()));
+ entry.insert(
+ "processor_type".to_string(),
+ Value::String("pipeline".into()),
+ );
entry.insert("status".to_string(), Value::String("success".into()));
if let Some(tag) = cfg.get("tag").and_then(Value::as_str) {
if !tag.is_empty() {
@@ -14772,8 +16127,15 @@ fn run_inline_pipeline_on_doc(
if verbose {
if let Err(msg) = run_one(
- name, cfg, &mut source, true, &mut results,
- &index, &id, &timestamp, None,
+ name,
+ cfg,
+ &mut source,
+ true,
+ &mut results,
+ &index,
+ &id,
+ &timestamp,
+ None,
) {
error = Some((name.clone(), msg));
break;
@@ -14789,13 +16151,18 @@ fn run_inline_pipeline_on_doc(
Ok(_) => "success",
Err(e) => {
let on_failure = cfg.get("on_failure").and_then(Value::as_array).cloned();
- let ignore_failure = cfg.get("ignore_failure").and_then(Value::as_bool).unwrap_or(false);
+ let ignore_failure = cfg
+ .get("ignore_failure")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
if let Some(of_procs) = on_failure {
let mut of_err: Option<String> = None;
for of_spec in &of_procs {
if let Some(obj2) = of_spec.as_object() {
if let Some((of_name, of_cfg)) = obj2.iter().next() {
- if let Err(of_e) = apply_processor_with_on_failure(of_name, of_cfg, &mut source) {
+ if let Err(of_e) =
+ apply_processor_with_on_failure(of_name, of_cfg, &mut source)
+ {
of_err = Some(of_e);
break;
}
@@ -14803,10 +16170,17 @@ fn run_inline_pipeline_on_doc(
}
}
match of_err {
- Some(msg) => { error = Some((name.clone(), msg)); "error" }
+ Some(msg) => {
+ error = Some((name.clone(), msg));
+ "error"
+ }
None => {
ignored_error_reason = Some(e.clone());
- if ignore_failure { "error_ignored" } else { "success" }
+ if ignore_failure {
+ "error_ignored"
+ } else {
+ "success"
+ }
}
}
} else if ignore_failure {
@@ -14819,7 +16193,9 @@ fn run_inline_pipeline_on_doc(
}
};
let _ = (status, ignored_error_reason);
- if error.is_some() { break; }
+ if error.is_some() {
+ break;
+ }
}
if verbose {
@@ -14839,7 +16215,10 @@ fn run_inline_pipeline_on_doc(
// config-validation errors and runtime_exception for
// script-level runtime failures. Most of what our
// processors raise falls into the first category.
- let etype = if msg.contains("not present") || msg.contains("required") || msg.contains("missing") {
+ let etype = if msg.contains("not present")
+ || msg.contains("required")
+ || msg.contains("missing")
+ {
"illegal_argument_exception"
} else {
"runtime_exception"
@@ -14870,17 +16249,24 @@ fn apply_processor_with_on_failure(
cfg: &Value,
source: &mut Value,
) -> Result<(), String> {
- let source_obj = source.as_object_mut().ok_or_else(|| "source is not an object".to_string())?;
+ let source_obj = source
+ .as_object_mut()
+ .ok_or_else(|| "source is not an object".to_string())?;
let cloned: Value = Value::Object(source_obj.clone());
let mut wrapped = cloned;
let r = apply_single_processor(name, cfg, &mut wrapped);
// Write wrapped back in case the processor mutated it.
if let (Some(dst), Some(src)) = (source.as_object_mut(), wrapped.as_object()) {
dst.clear();
- for (k, v) in src { dst.insert(k.clone(), v.clone()); }
+ for (k, v) in src {
+ dst.insert(k.clone(), v.clone());
+ }
}
if let Err(e) = r {
- let ignore_failure = cfg.get("ignore_failure").and_then(Value::as_bool).unwrap_or(false);
+ let ignore_failure = cfg
+ .get("ignore_failure")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
if let Some(of_procs) = cfg.get("on_failure").and_then(Value::as_array) {
for of_spec in of_procs {
if let Some(obj2) = of_spec.as_object() {
@@ -14905,14 +16291,18 @@ fn apply_processor_with_on_failure(
/// `{a: {b: {c: "d"}}}` (merging with any existing keys).
fn set_dotted_path(obj: &mut serde_json::Map<String, Value>, field: &str, value: Value) {
let segments: Vec<&str> = field.split('.').collect();
- if segments.is_empty() { return; }
+ if segments.is_empty() {
+ return;
+ }
if segments.len() == 1 {
obj.insert(segments[0].to_string(), value);
return;
}
let mut cur = obj;
for seg in &segments[..segments.len() - 1] {
- let entry = cur.entry(seg.to_string()).or_insert_with(|| Value::Object(serde_json::Map::new()));
+ let entry = cur
+ .entry(seg.to_string())
+ .or_insert_with(|| Value::Object(serde_json::Map::new()));
if !entry.is_object() {
*entry = Value::Object(serde_json::Map::new());
}
@@ -14923,9 +16313,14 @@ fn set_dotted_path(obj: &mut serde_json::Map<String, Value>, field: &str, value:
}
/// Get a mutable reference to the value at the dotted path `field`, if present.
-fn get_dotted_path_mut<'a>(obj: &'a mut serde_json::Map<String, Value>, field: &str) -> Option<&'a mut Value> {
+fn get_dotted_path_mut<'a>(
+ obj: &'a mut serde_json::Map<String, Value>,
+ field: &str,
+) -> Option<&'a mut Value> {
let segments: Vec<&str> = field.split('.').collect();
- if segments.is_empty() { return None; }
+ if segments.is_empty() {
+ return None;
+ }
let mut cur: &mut Value = obj.get_mut(segments[0])?;
for seg in &segments[1..] {
// Single-pass parse: was `if is_ok() { let idx = unwrap(); ... }`,
@@ -14940,11 +16335,7 @@ fn get_dotted_path_mut<'a>(obj: &'a mut serde_json::Map<String, Value>, field: &
Some(cur)
}
-fn apply_single_processor(
- name: &str,
- cfg: &Value,
- source: &mut Value,
-) -> Result<(), String> {
+fn apply_single_processor(name: &str, cfg: &Value, source: &mut Value) -> Result<(), String> {
let source_obj = source
.as_object_mut()
.ok_or_else(|| "source is not an object".to_string())?;
@@ -14968,9 +16359,20 @@ fn apply_single_processor(
Ok(())
}
"rename" => {
- let from = cfg.get("field").and_then(Value::as_str).ok_or_else(|| "rename processor: 'field' required".to_string())?.to_string();
- let to = cfg.get("target_field").and_then(Value::as_str).ok_or_else(|| "rename processor: 'target_field' required".to_string())?.to_string();
- let ignore_missing = cfg.get("ignore_missing").and_then(Value::as_bool).unwrap_or(false);
+ let from = cfg
+ .get("field")
+ .and_then(Value::as_str)
+ .ok_or_else(|| "rename processor: 'field' required".to_string())?
+ .to_string();
+ let to = cfg
+ .get("target_field")
+ .and_then(Value::as_str)
+ .ok_or_else(|| "rename processor: 'target_field' required".to_string())?
+ .to_string();
+ let ignore_missing = cfg
+ .get("ignore_missing")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
if let Some(v) = source_obj.remove(&from) {
source_obj.insert(to, v);
Ok(())
@@ -14981,7 +16383,11 @@ fn apply_single_processor(
}
}
"append" => {
- let field = cfg.get("field").and_then(Value::as_str).ok_or_else(|| "append processor: 'field' required".to_string())?.to_string();
+ let field = cfg
+ .get("field")
+ .and_then(Value::as_str)
+ .ok_or_else(|| "append processor: 'field' required".to_string())?
+ .to_string();
let value = cfg.get("value").cloned().unwrap_or(Value::Null);
let slot = source_obj.entry(field).or_insert(Value::Array(Vec::new()));
match slot {
@@ -14997,14 +16403,31 @@ fn apply_single_processor(
Ok(())
}
"convert" => {
- let field = cfg.get("field").and_then(Value::as_str).ok_or_else(|| "convert processor: 'field' required".to_string())?.to_string();
- let target_field = cfg.get("target_field").and_then(Value::as_str).unwrap_or(&field).to_string();
- let ty = cfg.get("type").and_then(Value::as_str).ok_or_else(|| "convert processor: 'type' required".to_string())?;
+ let field = cfg
+ .get("field")
+ .and_then(Value::as_str)
+ .ok_or_else(|| "convert processor: 'field' required".to_string())?
+ .to_string();
+ let target_field = cfg
+ .get("target_field")
+ .and_then(Value::as_str)
+ .unwrap_or(&field)
+ .to_string();
+ let ty = cfg
+ .get("type")
+ .and_then(Value::as_str)
+ .ok_or_else(|| "convert processor: 'type' required".to_string())?;
let Some(existing) = get_dotted_path_mut(source_obj, &field).map(|v| v.clone()) else {
- return if cfg.get("ignore_missing").and_then(Value::as_bool).unwrap_or(false) {
+ return if cfg
+ .get("ignore_missing")
+ .and_then(Value::as_bool)
+ .unwrap_or(false)
+ {
Ok(())
} else {
- Err(format!("field [{field}] not present as part of path [{field}]"))
+ Err(format!(
+ "field [{field}] not present as part of path [{field}]"
+ ))
};
};
let converted: Value = match ty {
@@ -15014,7 +16437,8 @@ fn apply_single_processor(
Value::String(s) => s.parse::<i64>().ok(),
Value::Bool(b) => Some(if *b { 1 } else { 0 }),
_ => None,
- }.ok_or_else(|| format!("unable to convert [{existing}] to integer"))?;
+ }
+ .ok_or_else(|| format!("unable to convert [{existing}] to integer"))?;
json!(n)
}
"float" | "double" => {
@@ -15023,8 +16447,11 @@ fn apply_single_processor(
Value::String(s) => s.parse::<f64>().ok(),
Value::Bool(b) => Some(if *b { 1.0 } else { 0.0 }),
_ => None,
- }.ok_or_else(|| format!("unable to convert [{existing}] to float"))?;
- serde_json::Number::from_f64(f).map(Value::Number).unwrap_or(Value::Null)
+ }
+ .ok_or_else(|| format!("unable to convert [{existing}] to float"))?;
+ serde_json::Number::from_f64(f)
+ .map(Value::Number)
+ .unwrap_or(Value::Null)
}
"string" => match &existing {
Value::String(s) => Value::String(s.clone()),
@@ -15047,8 +16474,15 @@ fn apply_single_processor(
Ok(())
}
"lowercase" | "uppercase" | "trim" => {
- let field = cfg.get("field").and_then(Value::as_str).ok_or_else(|| format!("{name} processor: 'field' required"))?.to_string();
- let ignore_missing = cfg.get("ignore_missing").and_then(Value::as_bool).unwrap_or(false);
+ let field = cfg
+ .get("field")
+ .and_then(Value::as_str)
+ .ok_or_else(|| format!("{name} processor: 'field' required"))?
+ .to_string();
+ let ignore_missing = cfg
+ .get("ignore_missing")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
match get_dotted_path_mut(source_obj, &field) {
Some(v) => {
if let Value::String(s) = v {
@@ -15068,7 +16502,9 @@ fn apply_single_processor(
// ES raises illegal_argument_exception when the
// referenced field isn't present and
// ignore_missing isn't true.
- Err(format!("field [{field}] not present as part of path [{field}]"))
+ Err(format!(
+ "field [{field}] not present as part of path [{field}]"
+ ))
}
}
}
@@ -15100,7 +16536,8 @@ pub async fn simulate_ingest_pipeline(
// when it was round-tripped verbatim, OR in xerj internal shape
// (`stages: [{type: "set", config: {...}}]`) after compilation.
// Accept either — convert stages back into the ES-processor shape.
- let processors: Vec<Value> = if let Some(p) = stored.get("processors").and_then(Value::as_array) {
+ let processors: Vec<Value> = if let Some(p) = stored.get("processors").and_then(Value::as_array)
+ {
p.clone()
} else if let Some(stages) = stored.get("stages").and_then(Value::as_array) {
stages
@@ -15108,7 +16545,10 @@ pub async fn simulate_ingest_pipeline(
.filter_map(|st| {
let obj = st.as_object()?;
let ty = obj.get("type").and_then(Value::as_str)?;
- let cfg = obj.get("config").cloned().unwrap_or(Value::Object(serde_json::Map::new()));
+ let cfg = obj
+ .get("config")
+ .cloned()
+ .unwrap_or(Value::Object(serde_json::Map::new()));
let mut m = serde_json::Map::new();
m.insert(ty.to_string(), cfg);
Some(Value::Object(m))
@@ -15152,7 +16592,8 @@ pub async fn close_index(
"acknowledged": true,
"shards_acknowledged": true,
"indices": { index: { "closed": true } }
- })).into_response()
+ }))
+ .into_response()
}
Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
@@ -15168,10 +16609,14 @@ pub async fn open_index(
// On reopen, re-evaluate index-sort from the CURRENT mapping —
// dynamic mapping may have since added a `@timestamp` date
// field. ES sorts segments on reopen when the mapping permits.
- let has_ts_declared = state.engine.index_mappings.get(&index)
+ let has_ts_declared = state
+ .engine
+ .index_mappings
+ .get(&index)
.map(|r| {
let m = r.value();
- let t = m.pointer("/mappings/properties/@timestamp/type")
+ let t = m
+ .pointer("/mappings/properties/@timestamp/type")
.or_else(|| m.pointer("/properties/@timestamp/type"))
.and_then(Value::as_str);
matches!(t, Some("date") | Some("date_nanos"))
@@ -15193,7 +16638,9 @@ pub async fn open_index(
..Default::default()
};
match idx.search(&req).await {
- Ok(r) => r.hits.first()
+ Ok(r) => r
+ .hits
+ .first()
.map(|h| h.source.get("@timestamp").is_some())
.unwrap_or(false),
_ => false,
@@ -15201,19 +16648,28 @@ pub async fn open_index(
};
let has_ts = has_ts_declared || has_ts_via_source;
if has_ts {
- let existing = state.engine.index_settings.get(&index).map(|r| r.value().clone()).unwrap_or(Value::Null);
+ let existing = state
+ .engine
+ .index_settings
+ .get(&index)
+ .map(|r| r.value().clone())
+ .unwrap_or(Value::Null);
let mut merged = match existing {
Value::Object(m) => m,
_ => serde_json::Map::new(),
};
merged.insert("__xy_index_sort_field".to_string(), json!("@timestamp"));
merged.insert("__xy_index_sort_order".to_string(), json!("desc"));
- state.engine.index_settings.insert(index.clone(), Value::Object(merged));
+ state
+ .engine
+ .index_settings
+ .insert(index.clone(), Value::Object(merged));
}
Json(json!({
"acknowledged": true,
"shards_acknowledged": true,
- })).into_response()
+ }))
+ .into_response()
}
Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
@@ -15313,7 +16769,11 @@ pub async fn admin_segments_fsck(
// a corruption hit as a hard incident, not a green ping.
StatusCode::INTERNAL_SERVER_ERROR
};
- (status, Json(serde_json::to_value(&report).unwrap_or(json!({})))).into_response()
+ (
+ status,
+ Json(serde_json::to_value(&report).unwrap_or(json!({}))),
+ )
+ .into_response()
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -15336,7 +16796,8 @@ pub async fn flush_index(
match state.engine.flush_index(&index).await {
Ok(()) => Json(json!({
"_shards": { "total": 1, "successful": 1, "failed": 0 }
- })).into_response(),
+ }))
+ .into_response(),
Err(e) => {
// index-not-found wraps as 404; everything else is 500
let msg = e.to_string();
@@ -15345,9 +16806,13 @@ pub async fn flush_index(
} else {
axum::http::StatusCode::INTERNAL_SERVER_ERROR
};
- (status, Json(json!({
- "error": { "type": "flush_failed", "reason": msg }
- }))).into_response()
+ (
+ status,
+ Json(json!({
+ "error": { "type": "flush_failed", "reason": msg }
+ })),
+ )
+ .into_response()
}
}
}
@@ -15356,9 +16821,7 @@ pub async fn flush_index(
// POST /_flush — flush every index in the cluster
// ─────────────────────────────────────────────────────────────────────────────
-pub async fn flush_all(
- State(state): State<AppState>,
-) -> impl IntoResponse {
+pub async fn flush_all(State(state): State<AppState>) -> impl IntoResponse {
let names: Vec<String> = state
.engine
.list_indices()
@@ -15376,7 +16839,8 @@ pub async fn flush_all(
}
Json(json!({
"_shards": { "total": succ + fail, "successful": succ, "failed": fail }
- })).into_response()
+ }))
+ .into_response()
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -15416,7 +16880,8 @@ pub async fn clear_cache(
match state.engine.get_index(&index) {
Ok(_) => Json(json!({
"_shards": { "total": 1, "successful": 1, "failed": 0 }
- })).into_response(),
+ }))
+ .into_response(),
Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
}
@@ -15454,7 +16919,10 @@ pub async fn cat_templates(State(state): State<AppState>) -> impl IntoResponse {
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -15501,7 +16969,8 @@ pub async fn get_data_stream(
Json(json!({ "data_streams": [body] })).into_response()
}
None => {
- let e = xerj_common::XerjError::index_not_found(format!("data_stream [{name}] missing"));
+ let e =
+ xerj_common::XerjError::index_not_found(format!("data_stream [{name}] missing"));
ApiError::new(e).into_response()
}
}
@@ -15642,9 +17111,9 @@ pub async fn get_component_template(
}))
.into_response(),
None => {
- let e = xerj_common::XerjError::index_not_found(
- format!("component template [{name}] missing"),
- );
+ let e = xerj_common::XerjError::index_not_found(format!(
+ "component template [{name}] missing"
+ ));
ApiError::new(e).into_response()
}
}
@@ -15657,9 +17126,8 @@ pub async fn delete_component_template(
if state.engine.component_templates.remove(&name).is_some() {
Json(json!({ "acknowledged": true })).into_response()
} else {
- let e = xerj_common::XerjError::index_not_found(
- format!("component template [{name}] missing"),
- );
+ let e =
+ xerj_common::XerjError::index_not_found(format!("component template [{name}] missing"));
ApiError::new(e).into_response()
}
}
@@ -15772,7 +17240,7 @@ pub async fn cat_allocation(State(state): State<AppState>) -> impl IntoResponse
.unwrap_or((10 * 1024 * 1024 * 1024, 8 * 1024 * 1024 * 1024));
let disk_used_bytes = disk_total.saturating_sub(disk_avail);
let disk_percent = if disk_total > 0 {
- ((disk_total - disk_avail) * 100 / disk_total) as u64
+ (disk_total - disk_avail) * 100 / disk_total
} else {
0
};
@@ -15783,7 +17251,10 @@ pub async fn cat_allocation(State(state): State<AppState>) -> impl IntoResponse
);
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -15799,9 +17270,13 @@ fn read_disk_stats(path: &str) -> Option<(u64, u64)> {
if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 {
return None;
}
- let bsize = if st.f_frsize > 0 { st.f_frsize } else { st.f_bsize } as u64;
- let total = (st.f_blocks as u64).saturating_mul(bsize);
- let avail = (st.f_bavail as u64).saturating_mul(bsize);
+ let bsize = if st.f_frsize > 0 {
+ st.f_frsize
+ } else {
+ st.f_bsize
+ };
+ let total = st.f_blocks.saturating_mul(bsize);
+ let avail = st.f_bavail.saturating_mul(bsize);
if total == 0 {
return None;
}
@@ -15842,9 +17317,7 @@ fn normalize_alias_meta(meta: Value) -> Value {
}
/// `HEAD /_alias` — 200 iff any alias exists on any index.
-pub async fn head_all_aliases_all_indices(
- State(state): State<AppState>,
-) -> impl IntoResponse {
+pub async fn head_all_aliases_all_indices(State(state): State<AppState>) -> impl IntoResponse {
if state.engine.aliases.is_empty() {
StatusCode::NOT_FOUND
} else {
@@ -15866,9 +17339,10 @@ pub async fn head_alias_all_indices(
.map(|i| i.name)
.collect();
let alias_matches = |a: &str| -> bool {
- alias.split(',').map(str::trim).any(|pat| {
- pat == "_all" || pat == "*" || glob_match_simple(pat, a) || pat == a
- })
+ alias
+ .split(',')
+ .map(str::trim)
+ .any(|pat| pat == "_all" || pat == "*" || glob_match_simple(pat, a) || pat == a)
};
let found = indices.iter().any(|idx| {
state
@@ -15917,9 +17391,10 @@ pub async fn head_index_alias(
return StatusCode::NOT_FOUND;
}
let alias_matches = |a: &str| -> bool {
- alias.split(',').map(str::trim).any(|pat| {
- pat == "_all" || pat == "*" || glob_match_simple(pat, a) || pat == a
- })
+ alias
+ .split(',')
+ .map(str::trim)
+ .any(|pat| pat == "_all" || pat == "*" || glob_match_simple(pat, a) || pat == a)
};
let found = targets.iter().any(|name| {
state
@@ -15936,9 +17411,7 @@ pub async fn head_index_alias(
}
/// `GET /_alias` — every alias on every index.
-pub async fn get_all_aliases_all_indices(
- State(state): State<AppState>,
-) -> impl IntoResponse {
+pub async fn get_all_aliases_all_indices(State(state): State<AppState>) -> impl IntoResponse {
let indices: Vec<String> = state
.engine
.list_indices()
@@ -15951,7 +17424,10 @@ pub async fn get_all_aliases_all_indices(
let mut aliases_map = serde_json::Map::new();
for entry in state.engine.aliases.iter() {
if entry.value().contains(idx) {
- aliases_map.insert(entry.key().clone(), alias_meta_for(&state, idx, entry.key()));
+ aliases_map.insert(
+ entry.key().clone(),
+ alias_meta_for(&state, idx, entry.key()),
+ );
}
}
result.insert(idx.clone(), json!({ "aliases": aliases_map }));
@@ -15975,9 +17451,10 @@ pub async fn get_alias_all_indices(
.collect();
let alias_matches = |a: &str| -> bool {
- alias.split(',').map(str::trim).any(|pat| {
- pat == "_all" || pat == "*" || glob_match_simple(pat, a) || pat == a
- })
+ alias
+ .split(',')
+ .map(str::trim)
+ .any(|pat| pat == "_all" || pat == "*" || glob_match_simple(pat, a) || pat == a)
};
let mut result = serde_json::Map::new();
@@ -15985,7 +17462,10 @@ pub async fn get_alias_all_indices(
let mut aliases_map = serde_json::Map::new();
for entry in state.engine.aliases.iter() {
if entry.value().contains(idx) && alias_matches(entry.key()) {
- aliases_map.insert(entry.key().clone(), alias_meta_for(&state, idx, entry.key()));
+ aliases_map.insert(
+ entry.key().clone(),
+ alias_meta_for(&state, idx, entry.key()),
+ );
}
}
if !aliases_map.is_empty() {
@@ -16018,7 +17498,10 @@ pub async fn get_index_aliases(
let mut aliases_map = serde_json::Map::new();
for entry in state.engine.aliases.iter() {
if entry.value().contains(name) {
- aliases_map.insert(entry.key().clone(), alias_meta_for(&state, name, entry.key()));
+ aliases_map.insert(
+ entry.key().clone(),
+ alias_meta_for(&state, name, entry.key()),
+ );
}
}
out.insert(name.clone(), json!({ "aliases": aliases_map }));
@@ -16037,9 +17520,10 @@ pub async fn get_index_alias(
}
let alias_matches = |a: &str| -> bool {
- alias.split(',').map(str::trim).any(|pat| {
- pat == "_all" || pat == "*" || glob_match_simple(pat, a) || pat == a
- })
+ alias
+ .split(',')
+ .map(str::trim)
+ .any(|pat| pat == "_all" || pat == "*" || glob_match_simple(pat, a) || pat == a)
};
let mut out = serde_json::Map::new();
@@ -16047,7 +17531,10 @@ pub async fn get_index_alias(
let mut aliases_map = serde_json::Map::new();
for entry in state.engine.aliases.iter() {
if entry.value().contains(name) && alias_matches(entry.key()) {
- aliases_map.insert(entry.key().clone(), alias_meta_for(&state, name, entry.key()));
+ aliases_map.insert(
+ entry.key().clone(),
+ alias_meta_for(&state, name, entry.key()),
+ );
}
}
if !aliases_map.is_empty() {
@@ -16131,9 +17618,17 @@ pub async fn create_snapshot(
.as_ref()
.and_then(|b| b.get("indices"))
.and_then(Value::as_array)
- .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect());
+ .map(|arr| {
+ arr.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect()
+ });
- match state.engine.create_snapshot(repo_path, &snapshot, indices).await {
+ match state
+ .engine
+ .create_snapshot(repo_path, &snapshot, indices)
+ .await
+ {
Ok(snap_info) => {
let key = format!("{}/{}", repo, snapshot);
state.engine.snapshots.insert(key, snap_info.clone());
@@ -16155,9 +17650,9 @@ pub async fn get_snapshot(
match state.engine.snapshots.get(&key) {
Some(info) => Json(json!({ "snapshots": [info.clone()] })).into_response(),
None => {
- let e = xerj_common::XerjError::index_not_found(
- format!("snapshot [{snapshot}] missing in repository [{repo}]"),
- );
+ let e = xerj_common::XerjError::index_not_found(format!(
+ "snapshot [{snapshot}] missing in repository [{repo}]"
+ ));
ApiError::new(e).into_response()
}
}
@@ -16415,7 +17910,10 @@ pub async fn cat_recovery(State(state): State<AppState>) -> impl IntoResponse {
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -16429,7 +17927,13 @@ pub async fn cat_segments(
let node = state.engine.node_id.as_str();
let mut lines: Vec<String> = Vec::new();
let indices_to_list: Vec<String> = if index == "_all" || index == "*" {
- state.engine.list_indices().await.into_iter().map(|i| i.name).collect()
+ state
+ .engine
+ .list_indices()
+ .await
+ .into_iter()
+ .map(|i| i.name)
+ .collect()
} else {
vec![index.clone()]
};
@@ -16459,7 +17963,10 @@ pub async fn cat_segments(
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -16476,8 +17983,16 @@ pub async fn cat_thread_pool(State(state): State<AppState>) -> impl IntoResponse
// node name so Kibana / cerebro render the panel correctly.
let node = state.engine.node_id.as_str();
let pools = [
- "search", "write", "bulk", "get", "analyze", "management", "flush", "refresh",
- "warmer", "generic",
+ "search",
+ "write",
+ "bulk",
+ "get",
+ "analyze",
+ "management",
+ "flush",
+ "refresh",
+ "warmer",
+ "generic",
];
let body = pools
.iter()
@@ -16487,7 +18002,10 @@ pub async fn cat_thread_pool(State(state): State<AppState>) -> impl IntoResponse
+ "\n";
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -16512,7 +18030,10 @@ pub async fn cat_fielddata(State(state): State<AppState>) -> impl IntoResponse {
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -16532,7 +18053,10 @@ pub async fn cat_pending_tasks(State(state): State<AppState>) -> impl IntoRespon
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -16552,7 +18076,10 @@ pub async fn cat_plugins(State(state): State<AppState>) -> impl IntoResponse {
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -16576,7 +18103,10 @@ pub async fn cat_nodeattrs(State(state): State<AppState>) -> impl IntoResponse {
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -16588,7 +18118,10 @@ pub async fn cat_master(State(state): State<AppState>) -> impl IntoResponse {
let body = format!("{id} 127.0.0.1 127.0.0.1 {id}\n");
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -16720,11 +18253,7 @@ pub async fn node_stats_by_id(
// POST /{index}/_split/{target}
// ─────────────────────────────────────────────────────────────────────────────
-async fn clone_index_to(
- state: &AppState,
- source: &str,
- target: &str,
-) -> Result<(), ApiError> {
+async fn clone_index_to(state: &AppState, source: &str, target: &str) -> Result<(), ApiError> {
// Get source index.
let src_idx = state
.engine
@@ -16749,9 +18278,7 @@ async fn clone_index_to(
"size": 10000,
"from": 0,
}))
- .map_err(|e| {
- ApiError::new(xerj_common::XerjError::invalid_query(e.to_string()))
- })?;
+ .map_err(|e| ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())))?;
let results = src_idx
.search(&search_req)
@@ -16760,9 +18287,7 @@ async fn clone_index_to(
for hit in results.hits {
if !hit.source.is_null() {
- let _ = dest_idx
- .index_document(Some(hit.id), hit.source)
- .await;
+ let _ = dest_idx.index_document(Some(hit.id), hit.source).await;
}
}
Ok(())
@@ -16854,9 +18379,9 @@ pub async fn get_enrich_policy(
}))
.into_response(),
None => {
- let e = xerj_common::XerjError::index_not_found(
- format!("enrich policy [{name}] not found"),
- );
+ let e = xerj_common::XerjError::index_not_found(format!(
+ "enrich policy [{name}] not found"
+ ));
ApiError::new(e).into_response()
}
}
@@ -16869,9 +18394,8 @@ pub async fn delete_enrich_policy(
if state.engine.enrich_policies.remove(&name).is_some() {
Json(json!({ "acknowledged": true })).into_response()
} else {
- let e = xerj_common::XerjError::index_not_found(
- format!("enrich policy [{name}] not found"),
- );
+ let e =
+ xerj_common::XerjError::index_not_found(format!("enrich policy [{name}] not found"));
ApiError::new(e).into_response()
}
}
@@ -16904,9 +18428,10 @@ pub async fn execute_enrich_policy(
// Source indices may be a single string or an array of strings.
let source_indices: Vec<String> = match config.get("indices") {
Some(Value::String(s)) => vec![s.clone()],
- Some(Value::Array(a)) => {
- a.iter().filter_map(|v| v.as_str().map(String::from)).collect()
- }
+ Some(Value::Array(a)) => a
+ .iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect(),
_ => vec![],
};
@@ -16980,7 +18505,11 @@ pub async fn put_watch(
) -> impl IntoResponse {
let created = !state.engine.watches.contains_key(&id);
state.engine.watches.insert(id.clone(), body);
- let status = if created { StatusCode::CREATED } else { StatusCode::OK };
+ let status = if created {
+ StatusCode::CREATED
+ } else {
+ StatusCode::OK
+ };
(
status,
Json(json!({
@@ -16993,10 +18522,7 @@ pub async fn put_watch(
.into_response()
}
-pub async fn get_watch(
- State(state): State<AppState>,
- Path(id): Path<String>,
-) -> impl IntoResponse {
+pub async fn get_watch(State(state): State<AppState>, Path(id): Path<String>) -> impl IntoResponse {
match state.engine.watches.get(&id) {
Some(watch) => Json(json!({
"_id": id,
@@ -17009,9 +18535,7 @@ pub async fn get_watch(
}))
.into_response(),
None => {
- let e = xerj_common::XerjError::index_not_found(
- format!("watch [{id}] not found"),
- );
+ let e = xerj_common::XerjError::index_not_found(format!("watch [{id}] not found"));
ApiError::new(e).into_response()
}
}
@@ -17029,9 +18553,7 @@ pub async fn delete_watch(
}))
.into_response()
} else {
- let e = xerj_common::XerjError::index_not_found(
- format!("watch [{id}] not found"),
- );
+ let e = xerj_common::XerjError::index_not_found(format!("watch [{id}] not found"));
ApiError::new(e).into_response()
}
}
@@ -17107,9 +18629,9 @@ pub async fn search_template(
match serde_json::from_str(&rendered) {
Ok(v) => v,
Err(e) => {
- let err = xerj_common::XerjError::invalid_query(
- format!("template rendered to invalid JSON: {e}")
- );
+ let err = xerj_common::XerjError::invalid_query(format!(
+ "template rendered to invalid JSON: {e}"
+ ));
return ApiError::new(err).into_response();
}
}
@@ -17135,24 +18657,25 @@ pub async fn search_template(
match serde_json::from_str(&tmpl_str) {
Ok(v) => v,
Err(e) => {
- let err = xerj_common::XerjError::invalid_query(
- format!("stored template '{id}' rendered to invalid JSON: {e}")
- );
+ let err = xerj_common::XerjError::invalid_query(format!(
+ "stored template '{id}' rendered to invalid JSON: {e}"
+ ));
return ApiError::new(err).into_response();
}
}
}
None => {
- let err = xerj_common::XerjError::index_not_found(
- format!("search template with id '{id}' not found")
- );
+ let err = xerj_common::XerjError::index_not_found(format!(
+ "search template with id '{id}' not found"
+ ));
return ApiError::new(err).into_response();
}
}
} else {
return ApiError::new(xerj_common::XerjError::invalid_query(
- "search template requires either `source` or `id`".to_string()
- )).into_response();
+ "search template requires either `source` or `id`".to_string(),
+ ))
+ .into_response();
};
// Parse and execute as a normal search.
@@ -17173,18 +18696,26 @@ pub async fn search_template(
let took_ms = started.elapsed().as_millis() as u64;
let total = result.total.value;
let max_score = result.hits.first().map(|h| h.score as f64);
- let hits: Vec<Value> = result.hits.into_iter().map(|h| {
- let source = if h.source.is_null() { None } else { Some(h.source) };
- json!({
- "_index": &index,
- "_id": h.id,
- "_score": h.score,
- "_version": 1,
- "_seq_no": 0,
- "_primary_term": 1,
- "_source": source,
+ let hits: Vec<Value> = result
+ .hits
+ .into_iter()
+ .map(|h| {
+ let source = if h.source.is_null() {
+ None
+ } else {
+ Some(h.source)
+ };
+ json!({
+ "_index": &index,
+ "_id": h.id,
+ "_score": h.score,
+ "_version": 1,
+ "_seq_no": 0,
+ "_primary_term": 1,
+ "_source": source,
+ })
})
- }).collect();
+ .collect();
Json(json!({
"took": took_ms,
"timed_out": false,
@@ -17194,7 +18725,8 @@ pub async fn search_template(
"max_score": max_score,
"hits": hits,
},
- })).into_response()
+ }))
+ .into_response()
}
Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
@@ -17302,7 +18834,9 @@ async fn msearch_template_impl(
}
}
} else {
- responses.push(json!({ "error": { "reason": "template requires source or id" }, "status": 400 }));
+ responses.push(
+ json!({ "error": { "reason": "template requires source or id" }, "status": 400 }),
+ );
continue;
};
@@ -17317,9 +18851,18 @@ async fn msearch_template_impl(
};
let index_names: Vec<String> = if index_name == "*" || index_name == "_all" {
- state.engine.list_indices().await.into_iter().map(|i| i.name).collect()
+ state
+ .engine
+ .list_indices()
+ .await
+ .into_iter()
+ .map(|i| i.name)
+ .collect()
} else {
- index_name.split(',').flat_map(|n| state.engine.resolve_alias(n.trim())).collect()
+ index_name
+ .split(',')
+ .flat_map(|n| state.engine.resolve_alias(n.trim()))
+ .collect()
};
let mut merged_hits: Vec<Value> = Vec::new();
@@ -17330,7 +18873,11 @@ async fn msearch_template_impl(
if let Ok(result) = idx.search(&search_req).await {
total_count += result.total.value;
for h in result.hits {
- let source = if h.source.is_null() { Value::Null } else { h.source };
+ let source = if h.source.is_null() {
+ Value::Null
+ } else {
+ h.source
+ };
merged_hits.push(json!({
"_index": idx_name,
"_id": h.id,
@@ -17357,7 +18904,8 @@ async fn msearch_template_impl(
Json(json!({
"took": started.elapsed().as_millis() as u64,
"responses": responses,
- })).into_response()
+ }))
+ .into_response()
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -17382,23 +18930,26 @@ pub async fn render_template_api(
match state.engine.search_templates.get(&id) {
Some(tmpl) => render_template(&tmpl.to_string(), &params),
None => {
- let err = xerj_common::XerjError::index_not_found(
- format!("search template '{id}' not found")
- );
+ let err = xerj_common::XerjError::index_not_found(format!(
+ "search template '{id}' not found"
+ ));
return ApiError::new(err).into_response();
}
}
} else {
return ApiError::new(xerj_common::XerjError::invalid_query(
- "render template requires `source` or `id`".to_string()
- )).into_response();
+ "render template requires `source` or `id`".to_string(),
+ ))
+ .into_response();
};
- let template_output: Value = serde_json::from_str(&rendered_str).unwrap_or(Value::String(rendered_str.clone()));
+ let template_output: Value =
+ serde_json::from_str(&rendered_str).unwrap_or(Value::String(rendered_str.clone()));
Json(json!({
"template_output": template_output,
- })).into_response()
+ }))
+ .into_response()
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -17429,7 +18980,8 @@ pub async fn get_script(
"_id": id,
"found": true,
"script": { "lang": "mustache", "source": source.clone() },
- })).into_response(),
+ }))
+ .into_response(),
None => {
let err = xerj_common::XerjError::index_not_found(format!("script '{id}' not found"));
ApiError::new(err).into_response()
@@ -17464,6 +19016,7 @@ pub async fn delete_script(
/// - `script.source` is truncated to 4096 bytes.
/// - At most 256 comma-separated double literals are parsed.
/// - Requests exceeding these limits receive `413 Payload Too Large`.
+///
/// `POST /_scripts/painless/_execute` — evaluate a Painless script standalone.
///
/// Honest + bounded: `MovingFunctions.*` reductions run over the real
@@ -17535,8 +19088,12 @@ pub async fn painless_execute(
.take(MAX_LITERAL_COUNT)
.filter_map(|s| s.trim().parse::<f64>().ok())
.collect()
- } else { Vec::new() }
- } else { Vec::new() };
+ } else {
+ Vec::new()
+ }
+ } else {
+ Vec::new()
+ };
let mf: Option<f64> = if source.contains("MovingFunctions.max") {
lits.iter().cloned().reduce(f64::max)
} else if source.contains("MovingFunctions.min") {
@@ -17544,13 +19101,21 @@ pub async fn painless_execute(
} else if source.contains("MovingFunctions.sum") {
Some(lits.iter().sum())
} else if source.contains("MovingFunctions.unweightedAvg") {
- if lits.is_empty() { None } else { Some(lits.iter().sum::<f64>() / lits.len() as f64) }
+ if lits.is_empty() {
+ None
+ } else {
+ Some(lits.iter().sum::<f64>() / lits.len() as f64)
+ }
} else {
None
};
return match mf {
Some(v) => {
- let out = if v.fract() == 0.0 { format!("{:.1}", v) } else { v.to_string() };
+ let out = if v.fract() == 0.0 {
+ format!("{:.1}", v)
+ } else {
+ v.to_string()
+ };
Json(json!({ "result": out })).into_response()
}
None => bad_request(format!("unsupported MovingFunctions script: {source}")),
@@ -17567,7 +19132,11 @@ pub async fn painless_execute(
let result = match painless_to_json(v) {
Value::Number(n) => {
let f = n.as_f64().unwrap_or(0.0);
- if f.fract() == 0.0 { format!("{:.1}", f) } else { f.to_string() }
+ if f.fract() == 0.0 {
+ format!("{:.1}", f)
+ } else {
+ f.to_string()
+ }
}
Value::String(s) => s,
Value::Bool(b) => b.to_string(),
@@ -17602,7 +19171,9 @@ pub struct TermsEnumBody {
pub case_insensitive: bool,
}
-fn default_terms_enum_size() -> usize { 10 }
+fn default_terms_enum_size() -> usize {
+ 10
+}
pub async fn terms_enum(
State(state): State<AppState>,
@@ -17684,7 +19255,8 @@ pub async fn terms_enum(
"terms": terms_vec,
"_shards": crate::responses::EsShards::search_success(),
"complete": true,
- })).into_response()
+ }))
+ .into_response()
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -17899,11 +19471,7 @@ pub async fn security_create_api_key(
.unwrap_or(Value::Null);
// Well-formed, unique-per-call key material. Not re-authenticatable.
let key_id = Uuid::new_v4().to_string();
- let raw_secret = format!(
- "{}{}",
- Uuid::new_v4().simple(),
- Uuid::new_v4().simple()
- );
+ let raw_secret = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple());
let api_key = base64_encode(&raw_secret);
// ES returns base64("id:api_key") as the `encoded` credential.
let encoded = base64_encode(&format!("{key_id}:{api_key}"));
@@ -17925,8 +19493,16 @@ fn base64_encode(input: &str) -> String {
let mut i = 0;
while i < bytes.len() {
let b0 = bytes[i] as u32;
- let b1 = if i + 1 < bytes.len() { bytes[i + 1] as u32 } else { 0 };
- let b2 = if i + 2 < bytes.len() { bytes[i + 2] as u32 } else { 0 };
+ let b1 = if i + 1 < bytes.len() {
+ bytes[i + 1] as u32
+ } else {
+ 0
+ };
+ let b2 = if i + 2 < bytes.len() {
+ bytes[i + 2] as u32
+ } else {
+ 0
+ };
output.push(alphabet[((b0 >> 2) & 0x3F) as usize] as char);
output.push(alphabet[(((b0 & 0x3) << 4) | (b1 >> 4)) as usize] as char);
if i + 1 < bytes.len() {
@@ -18095,16 +19671,23 @@ pub async fn open_pit(
// Use the live index list so indices created without explicit
// mapping/settings bodies still show up.
let all_indices = state.engine.index_name_list();
- let indices: Vec<String> = index.split(',')
+ let indices: Vec<String> = index
+ .split(',')
.flat_map(|n| {
let n = n.trim();
if n.contains('*') {
- state.engine.aliases.iter()
+ state
+ .engine
+ .aliases
+ .iter()
.filter(|e| glob_match(n, e.key()))
.map(|e| e.key().clone())
- .chain(all_indices.iter()
- .filter(|name| glob_match(n, name))
- .cloned())
+ .chain(
+ all_indices
+ .iter()
+ .filter(|name| glob_match(n, name))
+ .cloned(),
+ )
.collect::<Vec<_>>()
} else {
state.engine.resolve_alias(n)
@@ -18118,25 +19701,25 @@ pub async fn open_pit(
// `lookup_seq_no` exposes the ES-form (0-based) via subtract-1.
// The highest visible ES-form seq_no is therefore `current - 2`
// (or 0 when no writes). We store it as the inclusive boundary.
- let mut index_max_seq: std::collections::HashMap<String, u64> = std::collections::HashMap::new();
+ let mut index_max_seq: std::collections::HashMap<String, u64> =
+ std::collections::HashMap::new();
for ix in &indices {
if let Ok(idx) = state.engine.get_index(ix) {
let cur = idx.current_seq_no();
index_max_seq.insert(ix.clone(), cur.saturating_sub(2));
}
}
- state.engine.pits.insert(pit_id.clone(), xerj_engine::engine::PitContext {
- indices,
- index_filter,
- index_max_seq,
- created: now,
- expires_at,
- });
- (
- StatusCode::OK,
- Json(json!({ "id": pit_id })),
- )
- .into_response()
+ state.engine.pits.insert(
+ pit_id.clone(),
+ xerj_engine::engine::PitContext {
+ indices,
+ index_filter,
+ index_max_seq,
+ created: now,
+ expires_at,
+ },
+ );
+ (StatusCode::OK, Json(json!({ "id": pit_id }))).into_response()
}
/// GET /_internal/desired_balance — ES cluster desired-balance snapshot.
@@ -18145,9 +19728,7 @@ pub async fn open_pit(
/// shape. xerj runs single-primary-per-index so `current` has one
/// STARTED entry per shard with zero replicas; an unassigned `desired`
/// slot is reported when the index was created with `replicas >= 1`.
-pub async fn get_desired_balance(
- State(state): State<AppState>,
-) -> impl IntoResponse {
+pub async fn get_desired_balance(State(state): State<AppState>) -> impl IntoResponse {
// Node identity is shared with the `cluster_state` handler: the YAML tests
// read an "arbitrary key" out of cluster.state (the node id) and expect
// desired_balance to surface the same name, so this must track
@@ -18203,27 +19784,32 @@ pub async fn get_desired_balance(
let mut shards = serde_json::Map::new();
for shard_id in 0..num_shards {
let current: Vec<Value> = (0..copies)
- .map(|copy| json!({
- "state": "STARTED",
- "shard_id": shard_id,
- "index": idx_name,
- "node_id": node_id,
- "node_is_desired": true,
- "relocating_node": null,
- "relocating_node_is_desired": null,
- "primary": copy == 0,
- "tier_preference": ["data_content"],
- }))
+ .map(|copy| {
+ json!({
+ "state": "STARTED",
+ "shard_id": shard_id,
+ "index": idx_name,
+ "node_id": node_id,
+ "node_is_desired": true,
+ "relocating_node": null,
+ "relocating_node_is_desired": null,
+ "primary": copy == 0,
+ "tier_preference": ["data_content"],
+ })
+ })
.collect();
- shards.insert(shard_id.to_string(), json!({
- "current": current,
- "desired": {
- "total": copies,
- "unassigned": 0,
- "ignored": 0,
- "node_ids": [node_id],
- },
- }));
+ shards.insert(
+ shard_id.to_string(),
+ json!({
+ "current": current,
+ "desired": {
+ "total": copies,
+ "unassigned": 0,
+ "ignored": 0,
+ "node_ids": [node_id],
+ },
+ }),
+ );
}
routing_table.insert(idx_name, Value::Object(shards));
total_shards += num_shards;
@@ -18285,11 +19871,18 @@ pub async fn close_pit(
State(state): State<AppState>,
body: OptionalJson<Value>,
) -> impl IntoResponse {
- let id = body.as_ref()
+ let id = body
+ .as_ref()
.and_then(|v| v.get("id").and_then(Value::as_str))
.map(String::from);
let freed = match id {
- Some(i) => if state.engine.pits.remove(&i).is_some() { 1 } else { 0 },
+ Some(i) => {
+ if state.engine.pits.remove(&i).is_some() {
+ 1
+ } else {
+ 0
+ }
+ }
None => 0,
};
Json(json!({ "succeeded": freed > 0, "num_freed": freed })).into_response()
@@ -18330,8 +19923,12 @@ fn murmur3_routing_shard(routing: &str, shard_count: u32) -> u32 {
let tail_start = nblocks * 4;
let tail = &bytes[tail_start..];
let mut k: u32 = 0;
- if tail.len() >= 3 { k ^= (tail[2] as u32) << 16; }
- if tail.len() >= 2 { k ^= (tail[1] as u32) << 8; }
+ if tail.len() >= 3 {
+ k ^= (tail[2] as u32) << 16;
+ }
+ if tail.len() >= 2 {
+ k ^= (tail[1] as u32) << 8;
+ }
if !tail.is_empty() {
k ^= tail[0] as u32;
k = k.wrapping_mul(c1);
@@ -18356,7 +19953,9 @@ fn murmur3_routing_shard(routing: &str, shard_count: u32) -> u32 {
/// parent objects along the way.
fn remove_dotted_path(obj: &mut serde_json::Map<String, Value>, path: &str) {
let segs: Vec<&str> = path.split('.').collect();
- if segs.is_empty() { return; }
+ if segs.is_empty() {
+ return;
+ }
if segs.len() == 1 {
obj.remove(segs[0]);
return;
@@ -18369,7 +19968,9 @@ fn remove_dotted_path(obj: &mut serde_json::Map<String, Value>, path: &str) {
let c = unsafe { &mut *cur };
stack.push(cur);
match c.get_mut(*seg) {
- Some(Value::Object(next)) => { cur = next; }
+ Some(Value::Object(next)) => {
+ cur = next;
+ }
_ => return,
}
}
@@ -18379,22 +19980,30 @@ fn remove_dotted_path(obj: &mut serde_json::Map<String, Value>, path: &str) {
for i in (0..segs.len() - 1).rev() {
let parent = unsafe { &mut *stack[i] };
let seg = segs[i];
- let should_remove = parent.get(seg)
+ let should_remove = parent
+ .get(seg)
.and_then(Value::as_object)
.map(|o| o.is_empty())
.unwrap_or(false);
- if should_remove { parent.remove(seg); } else { break; }
+ if should_remove {
+ parent.remove(seg);
+ } else {
+ break;
+ }
}
}
-
/// Convert a primitive JSON value to its keyword-field string form
/// (ES coerces `true` → "true", `1` → "1" for keyword-mapped targets).
fn stringify_for_keyword(v: Value) -> Value {
match v {
Value::String(_) => v,
Value::Number(n) => Value::String(n.to_string()),
- Value::Bool(b) => Value::String(if b { "true".to_string() } else { "false".to_string() }),
+ Value::Bool(b) => Value::String(if b {
+ "true".to_string()
+ } else {
+ "false".to_string()
+ }),
Value::Null => Value::Null,
other => Value::String(other.to_string()),
}
@@ -18410,27 +20019,40 @@ fn strip_index_constraints(q: &mut Value, out: &mut Vec<String>) {
for key in keys {
let remove_this = match key.as_str() {
"match" | "term" => {
- let take_out = obj.get(&key).and_then(Value::as_object).and_then(|vo| {
- vo.get("_index").map(|idx_v| {
- if let Some(query_obj) = idx_v.as_object() {
- query_obj.get("query").or_else(|| query_obj.get("value"))
- .and_then(Value::as_str).map(str::to_string)
- } else {
- idx_v.as_str().map(str::to_string)
- }
+ let take_out = obj
+ .get(&key)
+ .and_then(Value::as_object)
+ .and_then(|vo| {
+ vo.get("_index").map(|idx_v| {
+ if let Some(query_obj) = idx_v.as_object() {
+ query_obj
+ .get("query")
+ .or_else(|| query_obj.get("value"))
+ .and_then(Value::as_str)
+ .map(str::to_string)
+ } else {
+ idx_v.as_str().map(str::to_string)
+ }
+ })
})
- }).flatten();
+ .flatten();
if let Some(pat) = take_out {
out.push(pat);
true
- } else { false }
+ } else {
+ false
+ }
}
"bool" => {
if let Some(b) = obj.get_mut(&key).and_then(Value::as_object_mut) {
for sub_key in ["must", "filter", "should"] {
if let Some(arr) = b.get_mut(sub_key).and_then(Value::as_array_mut) {
- for item in arr.iter_mut() { strip_index_constraints(item, out); }
- arr.retain(|item| !item.as_object().map(|o| o.is_empty()).unwrap_or(false));
+ for item in arr.iter_mut() {
+ strip_index_constraints(item, out);
+ }
+ arr.retain(|item| {
+ !item.as_object().map(|o| o.is_empty()).unwrap_or(false)
+ });
}
}
}
@@ -18453,7 +20075,9 @@ fn strip_index_constraints(q: &mut Value, out: &mut Vec<String>) {
/// `keep_alive=500ms` doesn't produce a zero-duration PIT.
fn parse_keep_alive_to_secs(s: &str) -> Option<u64> {
let s = s.trim();
- if s.is_empty() { return None; }
+ if s.is_empty() {
+ return None;
+ }
if let Some(ms) = s.strip_suffix("ms") {
let n: u64 = ms.parse().ok()?;
return Some((n / 1000).max(1));
@@ -18470,20 +20094,30 @@ fn parse_keep_alive_to_secs(s: &str) -> Option<u64> {
}
fn glob_match(pattern: &str, name: &str) -> bool {
- if pattern == "*" || pattern == "_all" { return true; }
- if !pattern.contains('*') { return pattern == name; }
+ if pattern == "*" || pattern == "_all" {
+ return true;
+ }
+ if !pattern.contains('*') {
+ return pattern == name;
+ }
let parts: Vec<&str> = pattern.split('*').collect();
let mut pos = 0usize;
for (i, part) in parts.iter().enumerate() {
- if part.is_empty() { continue; }
+ if part.is_empty() {
+ continue;
+ }
if i == 0 {
- if !name.starts_with(part) { return false; }
+ if !name.starts_with(part) {
+ return false;
+ }
pos = part.len();
} else if i == parts.len() - 1 {
return name[pos..].ends_with(part);
} else {
match name[pos..].find(part) {
- Some(idx) => { pos += idx + part.len(); }
+ Some(idx) => {
+ pos += idx + part.len();
+ }
None => return false,
}
}
@@ -18548,7 +20182,11 @@ pub async fn eql_search(
.hits
.into_iter()
.map(|h| {
- let source = if h.source.is_null() { Value::Null } else { h.source };
+ let source = if h.source.is_null() {
+ Value::Null
+ } else {
+ h.source
+ };
json!({
"_index": &index,
"_id": h.id,
@@ -18640,10 +20278,7 @@ pub async fn global_field_caps(
fields_map
.into_iter()
.map(|(field_name, type_map)| {
- (
- field_name,
- Value::Object(type_map.into_iter().collect()),
- )
+ (field_name, Value::Object(type_map.into_iter().collect()))
})
.collect(),
);
@@ -18685,7 +20320,11 @@ pub async fn get_mapping_field(
for (field_name, field_def) in &properties {
let matches = field_names.iter().any(|pat| {
- if *pat == "*" { true } else { field_name_matches(field_name, pat) }
+ if *pat == "*" {
+ true
+ } else {
+ field_name_matches(field_name, pat)
+ }
});
if matches {
let es_type = field_def
@@ -18728,7 +20367,13 @@ pub async fn put_index_block(
};
// Validate block name.
- let valid_blocks = ["read_only", "read_only_allow_delete", "write", "metadata", "read"];
+ let valid_blocks = [
+ "read_only",
+ "read_only_allow_delete",
+ "write",
+ "metadata",
+ "read",
+ ];
if !valid_blocks.contains(&block.as_str()) {
let e = xerj_common::XerjError::invalid_query(format!(
"invalid index block: {block}; valid values are: read_only, read_only_allow_delete, write, metadata, read"
@@ -18806,7 +20451,9 @@ pub async fn explain_doc(
must: vec![search_req.query.clone()],
should: vec![],
must_not: vec![],
- filter: vec![QueryNode::Ids { values: vec![id.clone()] }],
+ filter: vec![QueryNode::Ids {
+ values: vec![id.clone()],
+ }],
minimum_should_match: None,
};
ids_req.size = 1;
@@ -18879,11 +20526,7 @@ pub(crate) fn parse_stored_fields(stored_fields_val: &Value) -> (bool, Vec<Strin
.filter_map(|v| v.as_str().map(String::from))
.collect();
let opt_in_source = fields.iter().any(|f| f == "_source");
- let keepers: Vec<String> = fields
- .iter()
- .filter(|f| f != &"_source")
- .cloned()
- .collect();
+ let keepers: Vec<String> = fields.iter().filter(|f| f != &"_source").cloned().collect();
(!opt_in_source, keepers)
}
Value::String(s) => {
@@ -18905,7 +20548,7 @@ pub(crate) fn parse_stored_fields(stored_fields_val: &Value) -> (bool, Vec<Strin
/// Expand a wildcard pattern like `user.*` or `*_name` against a doc's
/// _source, returning the list of matching dotted field paths.
pub(crate) fn expand_field_wildcard(source: &Value, pattern: &str) -> Vec<String> {
- let mut out: Vec<(String, bool)> = Vec::new(); // (path, is_scalar_leaf)
+ let mut out: Vec<(String, bool)> = Vec::new(); // (path, is_scalar_leaf)
collect_field_paths(source, "", &mut out);
out.retain(|(p, _)| wildcard_match(pattern, p));
// ES's fields: [*] returns only LEAF paths (no intermediate object
@@ -18914,17 +20557,16 @@ pub(crate) fn expand_field_wildcard(source: &Value, pattern: &str) -> Vec<String
// happens when subobjects:false indexes split a dotted key. In that
// case both `root` and `root.subfield` are leaves and both should be
// fetchable.
- let is_scalar: std::collections::HashMap<String, bool> = out
- .iter()
- .cloned()
- .collect();
+ let is_scalar: std::collections::HashMap<String, bool> = out.iter().cloned().collect();
out.retain(|(p, this_is_scalar)| {
if *this_is_scalar {
// Scalar leaves are always leaves — keep regardless of
// whether some sibling-like key happens to start with p+"."
return true;
}
- !is_scalar.keys().any(|other| other != p && other.starts_with(&format!("{}.", p)))
+ !is_scalar
+ .keys()
+ .any(|other| other != p && other.starts_with(&format!("{}.", p)))
});
out.into_iter().map(|(p, _)| p).collect()
}
@@ -18932,7 +20574,11 @@ pub(crate) fn expand_field_wildcard(source: &Value, pattern: &str) -> Vec<String
fn collect_field_paths(v: &Value, prefix: &str, out: &mut Vec<(String, bool)>) {
if let Value::Object(map) = v {
for (k, child) in map {
- let path = if prefix.is_empty() { k.clone() } else { format!("{}.{}", prefix, k) };
+ let path = if prefix.is_empty() {
+ k.clone()
+ } else {
+ format!("{}.{}", prefix, k)
+ };
let is_scalar = !matches!(child, Value::Object(_));
out.push((path.clone(), is_scalar));
collect_field_paths(child, &path, out);
@@ -18948,16 +20594,23 @@ fn wildcard_match(pattern: &str, text: &str) -> bool {
let (mut star, mut star_t) = (None::<usize>, 0usize);
while ti < t.len() {
if pi < p.len() && (p[pi] == b'?' || p[pi] == t[ti]) {
- pi += 1; ti += 1;
+ pi += 1;
+ ti += 1;
} else if pi < p.len() && p[pi] == b'*' {
- star = Some(pi); star_t = ti; pi += 1;
+ star = Some(pi);
+ star_t = ti;
+ pi += 1;
} else if let Some(sp) = star {
- pi = sp + 1; star_t += 1; ti = star_t;
+ pi = sp + 1;
+ star_t += 1;
+ ti = star_t;
} else {
return false;
}
}
- while pi < p.len() && p[pi] == b'*' { pi += 1; }
+ while pi < p.len() && p[pi] == b'*' {
+ pi += 1;
+ }
pi == p.len()
}
@@ -18979,17 +20632,23 @@ fn first_range_copy_to_field_in_mapping(
index: &str,
doc: Option<&serde_json::Map<String, Value>>,
) -> Option<String> {
- let Some(doc) = doc else { return None };
+ let doc = doc?;
let mapping = state.engine.index_mappings.get(index)?.clone();
- let props = mapping.get("mappings").and_then(|m| m.get("properties"))
+ let props = mapping
+ .get("mappings")
+ .and_then(|m| m.get("properties"))
.or_else(|| mapping.get("properties"))?
.as_object()?
.clone();
for (fname, spec) in &props {
- if !doc.contains_key(fname) { continue; }
+ if !doc.contains_key(fname) {
+ continue;
+ }
let ftype = spec.get("type").and_then(Value::as_str).unwrap_or("");
let is_range = ftype.ends_with("_range");
- if !is_range { continue; }
+ if !is_range {
+ continue;
+ }
if spec.get("copy_to").is_some() {
return Some(fname.clone());
}
@@ -19025,7 +20684,10 @@ pub(crate) fn reshape_geo_point(val: &Value, want_wkt: bool) -> Value {
Value::String(s) => {
// "POINT (lon lat)" — WKT.
if let Some(rest) = s.trim().strip_prefix("POINT") {
- let inner = rest.trim_start().trim_start_matches('(').trim_end_matches(')');
+ let inner = rest
+ .trim_start()
+ .trim_start_matches('(')
+ .trim_end_matches(')');
let mut it = inner.split_whitespace();
if let (Some(lons), Some(lats)) = (it.next(), it.next()) {
if let (Ok(lon), Ok(lat)) = (lons.parse::<f64>(), lats.parse::<f64>()) {
@@ -19126,7 +20788,9 @@ pub(crate) fn build_docvalue_fields(
// precision (strict_date_optional_time → ms; epoch_millis →
// numeric epoch ms with optional fractional ns).
let formatted: Vec<Value> = if let Some(fmt) = format {
- arr.into_iter().map(|v| reformat_docvalue(&v, fmt)).collect()
+ arr.into_iter()
+ .map(|v| reformat_docvalue(&v, fmt))
+ .collect()
} else {
arr
};
@@ -19147,7 +20811,7 @@ fn reformat_docvalue(v: &Value, fmt: &str) -> Value {
Some(p) => (&s[..p], &s[p..]),
None => (s.as_str(), ""),
};
- let (frac, tz) = if let Some(end) = frac_with_z.find(|c: char| c == 'Z' || c == '+' || c == '-') {
+ let (frac, tz) = if let Some(end) = frac_with_z.find(['Z', '+', '-']) {
(&frac_with_z[..end], &frac_with_z[end..])
} else {
(frac_with_z, "")
@@ -19182,8 +20846,8 @@ fn reformat_docvalue(v: &Value, fmt: &str) -> Value {
|| fmt == "strict_date_optional_time_nanos"
|| fmt == "date_optional_time_nanos"
{
- let frac_padded: String = if frac.starts_with('.') {
- let mut digits = frac[1..].to_string();
+ let frac_padded: String = if let Some(rest) = frac.strip_prefix('.') {
+ let mut digits = rest.to_string();
while digits.len() < 9 {
digits.push('0');
}
@@ -19212,7 +20876,9 @@ fn painless_to_json(v: xerj_engine::painless::PainlessValue) -> Value {
match v {
P::Null => Value::Null,
P::Bool(b) => Value::Bool(b),
- P::Number(n) => serde_json::Number::from_f64(n).map(Value::Number).unwrap_or(Value::Null),
+ P::Number(n) => serde_json::Number::from_f64(n)
+ .map(Value::Number)
+ .unwrap_or(Value::Null),
P::String(s) => Value::String(s),
P::Array(a) => Value::Array(a.into_iter().map(painless_to_json).collect()),
P::Object(o) => Value::Object(o),
@@ -19224,9 +20890,13 @@ fn painless_to_json(v: xerj_engine::painless::PainlessValue) -> Value {
/// fetch the "nested" alternate of a path that ALSO has a dotted key
/// at the source root, so we can union the two.
pub(crate) fn get_field_value_via_walk(source: &Value, segs: &[&str]) -> Option<Value> {
- if segs.is_empty() { return Some(source.clone()); }
+ if segs.is_empty() {
+ return Some(source.clone());
+ }
fn walk(cur: &Value, segs: &[&str]) -> Option<Value> {
- if segs.is_empty() { return Some(cur.clone()); }
+ if segs.is_empty() {
+ return Some(cur.clone());
+ }
match cur {
Value::Object(obj) => {
let next = obj.get(segs[0])?;
@@ -19242,7 +20912,11 @@ pub(crate) fn get_field_value_via_walk(source: &Value, segs: &[&str]) -> Option<
}
}
}
- if out.is_empty() { None } else { Some(Value::Array(out)) }
+ if out.is_empty() {
+ None
+ } else {
+ Some(Value::Array(out))
+ }
}
_ => None,
}
@@ -19286,7 +20960,11 @@ pub(crate) fn get_source_value_by_path(source: &Value, path: &str) -> Option<Val
}
}
}
- if out.is_empty() { None } else { Some(Value::Array(out)) }
+ if out.is_empty() {
+ None
+ } else {
+ Some(Value::Array(out))
+ }
}
_ => None,
}
@@ -19309,22 +20987,41 @@ fn knn_body_to_query_node(knn_val: &Value) -> xerj_query::ast::QueryNode {
let vector: Vec<f32> = knn_val
.get("query_vector")
.and_then(Value::as_array)
- .map(|arr| arr.iter().filter_map(|v| v.as_f64().map(|f| f as f32)).collect())
+ .map(|arr| {
+ arr.iter()
+ .filter_map(|v| v.as_f64().map(|f| f as f32))
+ .collect()
+ })
.unwrap_or_default();
- let num_candidates = knn_val.get("num_candidates").and_then(Value::as_u64).map(|n| n as usize);
+ let num_candidates = knn_val
+ .get("num_candidates")
+ .and_then(Value::as_u64)
+ .map(|n| n as usize);
let k = knn_val
.get("k")
.and_then(Value::as_u64)
.map(|n| n as usize)
.or(num_candidates)
.unwrap_or(10);
- let boost = knn_val.get("boost").and_then(Value::as_f64).map(|f| f as f32);
- xerj_query::ast::QueryNode::Knn { field, vector, k, filter: None, boost }
+ let boost = knn_val
+ .get("boost")
+ .and_then(Value::as_f64)
+ .map(|f| f as f32);
+ xerj_query::ast::QueryNode::Knn {
+ field,
+ vector,
+ k,
+ filter: None,
+ boost,
+ }
}
/// Serialise a `QueryNode` to a `Value` for embedding into a bool query.
fn knn_query_node_to_json(node: &xerj_query::ast::QueryNode) -> Value {
- if let xerj_query::ast::QueryNode::Knn { field, vector, k, .. } = node {
+ if let xerj_query::ast::QueryNode::Knn {
+ field, vector, k, ..
+ } = node
+ {
json!({
"knn": {
"field": field,
@@ -19573,7 +21270,10 @@ pub(crate) fn rewrite_bulk_time_series_ids(
pub(crate) fn mapping_has_ignore_directives(mapping: &Value) -> bool {
match mapping {
Value::Object(m) => {
- if m.get("ignore_malformed").and_then(Value::as_bool).unwrap_or(false) {
+ if m.get("ignore_malformed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false)
+ {
return true;
}
if m.contains_key("ignore_above") {
@@ -19597,7 +21297,9 @@ pub(crate) fn rewrite_bulk_ignore_malformed(
out.push_str(action_line);
out.push('\n');
let action_trimmed = action_line.trim();
- if action_trimmed.is_empty() { continue; }
+ if action_trimmed.is_empty() {
+ continue;
+ }
let action: Value = match serde_json::from_str(action_trimmed) {
Ok(v) => v,
Err(_) => continue,
@@ -19607,8 +21309,12 @@ pub(crate) fn rewrite_bulk_ignore_malformed(
Some((k, v)) => (k.clone(), v.clone()),
None => continue,
};
- if op == "delete" { continue; } // no source body
- let Some(source_line) = lines.next() else { break; };
+ if op == "delete" {
+ continue;
+ } // no source body
+ let Some(source_line) = lines.next() else {
+ break;
+ };
let trimmed = source_line.trim();
if trimmed.is_empty() {
out.push_str(source_line);
@@ -19626,7 +21332,8 @@ pub(crate) fn rewrite_bulk_ignore_malformed(
match serde_json::from_str::<Value>(trimmed) {
Ok(Value::Object(mut outer)) => {
if let Some(Value::Object(inner)) = outer.remove("doc") {
- let rewritten_doc = apply_ignore_malformed(state, &idx, Value::Object(inner));
+ let rewritten_doc =
+ apply_ignore_malformed(state, &idx, Value::Object(inner));
outer.insert("doc".to_string(), rewritten_doc);
serde_json::to_string(&Value::Object(outer))
.unwrap_or_else(|_| source_line.to_string())
@@ -19699,18 +21406,35 @@ fn validate_doc_fields(
) {
let keys: Vec<String> = doc.keys().cloned().collect();
for field in keys {
- if field == "_ignored" { continue; }
- let full_name = if prefix.is_empty() { field.clone() } else { format!("{}.{}", prefix, field) };
- let Some(field_map) = props.get(&field) else { continue };
+ if field == "_ignored" {
+ continue;
+ }
+ let full_name = if prefix.is_empty() {
+ field.clone()
+ } else {
+ format!("{}.{}", prefix, field)
+ };
+ let Some(field_map) = props.get(&field) else {
+ continue;
+ };
let ftype = field_map.get("type").and_then(Value::as_str).unwrap_or("");
- let ignore_malformed = field_map.get("ignore_malformed").and_then(Value::as_bool).unwrap_or(false);
+ let ignore_malformed = field_map
+ .get("ignore_malformed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
let ignore_above = field_map.get("ignore_above").and_then(Value::as_u64);
if ftype == "object" || ftype == "nested" {
if let Some(child_props) = field_map.get("properties").and_then(Value::as_object) {
if let Some(Value::Object(child)) = doc.get_mut(&field) {
let mut child_ignored: Vec<String> = Vec::new();
- validate_doc_fields(&full_name, child, child_props, &mut child_ignored, ignored_values);
+ validate_doc_fields(
+ &full_name,
+ child,
+ child_props,
+ &mut child_ignored,
+ ignored_values,
+ );
ignored.extend(child_ignored);
}
}
@@ -19736,7 +21460,9 @@ fn validate_doc_fields(
continue;
}
- if !ignore_malformed { continue; }
+ if !ignore_malformed {
+ continue;
+ }
// When a date field declares an explicit `format`, validate using
// ONLY that format (or the aliases ES treats interchangeably for
@@ -19760,8 +21486,15 @@ fn validate_doc_fields(
matches!(v, Value::String(s) if s.is_empty())
&& matches!(
ftype,
- "integer" | "long" | "short" | "byte" | "float" | "double"
- | "half_float" | "scaled_float" | "boolean"
+ "integer"
+ | "long"
+ | "short"
+ | "byte"
+ | "float"
+ | "double"
+ | "half_float"
+ | "scaled_float"
+ | "boolean"
)
};
@@ -19856,7 +21589,11 @@ fn filter_keyword_length(val: &Value, max: usize) -> (Value, bool) {
}
}
(
- if out.is_empty() { Value::Null } else { Value::Array(out) },
+ if out.is_empty() {
+ Value::Null
+ } else {
+ Value::Array(out)
+ },
any_dropped,
)
}
@@ -19877,10 +21614,7 @@ fn is_date_value_valid_with_format(v: &Value, fmt: &str) -> bool {
|| matches!(v, Value::String(s) if s.parse::<i64>().is_ok());
}
let Some(s) = v.as_str() else {
- return match v {
- Value::Number(_) => true,
- _ => false,
- };
+ return matches!(v, Value::Number(_));
};
// Split combined format ("||") — ES allows multiple fallback formats.
for single_fmt in fmt.split("||") {
@@ -19918,26 +21652,52 @@ fn es_date_format_to_strftime(es_fmt: &str) -> String {
while i < bytes.len() {
// Longest-match substitution for tokens we care about.
let rest = &es_fmt[i..];
- if rest.starts_with("yyyy") { out.push_str("%Y"); i += 4; }
- else if rest.starts_with("yy") { out.push_str("%y"); i += 2; }
- else if rest.starts_with("MM") { out.push_str("%m"); i += 2; }
- else if rest.starts_with("dd") { out.push_str("%d"); i += 2; }
- else if rest.starts_with("HH") { out.push_str("%H"); i += 2; }
- else if rest.starts_with("mm") { out.push_str("%M"); i += 2; }
- else if rest.starts_with("ss") { out.push_str("%S"); i += 2; }
- else if rest.starts_with("SSSSSS") { out.push_str("%6f"); i += 6; }
- else if rest.starts_with("SSSSSSSSS") { out.push_str("%9f"); i += 9; }
- else if rest.starts_with("SSS") { out.push_str("%3f"); i += 3; }
- else if rest.starts_with("Z") { out.push_str("%z"); i += 1; }
- else if rest.starts_with("XXX") { out.push_str("%:z"); i += 3; }
- else if rest.starts_with("'") {
+ if rest.starts_with("yyyy") {
+ out.push_str("%Y");
+ i += 4;
+ } else if rest.starts_with("yy") {
+ out.push_str("%y");
+ i += 2;
+ } else if rest.starts_with("MM") {
+ out.push_str("%m");
+ i += 2;
+ } else if rest.starts_with("dd") {
+ out.push_str("%d");
+ i += 2;
+ } else if rest.starts_with("HH") {
+ out.push_str("%H");
+ i += 2;
+ } else if rest.starts_with("mm") {
+ out.push_str("%M");
+ i += 2;
+ } else if rest.starts_with("ss") {
+ out.push_str("%S");
+ i += 2;
+ } else if rest.starts_with("SSSSSS") {
+ out.push_str("%6f");
+ i += 6;
+ } else if rest.starts_with("SSSSSSSSS") {
+ out.push_str("%9f");
+ i += 9;
+ } else if rest.starts_with("SSS") {
+ out.push_str("%3f");
+ i += 3;
+ } else if rest.starts_with("Z") {
+ out.push_str("%z");
+ i += 1;
+ } else if rest.starts_with("XXX") {
+ out.push_str("%:z");
+ i += 3;
+ } else if rest.starts_with("'") {
// Literal quoted text until next quote.
i += 1;
while i < bytes.len() && bytes[i] != b'\'' {
out.push(bytes[i] as char);
i += 1;
}
- if i < bytes.len() { i += 1; }
+ if i < bytes.len() {
+ i += 1;
+ }
} else {
out.push(bytes[i] as char);
i += 1;
@@ -19966,7 +21726,10 @@ fn is_field_value_valid(ftype: &str, v: &Value) -> bool {
},
"integer" | "long" | "short" | "byte" => match v {
Value::Number(n) => n.is_i64() || n.as_f64().map(|f| f.fract() == 0.0).unwrap_or(false),
- Value::String(s) => s.parse::<i64>().is_ok() || s.parse::<f64>().map(|f| f.fract() == 0.0).unwrap_or(false),
+ Value::String(s) => {
+ s.parse::<i64>().is_ok()
+ || s.parse::<f64>().map(|f| f.fract() == 0.0).unwrap_or(false)
+ }
_ => false,
},
"float" | "double" | "half_float" | "scaled_float" => match v {
@@ -19974,11 +21737,13 @@ fn is_field_value_valid(ftype: &str, v: &Value) -> bool {
Value::String(s) => s.parse::<f64>().is_ok(),
_ => false,
},
- "boolean" => matches!(v, Value::Bool(_))
- || match v {
- Value::String(s) => matches!(s.as_str(), "true" | "false"),
- _ => false,
- },
+ "boolean" => {
+ matches!(v, Value::Bool(_))
+ || match v {
+ Value::String(s) => matches!(s.as_str(), "true" | "false"),
+ _ => false,
+ }
+ }
"geo_point" => {
// Parse a number OR number-shaped string ("20.12") to f64.
let parse_num = |x: &Value| -> Option<f64> {
@@ -19995,7 +21760,9 @@ fn is_field_value_valid(ftype: &str, v: &Value) -> bool {
let valid_lon = |lon: f64| lon.is_finite();
// "lat,lon" string form (latitude first), or a WKT POINT literal.
let is_latlon_string = |s: &str| -> bool {
- if s.starts_with("POINT") { return true; }
+ if s.starts_with("POINT") {
+ return true;
+ }
if let Some((a, b)) = s.split_once(',') {
if let (Ok(lat), Ok(lon)) = (a.trim().parse::<f64>(), b.trim().parse::<f64>()) {
return valid_lat(lat) && valid_lon(lon);
@@ -20005,19 +21772,19 @@ fn is_field_value_valid(ftype: &str, v: &Value) -> bool {
};
match v {
Value::Object(o) => {
- match (parse_num(o.get("lat").unwrap_or(&Value::Null)),
- parse_num(o.get("lon").unwrap_or(&Value::Null))) {
+ match (
+ parse_num(o.get("lat").unwrap_or(&Value::Null)),
+ parse_num(o.get("lon").unwrap_or(&Value::Null)),
+ ) {
(Some(lat), Some(lon)) => valid_lat(lat) && valid_lon(lon),
_ => false,
}
}
// GeoJSON / ES array form is [lon, lat].
- Value::Array(a) if a.len() == 2 => {
- match (parse_num(&a[0]), parse_num(&a[1])) {
- (Some(lon), Some(lat)) => valid_lon(lon) && valid_lat(lat),
- _ => false,
- }
- }
+ Value::Array(a) if a.len() == 2 => match (parse_num(&a[0]), parse_num(&a[1])) {
+ (Some(lon), Some(lat)) => valid_lon(lon) && valid_lat(lat),
+ _ => false,
+ },
// A single-element array is NOT a valid geo_point in ES — the
// array form must be exactly [lon, lat]. So `["45.33, 8.20"]`
// is malformed (whereas the bare string "45.33, 8.20" is fine).
@@ -20060,10 +21827,7 @@ pub(crate) fn build_inner_hits(
.and_then(Value::as_u64)
.map(|n| n as usize)
.unwrap_or(0);
- let source_enabled = match opts.get("_source") {
- Some(Value::Bool(false)) => false,
- _ => true,
- };
+ let source_enabled = !matches!(opts.get("_source"), Some(Value::Bool(false)));
// Field expressions: ES accepts `[ "a.b", "c" ]` OR the newer
// `[ { field: "a.b" }, ... ]` shape.
let field_exprs: Vec<String> = opts
@@ -20086,8 +21850,7 @@ pub(crate) fn build_inner_hits(
// Resolve the nested path (may be dotted) and emit one inner
// hit per element of the nested array.
- let nested_raw = get_source_value_by_path(source, path)
- .unwrap_or(Value::Null);
+ let nested_raw = get_source_value_by_path(source, path).unwrap_or(Value::Null);
let elements: Vec<Value> = match nested_raw {
Value::Array(arr) => arr,
Value::Object(_) => vec![nested_raw],
@@ -20107,7 +21870,10 @@ pub(crate) fn build_inner_hits(
let mut hit = serde_json::Map::new();
hit.insert("_index".to_string(), Value::String(index.to_string()));
hit.insert("_id".to_string(), Value::String(doc_id.to_string()));
- hit.insert("_nested".to_string(), json!({ "field": path, "offset": offset }));
+ hit.insert(
+ "_nested".to_string(),
+ json!({ "field": path, "offset": offset }),
+ );
hit.insert("_score".to_string(), json!(1.0));
if source_enabled {
hit.insert("_source".to_string(), elem.clone());
@@ -20210,12 +21976,18 @@ pub async fn async_search_submit(
// Build a normal search response body to embed.
let max_score = result.hits.first().map(|h| h.score as f64);
- let hits: Vec<Value> = result.hits.iter().map(|h| json!({
- "_index": index,
- "_id": h.id,
- "_score": h.score,
- "_source": h.source,
- })).collect();
+ let hits: Vec<Value> = result
+ .hits
+ .iter()
+ .map(|h| {
+ json!({
+ "_index": index,
+ "_id": h.id,
+ "_score": h.score,
+ "_source": h.source,
+ })
+ })
+ .collect();
let mut search_response = json!({
"took": took_ms,
"timed_out": result.timed_out,
@@ -20247,7 +22019,10 @@ pub async fn async_search_submit(
"response": search_response
});
- state.engine.async_searches.insert(async_id.clone(), stored.clone());
+ state
+ .engine
+ .async_searches
+ .insert(async_id.clone(), stored.clone());
Json(stored).into_response()
}
@@ -20320,9 +22095,7 @@ pub async fn sql_query(
Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),
};
- let limit = body.fetch_size
- .or(parsed.limit)
- .unwrap_or(10);
+ let limit = body.fetch_size.or(parsed.limit).unwrap_or(10);
let mut req = xerj_query::ast::SearchRequest {
query: parsed.query,
@@ -20344,31 +22117,46 @@ pub async fn sql_query(
// Build column metadata from first hit or requested fields.
let columns: Vec<Value> = if parsed.fields.contains(&"*".to_string()) {
// Infer columns from first hit's keys.
- result.hits.first()
+ result
+ .hits
+ .first()
.and_then(|h| h.source.as_object())
- .map(|obj| obj.keys().map(|k| json!({"name": k, "type": "text"})).collect())
+ .map(|obj| {
+ obj.keys()
+ .map(|k| json!({"name": k, "type": "text"}))
+ .collect()
+ })
.unwrap_or_default()
} else {
- parsed.fields.iter()
+ parsed
+ .fields
+ .iter()
.map(|f| json!({"name": f, "type": "text"}))
.collect()
};
- let field_names: Vec<String> = columns.iter()
+ let field_names: Vec<String> = columns
+ .iter()
.filter_map(|c| c.get("name").and_then(Value::as_str).map(String::from))
.collect();
- let rows: Vec<Value> = result.hits.iter().map(|h| {
- let row: Vec<Value> = field_names.iter()
- .map(|f| h.source.get(f).cloned().unwrap_or(Value::Null))
- .collect();
- Value::Array(row)
- }).collect();
+ let rows: Vec<Value> = result
+ .hits
+ .iter()
+ .map(|h| {
+ let row: Vec<Value> = field_names
+ .iter()
+ .map(|f| h.source.get(f).cloned().unwrap_or(Value::Null))
+ .collect();
+ Value::Array(row)
+ })
+ .collect();
Json(json!({
"columns": columns,
"rows": rows
- })).into_response()
+ }))
+ .into_response()
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -20409,7 +22197,8 @@ pub async fn rank_eval(
let mut all_recall: Vec<f64> = Vec::new();
// Default metric: precision at 10.
- let k = body.metric
+ let k = body
+ .metric
.as_ref()
.and_then(|m| m.get("precision").or_else(|| m.get("recall")))
.and_then(|m| m.get("k"))
@@ -20417,19 +22206,22 @@ pub async fn rank_eval(
.unwrap_or(10) as usize;
for req_spec in &body.requests {
- let query_val = req_spec.request.get("query")
+ let query_val = req_spec
+ .request
+ .get("query")
.cloned()
.unwrap_or(json!({"match_all": {}}));
- let size = req_spec.request.get("size")
+ let size = req_spec
+ .request
+ .get("size")
.and_then(Value::as_u64)
.unwrap_or(k as u64) as usize;
- let search_req = match xerj_query::parse_request(
- &json!({"query": query_val, "size": size.max(k)})
- ) {
- Ok(r) => r,
- Err(_) => continue,
- };
+ let search_req =
+ match xerj_query::parse_request(&json!({"query": query_val, "size": size.max(k)})) {
+ Ok(r) => r,
+ Err(_) => continue,
+ };
let idx = match state.engine.get_index(&index) {
Ok(i) => i,
@@ -20442,17 +22234,17 @@ pub async fn rank_eval(
};
// Build set of relevant doc ids (rating >= 1).
- let relevant_ids: std::collections::HashSet<String> = req_spec.ratings.iter()
+ let relevant_ids: std::collections::HashSet<String> = req_spec
+ .ratings
+ .iter()
.filter(|r| r.rating >= 1)
.map(|r| r.id.clone())
.collect();
- let retrieved_ids: Vec<String> = result.hits.iter()
- .take(k)
- .map(|h| h.id.clone())
- .collect();
+ let retrieved_ids: Vec<String> = result.hits.iter().take(k).map(|h| h.id.clone()).collect();
- let relevant_retrieved: usize = retrieved_ids.iter()
+ let relevant_retrieved: usize = retrieved_ids
+ .iter()
.filter(|id| relevant_ids.contains(*id))
.count();
@@ -20495,7 +22287,8 @@ pub async fn rank_eval(
"metric_score": mean_precision,
"details": details,
"failures": {}
- })).into_response()
+ }))
+ .into_response()
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -20773,7 +22566,11 @@ pub async fn put_ml_anomaly_detector(
Value::Array(a) => a.first().and_then(Value::as_str).unwrap_or("").to_string(),
_ => String::new(),
})
- .or_else(|| body.get("index").and_then(Value::as_str).map(str::to_string))
+ .or_else(|| {
+ body.get("index")
+ .and_then(Value::as_str)
+ .map(str::to_string)
+ })
.unwrap_or_default();
if source_index.is_empty() {
return ml_error(
@@ -20811,8 +22608,13 @@ pub async fn put_ml_anomaly_detector(
.and_then(|a| a.first())
{
(
- det.get("function").and_then(Value::as_str).unwrap_or("count").to_string(),
- det.get("field_name").and_then(Value::as_str).map(str::to_string),
+ det.get("function")
+ .and_then(Value::as_str)
+ .unwrap_or("count")
+ .to_string(),
+ det.get("field_name")
+ .and_then(Value::as_str)
+ .map(str::to_string),
)
} else {
("count".to_string(), None)
@@ -20976,8 +22778,8 @@ pub async fn score_ml_anomaly_detector(
.filter(|v| *v > 0.0)
.unwrap_or(detector.anomaly_threshold);
- let (metric_agg, needs_field) = ml_metric_agg(&detector.function)
- .unwrap_or(("value_count", false));
+ let (metric_agg, needs_field) =
+ ml_metric_agg(&detector.function).unwrap_or(("value_count", false));
// Build the date_histogram (+ optional metric sub-agg) request.
let mut hist = json!({
@@ -21071,7 +22873,11 @@ pub async fn score_ml_anomaly_detector(
let (mean, std) = ml_mean_std(&normal);
let z = if std < 1e-9 {
- if (actual - mean).abs() < 1e-9 { 0.0 } else { (actual - mean).signum() * 1e6 }
+ if (actual - mean).abs() < 1e-9 {
+ 0.0
+ } else {
+ (actual - mean).signum() * 1e6
+ }
} else {
(actual - mean) / std
};
@@ -21104,12 +22910,22 @@ pub async fn score_ml_anomaly_detector(
// Anomalies ranked by score, most-severe first.
let mut anomalies: Vec<Value> = records
.iter()
- .filter(|r| r.get("is_anomaly").and_then(Value::as_bool).unwrap_or(false))
+ .filter(|r| {
+ r.get("is_anomaly")
+ .and_then(Value::as_bool)
+ .unwrap_or(false)
+ })
.cloned()
.collect();
anomalies.sort_by(|a, b| {
- let sa = a.get("anomaly_score").and_then(Value::as_f64).unwrap_or(0.0);
- let sb = b.get("anomaly_score").and_then(Value::as_f64).unwrap_or(0.0);
+ let sa = a
+ .get("anomaly_score")
+ .and_then(Value::as_f64)
+ .unwrap_or(0.0);
+ let sb = b
+ .get("anomaly_score")
+ .and_then(Value::as_f64)
+ .unwrap_or(0.0);
sb.partial_cmp(&sa).unwrap_or(std::cmp::Ordering::Equal)
});
@@ -21155,7 +22971,10 @@ pub async fn cat_ml_anomaly_detectors(State(state): State<AppState>) -> impl Int
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -21164,7 +22983,10 @@ pub async fn cat_ml_anomaly_detectors(State(state): State<AppState>) -> impl Int
pub async fn cat_ml_datafeeds() -> impl IntoResponse {
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
"",
)
}
@@ -21172,7 +22994,10 @@ pub async fn cat_ml_datafeeds() -> impl IntoResponse {
pub async fn cat_ml_trained_models() -> impl IntoResponse {
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
"",
)
}
@@ -21481,18 +23306,30 @@ pub async fn get_rollup_data(
let mut jobs: Vec<Value> = Vec::new();
for e in state.engine.rollup_jobs.iter() {
let cfg = e.value();
- let rollup_index = cfg.get("rollup_index").and_then(|v| v.as_str()).unwrap_or("");
+ let rollup_index = cfg
+ .get("rollup_index")
+ .and_then(|v| v.as_str())
+ .unwrap_or("");
if rollup_index != index {
continue;
}
- let index_pattern = cfg.get("index_pattern").and_then(|v| v.as_str()).unwrap_or("");
+ let index_pattern = cfg
+ .get("index_pattern")
+ .and_then(|v| v.as_str())
+ .unwrap_or("");
let mut fields = serde_json::Map::new();
if let Some(groups) = cfg.get("groups").and_then(|g| g.as_object()) {
if let Some(dh) = groups.get("date_histogram").and_then(|d| d.as_object()) {
if let Some(field) = dh.get("field").and_then(|f| f.as_str()) {
let mut agg = serde_json::Map::new();
agg.insert("agg".to_string(), json!("date_histogram"));
- for k in ["fixed_interval", "calendar_interval", "interval", "delay", "time_zone"] {
+ for k in [
+ "fixed_interval",
+ "calendar_interval",
+ "interval",
+ "delay",
+ "time_zone",
+ ] {
if let Some(v) = dh.get(k) {
agg.insert(k.to_string(), v.clone());
}
@@ -21503,7 +23340,11 @@ pub async fn get_rollup_data(
fields.insert(field.to_string(), json!([Value::Object(agg)]));
}
}
- if let Some(terms) = groups.get("terms").and_then(|t| t.get("fields")).and_then(|f| f.as_array()) {
+ if let Some(terms) = groups
+ .get("terms")
+ .and_then(|t| t.get("fields"))
+ .and_then(|f| f.as_array())
+ {
for f in terms {
if let Some(field) = f.as_str() {
fields.entry(field.to_string()).or_insert_with(|| json!([]));
@@ -21519,7 +23360,8 @@ pub async fn get_rollup_data(
for f in hfields {
if let Some(field) = f.as_str() {
fields.entry(field.to_string()).or_insert_with(|| json!([]));
- if let Some(arr) = fields.get_mut(field).and_then(|v| v.as_array_mut()) {
+ if let Some(arr) = fields.get_mut(field).and_then(|v| v.as_array_mut())
+ {
arr.push(json!({ "agg": "histogram", "interval": interval }));
}
}
@@ -21587,7 +23429,10 @@ fn agg_key_doc_id(prefix: &str, key: &Value, order: &[String]) -> String {
/// Flatten a composite bucket's sub-aggregation results onto `doc`. Single-value
/// metrics (`{"value": x}`) flatten to the scalar under `name`; multi-value
/// metrics (stats/percentiles) are kept as the nested object.
-fn flatten_bucket_metrics(doc: &mut serde_json::Map<String, Value>, bucket: &serde_json::Map<String, Value>) {
+fn flatten_bucket_metrics(
+ doc: &mut serde_json::Map<String, Value>,
+ bucket: &serde_json::Map<String, Value>,
+) {
for (k, v) in bucket {
if k == "key" || k == "doc_count" {
continue;
@@ -21605,9 +23450,12 @@ async fn run_pivot_transform(state: &AppState, config: &Value) -> Result<usize,
let source_index = config
.pointer("/source/index")
.and_then(|v| {
- v.as_str()
- .map(|s| s.to_string())
- .or_else(|| v.as_array().and_then(|a| a.first()).and_then(|x| x.as_str()).map(|s| s.to_string()))
+ v.as_str().map(|s| s.to_string()).or_else(|| {
+ v.as_array()
+ .and_then(|a| a.first())
+ .and_then(|x| x.as_str())
+ .map(|s| s.to_string())
+ })
})
.ok_or_else(|| "transform has no source.index".to_string())?;
let dest_index = config
@@ -21629,10 +23477,19 @@ async fn run_pivot_transform(state: &AppState, config: &Value) -> Result<usize,
// Each group_by entry is already a composite source spec: {name: {terms|date_histogram|histogram: {...}}}.
let order: Vec<String> = group_by.keys().cloned().collect();
- let sources: Vec<Value> = order.iter().map(|name| json!({ name.clone(): group_by[name] })).collect();
+ let sources: Vec<Value> = order
+ .iter()
+ .map(|name| json!({ name.clone(): group_by[name] }))
+ .collect();
- let src_idx = state.engine.get_index(&source_index).map_err(|e| e.to_string())?;
- let dest_idx = state.engine.get_or_create_index(&dest_index).map_err(|e| e.to_string())?;
+ let src_idx = state
+ .engine
+ .get_index(&source_index)
+ .map_err(|e| e.to_string())?;
+ let dest_idx = state
+ .engine
+ .get_or_create_index(&dest_index)
+ .map_err(|e| e.to_string())?;
let mut after: Option<Value> = None;
let mut written = 0usize;
@@ -21658,7 +23515,11 @@ async fn run_pivot_transform(state: &AppState, config: &Value) -> Result<usize,
Some(p) => p,
None => break,
};
- let buckets = pivot.get("buckets").and_then(|b| b.as_array()).cloned().unwrap_or_default();
+ let buckets = pivot
+ .get("buckets")
+ .and_then(|b| b.as_array())
+ .cloned()
+ .unwrap_or_default();
if buckets.is_empty() {
break;
}
@@ -21673,7 +23534,10 @@ async fn run_pivot_transform(state: &AppState, config: &Value) -> Result<usize,
if let Some(bo) = b.as_object() {
flatten_bucket_metrics(&mut doc, bo);
}
- doc.insert("doc_count".to_string(), b.get("doc_count").cloned().unwrap_or(json!(0)));
+ doc.insert(
+ "doc_count".to_string(),
+ b.get("doc_count").cloned().unwrap_or(json!(0)),
+ );
let id = agg_key_doc_id("", &key, &order);
dest_idx
.index_document(Some(id), Value::Object(doc))
@@ -21710,7 +23574,11 @@ async fn run_rollup_job(state: &AppState, job_id: &str, config: &Value) -> Resul
.get("groups")
.and_then(|v| v.as_object())
.ok_or_else(|| "rollup job has no groups".to_string())?;
- let metrics = config.get("metrics").and_then(|v| v.as_array()).cloned().unwrap_or_default();
+ let metrics = config
+ .get("metrics")
+ .and_then(|v| v.as_array())
+ .cloned()
+ .unwrap_or_default();
// Composite sources from groups (preserve a stable order: date_histogram first, then terms, then histogram).
let mut sources: Vec<Value> = Vec::new();
@@ -21721,7 +23589,13 @@ async fn run_rollup_job(state: &AppState, job_id: &str, config: &Value) -> Resul
// composite parser understands (drop rollup-only keys like `delay`).
let mut src = serde_json::Map::new();
src.insert("field".to_string(), json!(field));
- for k in ["fixed_interval", "calendar_interval", "interval", "time_zone", "format"] {
+ for k in [
+ "fixed_interval",
+ "calendar_interval",
+ "interval",
+ "time_zone",
+ "format",
+ ] {
if let Some(v) = dh.get(k) {
src.insert(k.to_string(), v.clone());
}
@@ -21730,7 +23604,11 @@ async fn run_rollup_job(state: &AppState, job_id: &str, config: &Value) -> Resul
order.push(field.to_string());
}
}
- if let Some(terms) = groups.get("terms").and_then(|t| t.get("fields")).and_then(|f| f.as_array()) {
+ if let Some(terms) = groups
+ .get("terms")
+ .and_then(|t| t.get("fields"))
+ .and_then(|f| f.as_array())
+ {
for f in terms {
if let Some(field) = f.as_str() {
sources.push(json!({ field: { "terms": { "field": field } } }));
@@ -21743,7 +23621,9 @@ async fn run_rollup_job(state: &AppState, job_id: &str, config: &Value) -> Resul
if let Some(hfields) = hist.get("fields").and_then(|f| f.as_array()) {
for f in hfields {
if let Some(field) = f.as_str() {
- sources.push(json!({ field: { "histogram": { "field": field, "interval": interval } } }));
+ sources.push(
+ json!({ field: { "histogram": { "field": field, "interval": interval } } }),
+ );
order.push(field.to_string());
}
}
@@ -21783,7 +23663,10 @@ async fn run_rollup_job(state: &AppState, job_id: &str, config: &Value) -> Resul
return Err(format!("no index matches pattern [{index_pattern}]"));
}
- let dest_idx = state.engine.get_or_create_index(&rollup_index).map_err(|e| e.to_string())?;
+ let dest_idx = state
+ .engine
+ .get_or_create_index(&rollup_index)
+ .map_err(|e| e.to_string())?;
let mut written = 0usize;
for source_index in &matches {
@@ -21811,7 +23694,11 @@ async fn run_rollup_job(state: &AppState, job_id: &str, config: &Value) -> Resul
Some(r) => r,
None => break,
};
- let buckets = rollup.get("buckets").and_then(|b| b.as_array()).cloned().unwrap_or_default();
+ let buckets = rollup
+ .get("buckets")
+ .and_then(|b| b.as_array())
+ .cloned()
+ .unwrap_or_default();
if buckets.is_empty() {
break;
}
@@ -21826,7 +23713,10 @@ async fn run_rollup_job(state: &AppState, job_id: &str, config: &Value) -> Resul
if let Some(bo) = b.as_object() {
flatten_bucket_metrics(&mut doc, bo);
}
- doc.insert("doc_count".to_string(), b.get("doc_count").cloned().unwrap_or(json!(0)));
+ doc.insert(
+ "doc_count".to_string(),
+ b.get("doc_count").cloned().unwrap_or(json!(0)),
+ );
doc.insert("_rollup.id".to_string(), json!(job_id));
doc.insert("_rollup.source_index".to_string(), json!(source_index));
let id = agg_key_doc_id(source_index, &key, &order);
@@ -21974,7 +23864,9 @@ pub async fn get_ccr_auto_follow(
Json(json!({ "patterns": [v] })).into_response()
}
None => {
- let e = xerj_common::XerjError::index_not_found(format!("auto-follow pattern [{name}] not found"));
+ let e = xerj_common::XerjError::index_not_found(format!(
+ "auto-follow pattern [{name}] not found"
+ ));
ApiError::new(e).into_response()
}
}
@@ -21987,7 +23879,9 @@ pub async fn delete_ccr_auto_follow(
if state.engine.ccr_auto_follow.remove(&name).is_some() {
Json(json!({ "acknowledged": true })).into_response()
} else {
- let e = xerj_common::XerjError::index_not_found(format!("auto-follow pattern [{name}] not found"));
+ let e = xerj_common::XerjError::index_not_found(format!(
+ "auto-follow pattern [{name}] not found"
+ ));
ApiError::new(e).into_response()
}
}
@@ -22104,7 +23998,8 @@ pub async fn get_legacy_template(
Json(Value::Object(result)).into_response()
}
None => {
- let e = xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));
+ let e =
+ xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));
ApiError::new(e).into_response()
}
}
@@ -22176,9 +24071,7 @@ pub async fn simulate_index_template(
let overlapping: Vec<Value> = matching
.iter()
.skip(1)
- .map(|(name, _, _, _, patterns)| {
- json!({ "name": name, "index_patterns": patterns })
- })
+ .map(|(name, _, _, _, patterns)| json!({ "name": name, "index_patterns": patterns }))
.collect();
Json(json!({
@@ -22213,10 +24106,17 @@ pub async fn cat_tasks(State(state): State<AppState>) -> impl IntoResponse {
t.node,
));
}
- let body = if lines.is_empty() { String::new() } else { lines.join("\n") + "\n" };
+ let body = if lines.is_empty() {
+ String::new()
+ } else {
+ lines.join("\n") + "\n"
+ };
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
.into_response()
@@ -22225,7 +24125,8 @@ pub async fn cat_tasks(State(state): State<AppState>) -> impl IntoResponse {
pub async fn cat_repositories(State(state): State<AppState>) -> impl IntoResponse {
let mut lines = Vec::new();
for entry in state.engine.snapshot_repos.iter() {
- let repo_type = entry.value()
+ let repo_type = entry
+ .value()
.get("type")
.and_then(Value::as_str)
.unwrap_or("fs");
@@ -22238,7 +24139,10 @@ pub async fn cat_repositories(State(state): State<AppState>) -> impl IntoRespons
};
(
StatusCode::OK,
- [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
body,
)
}
@@ -22425,9 +24329,18 @@ fn eql_ci_find(hay: &str, needle: &str) -> Option<(usize, usize)> {
let ncount = needle.chars().count();
for (start, _) in hay.char_indices() {
let slice = &hay[start..];
- let cand: String = slice.chars().take(ncount).flat_map(|c| c.to_lowercase()).collect();
+ let cand: String = slice
+ .chars()
+ .take(ncount)
+ .flat_map(|c| c.to_lowercase())
+ .collect();
if cand == needle {
- let end = start + slice.chars().take(ncount).map(|c| c.len_utf8()).sum::<usize>();
+ let end = start
+ + slice
+ .chars()
+ .take(ncount)
+ .map(|c| c.len_utf8())
+ .sum::<usize>();
return Some((start, end));
}
}
@@ -22439,8 +24352,7 @@ fn eql_ci_find(hay: &str, needle: &str) -> Option<(usize, usize)> {
fn eql_parse_value(s: &str) -> (Value, bool) {
let s = s.trim();
if s.len() >= 2
- && ((s.starts_with('"') && s.ends_with('"'))
- || (s.starts_with('\'') && s.ends_with('\'')))
+ && ((s.starts_with('"') && s.ends_with('"')) || (s.starts_with('\'') && s.ends_with('\'')))
{
return (Value::String(s[1..s.len() - 1].to_string()), true);
}
@@ -22471,7 +24383,11 @@ fn eql_field_obj(wrapper: &str, field: &str, inner: Value) -> Value {
/// Translate a single `field <op> value` EQL predicate into a DSL leaf query.
/// Returns None when nothing usable can be extracted.
fn eql_predicate_to_leaf(pred: &str) -> Option<Value> {
- let pred = pred.trim().trim_start_matches('(').trim_end_matches(')').trim();
+ let pred = pred
+ .trim()
+ .trim_start_matches('(')
+ .trim_end_matches(')')
+ .trim();
if pred.is_empty() {
return None;
}
@@ -22515,7 +24431,11 @@ fn eql_split_predicates(cond: &str) -> (Vec<String>, bool) {
let or_pos = eql_ci_find(rest, " or ");
let next = match (and_pos, or_pos) {
(Some(a), Some(o)) => {
- if a.0 <= o.0 { Some((a, false)) } else { Some((o, true)) }
+ if a.0 <= o.0 {
+ Some((a, false))
+ } else {
+ Some((o, true))
+ }
}
(Some(a), None) => Some((a, false)),
(None, Some(o)) => Some((o, true)),
@@ -22549,7 +24469,10 @@ fn eql_to_query(eql: &str) -> Value {
None => return json!({ "match_all": {} }),
};
let (preds, is_or) = eql_split_predicates(cond);
- let leaves: Vec<Value> = preds.iter().filter_map(|p| eql_predicate_to_leaf(p)).collect();
+ let leaves: Vec<Value> = preds
+ .iter()
+ .filter_map(|p| eql_predicate_to_leaf(p))
+ .collect();
if leaves.is_empty() {
return json!({ "match_all": {} });
}
@@ -22674,7 +24597,12 @@ mod scripted_update_tests {
#[test]
fn read_other_source_field_in_rhs() {
let mut src = json!({ "a": 10, "b": 4 });
- apply_painless_update(&mut src, "ctx._source.c = ctx._source.a - ctx._source.b", &json!({})).unwrap();
+ apply_painless_update(
+ &mut src,
+ "ctx._source.c = ctx._source.a - ctx._source.b",
+ &json!({}),
+ )
+ .unwrap();
assert_eq!(src["c"], json!(6));
}
}
diff --git a/engine/crates/xerj-api/src/extract.rs b/engine/crates/xerj-api/src/extract.rs
index 4907c4bfb..5039406aa 100644
--- a/engine/crates/xerj-api/src/extract.rs
+++ b/engine/crates/xerj-api/src/extract.rs
@@ -142,8 +142,7 @@ where
}
}
- let value: T = serde_json::from_slice(&bytes)
- .map_err(OptionalJsonRejection::JsonParse)?;
+ let value: T = serde_json::from_slice(&bytes).map_err(OptionalJsonRejection::JsonParse)?;
Ok(OptionalJson(Some(value)))
}
}
@@ -164,10 +163,14 @@ impl<T: Default> OptionalJson<T> {
impl<T> std::ops::Deref for OptionalJson<T> {
type Target = Option<T>;
#[inline]
- fn deref(&self) -> &Self::Target { &self.0 }
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
}
impl<T> std::ops::DerefMut for OptionalJson<T> {
#[inline]
- fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut self.0
+ }
}
diff --git a/engine/crates/xerj-api/src/memory_api.rs b/engine/crates/xerj-api/src/memory_api.rs
index 3014ab66e..d9ec72243 100644
--- a/engine/crates/xerj-api/src/memory_api.rs
+++ b/engine/crates/xerj-api/src/memory_api.rs
@@ -375,10 +375,7 @@ fn extract_hits(search_response: &Value) -> Vec<Value> {
// ─────────────────────────────────────────────────────────────────────────────
/// `GET /_memory/{namespace}` — count + most-recent entries (bounded).
-pub async fn list(
- State(state): State<AppState>,
- Path(namespace): Path<String>,
-) -> Response {
+pub async fn list(State(state): State<AppState>, Path(namespace): Path<String>) -> Response {
if let Err(reason) = validate_namespace(&namespace) {
return error_response(StatusCode::BAD_REQUEST, reason);
}
@@ -529,13 +526,23 @@ mod tests {
async fn store_mem(state: &AppState, ns: &str, body: Value) -> (StatusCode, Value) {
let b: StoreBody = serde_json::from_value(body).unwrap();
- let resp = store(State(state.clone()), Path(ns.to_string()), OptionalJson(Some(b))).await;
+ let resp = store(
+ State(state.clone()),
+ Path(ns.to_string()),
+ OptionalJson(Some(b)),
+ )
+ .await;
drain_json(resp).await
}
async fn recall_mem(state: &AppState, ns: &str, body: Value) -> (StatusCode, Value) {
let b: RecallBody = serde_json::from_value(body).unwrap();
- let resp = recall(State(state.clone()), Path(ns.to_string()), OptionalJson(Some(b))).await;
+ let resp = recall(
+ State(state.clone()),
+ Path(ns.to_string()),
+ OptionalJson(Some(b)),
+ )
+ .await;
drain_json(resp).await
}
diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rs
index 918b4315d..8dc4d736b 100644
--- a/engine/crates/xerj-api/src/native.rs
+++ b/engine/crates/xerj-api/src/native.rs
@@ -109,8 +109,12 @@ pub async fn create_index(
if let Err(e) = state.engine.create_index(&name_str, schema) {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
let took_ms = started.elapsed().as_millis() as u64;
@@ -137,8 +141,12 @@ pub async fn get_index(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -173,8 +181,12 @@ pub async fn delete_index(
if let Err(e) = state.engine.delete_index(&name).await {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
let took_ms = started.elapsed().as_millis() as u64;
@@ -202,8 +214,12 @@ pub async fn ingest_docs(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -293,8 +309,12 @@ pub async fn get_doc(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -314,8 +334,12 @@ pub async fn get_doc(
}
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response()
+ native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response()
}
}
}
@@ -335,8 +359,12 @@ pub async fn delete_doc(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -352,8 +380,12 @@ pub async fn delete_doc(
}
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response()
+ native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response()
}
}
}
@@ -374,13 +406,21 @@ pub async fn search(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
state.metrics.queries_executed.inc();
- state.metrics.queries_by_index.with_label_values(&[&name]).inc();
+ state
+ .metrics
+ .queries_by_index
+ .with_label_values(&[&name])
+ .inc();
// Build a query body for the parser.
let query_body = if let Some(q_str) = &req.q {
@@ -446,8 +486,12 @@ pub async fn search(
}
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response()
+ native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response()
}
}
}
@@ -469,8 +513,12 @@ pub async fn flush_index(
if let Err(e) = state.engine.flush_index(&name).await {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
let took_ms = started.elapsed().as_millis() as u64;
@@ -646,7 +694,11 @@ pub async fn admin_backup(State(state): State<AppState>, body: Bytes) -> impl In
.name
.unwrap_or_else(|| format!("backup-{}", Uuid::new_v4()));
- match state.engine.create_snapshot(&repo_path, &name, req.indices).await {
+ match state
+ .engine
+ .create_snapshot(&repo_path, &name, req.indices)
+ .await
+ {
Ok(manifest) => {
let took_ms = started.elapsed().as_millis() as u64;
let resp = NativeResponse::new(
@@ -662,8 +714,12 @@ pub async fn admin_backup(State(state): State<AppState>, body: Bytes) -> impl In
}
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response()
+ native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response()
}
}
}
@@ -742,7 +798,11 @@ pub async fn rbac_get_role(
) -> impl IntoResponse {
match state.engine.roles.get(&name) {
Some(r) => Json(r).into_response(),
- None => (axum::http::StatusCode::NOT_FOUND, format!("role '{name}' not found")).into_response(),
+ None => (
+ axum::http::StatusCode::NOT_FOUND,
+ format!("role '{name}' not found"),
+ )
+ .into_response(),
}
}
@@ -764,7 +824,11 @@ pub async fn rbac_delete_role(
) -> impl IntoResponse {
match state.engine.roles.delete(&name) {
Some(_) => (axum::http::StatusCode::OK, "deleted").into_response(),
- None => (axum::http::StatusCode::NOT_FOUND, format!("role '{name}' not found")).into_response(),
+ None => (
+ axum::http::StatusCode::NOT_FOUND,
+ format!("role '{name}' not found"),
+ )
+ .into_response(),
}
}
@@ -805,8 +869,12 @@ pub async fn get_schema(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -839,8 +907,12 @@ pub async fn evolve_schema(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -890,8 +962,12 @@ pub async fn bulk_ingest(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -944,8 +1020,12 @@ pub async fn ingest_logs(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -1026,8 +1106,12 @@ pub async fn ingest_otlp(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -1062,13 +1146,10 @@ pub async fn ingest_otlp(
.unwrap_or("")
.to_string();
- if let Some(records) =
- scope_log.get("logRecords").and_then(Value::as_array)
- {
+ if let Some(records) = scope_log.get("logRecords").and_then(Value::as_array) {
for record in records {
total += 1;
- let doc =
- otlp_record_to_doc(record, &resource_attrs, &scope_name);
+ let doc = otlp_record_to_doc(record, &resource_attrs, &scope_name);
let id = Uuid::new_v4().to_string();
match idx.index_document(Some(id), doc).await {
Ok(_) => {
@@ -1132,23 +1213,14 @@ fn otlp_record_to_doc(
let secs = (ts_nano / 1_000_000_000) as i64;
let nanos = (ts_nano % 1_000_000_000) as u32;
if let Some(dt) = chrono::DateTime::<chrono::Utc>::from_timestamp(secs, nanos) {
- doc.insert(
- "@timestamp".to_string(),
- Value::String(dt.to_rfc3339()),
- );
+ doc.insert("@timestamp".to_string(), Value::String(dt.to_rfc3339()));
doc.insert("timestamp_nanos".to_string(), Value::Number(ts_nano.into()));
}
- } else if let Some(ts_nano) = record
- .get("timeUnixNano")
- .and_then(Value::as_u64)
- {
+ } else if let Some(ts_nano) = record.get("timeUnixNano").and_then(Value::as_u64) {
let secs = (ts_nano / 1_000_000_000) as i64;
let nanos = (ts_nano % 1_000_000_000) as u32;
if let Some(dt) = chrono::DateTime::<chrono::Utc>::from_timestamp(secs, nanos) {
- doc.insert(
- "@timestamp".to_string(),
- Value::String(dt.to_rfc3339()),
- );
+ doc.insert("@timestamp".to_string(), Value::String(dt.to_rfc3339()));
doc.insert("timestamp_nanos".to_string(), Value::Number(ts_nano.into()));
}
}
@@ -1216,7 +1288,11 @@ fn extract_otlp_any_value(any_val: &Value) -> Value {
if let Some(b) = any_val.get("boolValue").and_then(Value::as_bool) {
return Value::Bool(b);
}
- if let Some(arr) = any_val.get("arrayValue").and_then(|a| a.get("values")).and_then(Value::as_array) {
+ if let Some(arr) = any_val
+ .get("arrayValue")
+ .and_then(|a| a.get("values"))
+ .and_then(Value::as_array)
+ {
let items: Vec<Value> = arr.iter().map(extract_otlp_any_value).collect();
return Value::Array(items);
}
@@ -1252,8 +1328,12 @@ pub async fn ingest_syslog(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -1378,7 +1458,10 @@ fn try_parse_rfc5424<'a>(s: &'a str, doc: &mut serde_json::Map<String, Value>) -
let msg = parts.next().unwrap_or("");
if timestamp != "-" {
- doc.insert("@timestamp".to_string(), Value::String(timestamp.to_string()));
+ doc.insert(
+ "@timestamp".to_string(),
+ Value::String(timestamp.to_string()),
+ );
}
if hostname != "-" {
doc.insert("hostname".to_string(), Value::String(hostname.to_string()));
@@ -1389,7 +1472,10 @@ fn try_parse_rfc5424<'a>(s: &'a str, doc: &mut serde_json::Map<String, Value>) -
if procid != "-" {
doc.insert("pid".to_string(), Value::String(procid.to_string()));
}
- doc.insert("syslog_format".to_string(), Value::String("rfc5424".to_string()));
+ doc.insert(
+ "syslog_format".to_string(),
+ Value::String("rfc5424".to_string()),
+ );
// Strip optional structured data block `[...]` from the start of msg.
let msg = msg.trim_start();
@@ -1399,8 +1485,8 @@ fn try_parse_rfc5424<'a>(s: &'a str, doc: &mut serde_json::Map<String, Value>) -
} else {
msg
}
- } else if msg.starts_with('-') {
- msg[1..].trim()
+ } else if let Some(rest) = msg.strip_prefix('-') {
+ rest.trim()
} else {
msg
};
@@ -1414,8 +1500,7 @@ fn try_parse_rfc5424<'a>(s: &'a str, doc: &mut serde_json::Map<String, Value>) -
fn try_parse_rfc3164<'a>(s: &'a str, doc: &mut serde_json::Map<String, Value>) -> Option<&'a str> {
// Month abbreviations used in BSD syslog.
const MONTHS: &[&str] = &[
- "Jan", "Feb", "Mar", "Apr", "May", "Jun",
- "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
+ "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
];
// The first token should be a month abbreviation.
@@ -1435,24 +1520,35 @@ fn try_parse_rfc3164<'a>(s: &'a str, doc: &mut serde_json::Map<String, Value>) -
// HH:MM:SS
let time_end = rest.find(' ').unwrap_or(rest.len());
let time_str = &rest[..time_end];
- let rest = if time_end < rest.len() { rest[time_end..].trim_start() } else { "" };
+ let rest = if time_end < rest.len() {
+ rest[time_end..].trim_start()
+ } else {
+ ""
+ };
// Build a timestamp string.
let year = chrono::Utc::now().year();
let ts = format!("{month_str} {day_str} {time_str} {year}");
doc.insert("@timestamp".to_string(), Value::String(ts));
- doc.insert("syslog_format".to_string(), Value::String("rfc3164".to_string()));
+ doc.insert(
+ "syslog_format".to_string(),
+ Value::String("rfc3164".to_string()),
+ );
// HOSTNAME
let hostname_end = rest.find(' ').unwrap_or(rest.len());
let hostname = &rest[..hostname_end];
- let rest = if hostname_end < rest.len() { rest[hostname_end..].trim_start() } else { "" };
+ let rest = if hostname_end < rest.len() {
+ rest[hostname_end..].trim_start()
+ } else {
+ ""
+ };
if !hostname.is_empty() {
doc.insert("hostname".to_string(), Value::String(hostname.to_string()));
}
// TAG (optional, ends at ':' or '[').
- let tag_end = rest.find(|c| c == ':' || c == '[').unwrap_or(rest.len());
+ let tag_end = rest.find([':', '[']).unwrap_or(rest.len());
let tag = &rest[..tag_end];
let rest = &rest[tag_end..];
@@ -1581,8 +1677,12 @@ pub async fn turbo_ingest(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -1649,8 +1749,12 @@ pub async fn turbo_ingest(
}
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response()
+ native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response()
}
}
}
@@ -1676,8 +1780,12 @@ pub async fn get_index_encodings(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -1755,7 +1863,9 @@ pub async fn dashboard_summary(State(state): State<AppState>) -> impl IntoRespon
// Approximate disk size: rough heuristic — 200 bytes per doc in segments,
// 500 bytes per memtable doc (uncompressed).
- let size_bytes = stats.doc_count.saturating_sub(stats.memtable_doc_count as u64)
+ let size_bytes = stats
+ .doc_count
+ .saturating_sub(stats.memtable_doc_count as u64)
* 200
+ stats.memtable_doc_count as u64 * 500;
@@ -1832,8 +1942,12 @@ pub async fn enrich_index(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -1854,7 +1968,9 @@ pub async fn enrich_index(
Some(f) => f.to_string(),
None => {
return native_error(
- xerj_common::XerjError::invalid_query("enrich request must include a 'match_field' field"),
+ xerj_common::XerjError::invalid_query(
+ "enrich request must include a 'match_field' field",
+ ),
Some(&request_id),
started.elapsed().as_millis() as u64,
)
@@ -1866,7 +1982,9 @@ pub async fn enrich_index(
Some(o) => o.clone(),
None => {
return native_error(
- xerj_common::XerjError::invalid_query("enrich request must include a 'lookup' object"),
+ xerj_common::XerjError::invalid_query(
+ "enrich request must include a 'lookup' object",
+ ),
Some(&request_id),
started.elapsed().as_millis() as u64,
)
@@ -1874,10 +1992,8 @@ pub async fn enrich_index(
}
};
- let lookup: std::collections::HashMap<String, serde_json::Value> = lookup_raw
- .into_iter()
- .map(|(k, v)| (k, v))
- .collect();
+ let lookup: std::collections::HashMap<String, serde_json::Value> =
+ lookup_raw.into_iter().collect();
let entry_count = lookup.len();
let table = EnrichTable {
@@ -1931,8 +2047,12 @@ pub async fn explain_plan(
Ok(i) => i,
Err(e) => {
let xerj_err: xerj_common::XerjError = e.into();
- return native_error(xerj_err, Some(&request_id), started.elapsed().as_millis() as u64)
- .into_response();
+ return native_error(
+ xerj_err,
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
}
};
@@ -2001,7 +2121,9 @@ fn build_explain_plan(query: &xerj_query::ast::QueryNode, doc_count: u64) -> ser
"estimated_cost": "O(1)",
"fields_accessed": [],
}),
- QueryNode::Match { field, query: q, .. } => serde_json::json!({
+ QueryNode::Match {
+ field, query: q, ..
+ } => serde_json::json!({
"engine": "fts",
"description": format!("match query on field '{field}' for '{q}'"),
"estimated_docs_scanned": doc_count / 10,
@@ -2029,8 +2151,15 @@ fn build_explain_plan(query: &xerj_query::ast::QueryNode, doc_count: u64) -> ser
"estimated_cost": "O(log N)",
"fields_accessed": [field],
}),
- QueryNode::Bool { must, should, filter, must_not, .. } => {
- let sub_plans: Vec<_> = must.iter()
+ QueryNode::Bool {
+ must,
+ should,
+ filter,
+ must_not,
+ ..
+ } => {
+ let sub_plans: Vec<_> = must
+ .iter()
.chain(should.iter())
.chain(filter.iter())
.chain(must_not.iter())
@@ -2132,7 +2261,10 @@ pub async fn ingest_with_pipeline(
IngestRequest::Batch(docs) => docs,
};
- let processed = match state.engine.process_through_pipeline(&params.pipeline, docs) {
+ let processed = match state
+ .engine
+ .process_through_pipeline(&params.pipeline, docs)
+ {
Ok(p) => p,
Err(e) => {
let ze = xerj_common::XerjError::internal(e.to_string());
diff --git a/engine/crates/xerj-api/src/responses.rs b/engine/crates/xerj-api/src/responses.rs
index 63b5f4b21..3352ea5b5 100644
--- a/engine/crates/xerj-api/src/responses.rs
+++ b/engine/crates/xerj-api/src/responses.rs
@@ -340,12 +340,7 @@ impl EsSearchResponse {
}
}
- pub fn with_hits(
- took_ms: u64,
- hits: Vec<EsHit>,
- total: u64,
- max_score: Option<f64>,
- ) -> Self {
+ pub fn with_hits(took_ms: u64, hits: Vec<EsHit>, total: u64, max_score: Option<f64>) -> Self {
Self {
took: took_ms,
timed_out: false,
diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rs
index c4407e924..ab01a8b99 100644
--- a/engine/crates/xerj-api/src/router.rs
+++ b/engine/crates/xerj-api/src/router.rs
@@ -73,15 +73,9 @@ pub fn build_native_router(state: AppState) -> Router {
get(native::get_doc).delete(native::delete_doc),
)
// Bulk ingest
- .route(
- "/v1/indices/:name/docs/_bulk",
- post(native::bulk_ingest),
- )
+ .route("/v1/indices/:name/docs/_bulk", post(native::bulk_ingest))
// Turbo ingest — high-throughput batched parallel ingest (opt-in)
- .route(
- "/v1/indices/:name/turbo-ingest",
- post(native::turbo_ingest),
- )
+ .route("/v1/indices/:name/turbo-ingest", post(native::turbo_ingest))
// Log ingest (auto-ID for every record)
.route("/v1/indices/:name/logs", post(native::ingest_logs))
// OTLP log ingest (OpenTelemetry JSON format)
@@ -91,7 +85,10 @@ pub fn build_native_router(state: AppState) -> Router {
// Search
.route("/v1/indices/:name/search", post(native::search))
// Per-field smart encoding analysis
- .route("/v1/indices/:name/encodings", get(native::get_index_encodings))
+ .route(
+ "/v1/indices/:name/encodings",
+ get(native::get_index_encodings),
+ )
// Flush memtable to durable segment
.route("/v1/indices/:name/_flush", post(native::flush_index))
// Cluster / observability
@@ -106,17 +103,25 @@ pub fn build_native_router(state: AppState) -> Router {
.route("/v1/admin/flush", post(native::admin_flush))
.route("/v1/admin/backup", post(native::admin_backup))
// Admin: slow query log — v0.8 8-P6
- .route("/v1/admin/slow_queries", get(native::admin_slow_queries).delete(native::admin_slow_queries_clear))
- .route("/v1/admin/slow_queries/threshold/:ms", put(native::admin_slow_queries_set_threshold))
+ .route(
+ "/v1/admin/slow_queries",
+ get(native::admin_slow_queries).delete(native::admin_slow_queries_clear),
+ )
+ .route(
+ "/v1/admin/slow_queries/threshold/:ms",
+ put(native::admin_slow_queries_set_threshold),
+ )
// v0.9 9-P4 — Audit log
.route("/_audit/_search", get(native::audit_search))
.route("/_audit/_verify", get(native::audit_verify))
// v0.9 9-P2 — RBAC roles
.route("/_security/roles", get(native::rbac_list_roles))
- .route("/_security/role/:name",
+ .route(
+ "/_security/role/:name",
get(native::rbac_get_role)
.put(native::rbac_put_role)
- .delete(native::rbac_delete_role))
+ .delete(native::rbac_delete_role),
+ )
// (Xerj Console SPA + API — `/_xerj-console/*` — is served by xerj-console-api,
// merged at the server level. See xerj-server/src/main.rs.)
// Dashboard — overview of all indices (first step toward built-in UI)
@@ -131,7 +136,10 @@ pub fn build_native_router(state: AppState) -> Router {
// Transform pipeline management
.route("/v1/pipelines/:name", put(native::put_pipeline))
// Ingest with pipeline transformation
- .route("/v1/indices/:name/ingest", post(native::ingest_with_pipeline))
+ .route(
+ "/v1/indices/:name/ingest",
+ post(native::ingest_with_pipeline),
+ )
// Shared state
.with_state(state.clone())
// Middleware stack (applied outermost-last)
@@ -185,7 +193,10 @@ pub fn build_es_compat_router(state: AppState) -> Router {
// ── Cluster-level ──────────────────────────────────────────────────
.route("/", get(es_compat::es_info))
.route("/_cluster/health", get(es_compat::cluster_health))
- .route("/_cluster/health/:index", get(es_compat::cluster_health_for_index))
+ .route(
+ "/_cluster/health/:index",
+ get(es_compat::cluster_health_for_index),
+ )
.route("/_cat/indices", get(es_compat::cat_indices))
.route("/_cat/health", get(es_compat::cat_health))
.route("/_cat/nodes", get(es_compat::cat_nodes))
@@ -231,8 +242,14 @@ pub fn build_es_compat_router(state: AppState) -> Router {
)
.route("/:index/_stats", get(es_compat::index_stats))
.route("/:index/_disk_usage", post(es_compat::index_disk_usage))
- .route("/:index/_count", get(es_compat::count_docs).post(es_compat::count_docs))
- .route("/_count", get(es_compat::count_docs_global).post(es_compat::count_docs_global))
+ .route(
+ "/:index/_count",
+ get(es_compat::count_docs).post(es_compat::count_docs),
+ )
+ .route(
+ "/_count",
+ get(es_compat::count_docs_global).post(es_compat::count_docs_global),
+ )
.route("/:index/_refresh", post(es_compat::refresh_index))
.route("/:index/_analyze", post(es_compat::analyze_text))
.route("/_analyze", post(es_compat::analyze_text_global))
@@ -268,7 +285,10 @@ pub fn build_es_compat_router(state: AppState) -> Router {
// ── Bulk ───────────────────────────────────────────────────────────
.route("/:index/_bulk", post(es_compat::bulk_ops))
// ── Aliases ────────────────────────────────────────────────────────
- .route("/_aliases", post(es_compat::post_aliases).get(es_compat::get_aliases))
+ .route(
+ "/_aliases",
+ post(es_compat::post_aliases).get(es_compat::get_aliases),
+ )
// ── Index Templates ────────────────────────────────────────────────
.route(
"/_index_template/:name",
@@ -312,7 +332,10 @@ pub fn build_es_compat_router(state: AppState) -> Router {
// ── Cat templates ──────────────────────────────────────────────────
.route("/_cat/templates", get(es_compat::cat_templates))
// ── Ingest pipelines ───────────────────────────────────────────────
- .route("/_ingest/pipeline", get(es_compat::get_all_ingest_pipelines))
+ .route(
+ "/_ingest/pipeline",
+ get(es_compat::get_all_ingest_pipelines),
+ )
.route(
"/_ingest/pipeline/_simulate",
post(es_compat::simulate_inline_pipeline),
@@ -337,7 +360,10 @@ pub fn build_es_compat_router(state: AppState) -> Router {
// ── Admin: per-section CRC32C re-validation across every segment ──
// 200 with `corrupt_sections: 0` on healthy index; 500 on any
// corruption so external monitors fire on hits.
- .route("/:index/_admin/segments/fsck", post(es_compat::admin_segments_fsck))
+ .route(
+ "/:index/_admin/segments/fsck",
+ post(es_compat::admin_segments_fsck),
+ )
// ── Data streams ───────────────────────────────────────────────────
.route(
"/_data_stream/:name",
@@ -365,15 +391,15 @@ pub fn build_es_compat_router(state: AppState) -> Router {
// ── Cluster allocation explain ─────────────────────────────────────
.route(
"/_cluster/allocation/explain",
- get(es_compat::cluster_allocation_explain)
- .post(es_compat::cluster_allocation_explain),
+ get(es_compat::cluster_allocation_explain).post(es_compat::cluster_allocation_explain),
)
// ── Cat allocation ─────────────────────────────────────────────────
.route("/_cat/allocation", get(es_compat::cat_allocation))
// ── Index alias checks ─────────────────────────────────────────────
.route(
"/_alias",
- get(es_compat::get_all_aliases_all_indices).head(es_compat::head_all_aliases_all_indices),
+ get(es_compat::get_all_aliases_all_indices)
+ .head(es_compat::head_all_aliases_all_indices),
)
.route(
"/_alias/:alias",
@@ -411,7 +437,10 @@ pub fn build_es_compat_router(state: AppState) -> Router {
get(es_compat::get_cluster_settings).put(es_compat::put_cluster_settings),
)
.route("/_cluster/reroute", post(es_compat::cluster_reroute))
- .route("/_cluster/pending_tasks", get(es_compat::cluster_pending_tasks))
+ .route(
+ "/_cluster/pending_tasks",
+ get(es_compat::cluster_pending_tasks),
+ )
// ── More _cat APIs ──────────────────────────────────────────────────
.route("/_cat/recovery", get(es_compat::cat_recovery))
.route("/_cat/segments/:index", get(es_compat::cat_segments))
@@ -467,15 +496,24 @@ pub fn build_es_compat_router(state: AppState) -> Router {
.get(es_compat::get_script)
.delete(es_compat::delete_script),
)
- .route("/_scripts/painless/_execute", post(es_compat::painless_execute))
+ .route(
+ "/_scripts/painless/_execute",
+ post(es_compat::painless_execute),
+ )
// ── Terms enum ───────────────────────────────────────────────────────
.route("/:index/_terms_enum", post(es_compat::terms_enum))
// ── X-Pack APIs (needed for Kibana compatibility) ─────────────────────
.route("/_xpack", get(es_compat::xpack_info))
.route("/_xpack/usage", get(es_compat::xpack_usage))
// ── Security APIs ─────────────────────────────────────────────────────
- .route("/_security/_authenticate", get(es_compat::security_authenticate))
- .route("/_security/api_key", post(es_compat::security_create_api_key))
+ .route(
+ "/_security/_authenticate",
+ get(es_compat::security_authenticate),
+ )
+ .route(
+ "/_security/api_key",
+ post(es_compat::security_create_api_key),
+ )
// ── License ───────────────────────────────────────────────────────────
.route(
"/_license",
@@ -485,7 +523,10 @@ pub fn build_es_compat_router(state: AppState) -> Router {
.route("/:index/_pit", post(es_compat::open_pit))
.route("/_pit", delete(es_compat::close_pit))
// ── Internal cluster APIs ────────────────────────────────────────────
- .route("/_internal/desired_balance", get(es_compat::get_desired_balance))
+ .route(
+ "/_internal/desired_balance",
+ get(es_compat::get_desired_balance),
+ )
// ── EQL search ────────────────────────────────────────────────────────
.route("/:index/_eql/search", post(es_compat::eql_search))
// ── Global field_caps (across all indices) ────────────────────────────
@@ -528,9 +569,15 @@ pub fn build_es_compat_router(state: AppState) -> Router {
post(es_compat::score_ml_anomaly_detector),
)
// ── _cat/ml APIs ──────────────────────────────────────────────────────
- .route("/_cat/ml/anomaly_detectors", get(es_compat::cat_ml_anomaly_detectors))
+ .route(
+ "/_cat/ml/anomaly_detectors",
+ get(es_compat::cat_ml_anomaly_detectors),
+ )
.route("/_cat/ml/datafeeds", get(es_compat::cat_ml_datafeeds))
- .route("/_cat/ml/trained_models", get(es_compat::cat_ml_trained_models))
+ .route(
+ "/_cat/ml/trained_models",
+ get(es_compat::cat_ml_trained_models),
+ )
// ── Monitoring ────────────────────────────────────────────────────────
.route("/_monitoring/bulk", post(es_compat::monitoring_bulk))
// ── Transform APIs ────────────────────────────────────────────────────
@@ -561,8 +608,14 @@ pub fn build_es_compat_router(state: AppState) -> Router {
.delete(es_compat::delete_ccr_auto_follow),
)
.route("/:index/_ccr/follow", put(es_compat::ccr_follow))
- .route("/:index/_ccr/pause_follow", post(es_compat::ccr_pause_follow))
- .route("/:index/_ccr/resume_follow", post(es_compat::ccr_resume_follow))
+ .route(
+ "/:index/_ccr/pause_follow",
+ post(es_compat::ccr_pause_follow),
+ )
+ .route(
+ "/:index/_ccr/resume_follow",
+ post(es_compat::ccr_resume_follow),
+ )
.route("/:index/_ccr/unfollow", post(es_compat::ccr_unfollow))
.route("/:index/_ccr/info", get(es_compat::ccr_info))
// ── Legacy index templates (v1) ────────────────────────────────────────────
diff --git a/engine/crates/xerj-api/src/state.rs b/engine/crates/xerj-api/src/state.rs
index 8277502b3..5239b22bf 100644
--- a/engine/crates/xerj-api/src/state.rs
+++ b/engine/crates/xerj-api/src/state.rs
@@ -5,8 +5,8 @@
//! `Arc<RwLock<…>>` so concurrent requests never block each other on a
//! global lock.
-use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::Arc;
use std::time::Instant;
use chrono::{DateTime, Utc};
@@ -85,8 +85,7 @@ impl IndexHandle {
/// Approximate document count (relaxed load — fine for stats).
pub fn doc_count(&self) -> u64 {
- self.doc_count
- .load(std::sync::atomic::Ordering::Relaxed)
+ self.doc_count.load(std::sync::atomic::Ordering::Relaxed)
}
/// Increment the document counter by `n`.
@@ -190,11 +189,18 @@ impl TaskRegistry {
};
let key = entry.key();
self.inner.insert(key.clone(), entry);
- TaskHandle { inner: self.inner.clone(), key, cancelled }
+ TaskHandle {
+ inner: self.inner.clone(),
+ key,
+ cancelled,
+ }
}
pub fn cancel(&self, id: &str) -> bool {
match self.inner.get(id) {
- Some(e) => { e.cancelled.store(true, Ordering::Relaxed); true }
+ Some(e) => {
+ e.cancelled.store(true, Ordering::Relaxed);
+ true
+ }
None => false,
}
}
@@ -251,7 +257,9 @@ pub fn default_anomaly_threshold() -> f64 {
impl MlDetector {
/// Absolute path of the on-disk detector registry file.
fn registry_path(data_dir: &str) -> std::path::PathBuf {
- std::path::Path::new(data_dir).join("_ml").join("detectors.json")
+ std::path::Path::new(data_dir)
+ .join("_ml")
+ .join("detectors.json")
}
/// Load all persisted detectors from `<data_dir>/_ml/detectors.json`.
@@ -364,7 +372,6 @@ impl AppState {
impl std::fmt::Debug for AppState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("AppState")
- .finish_non_exhaustive()
+ f.debug_struct("AppState").finish_non_exhaustive()
}
}
diff --git a/engine/crates/xerj-api/src/stub.rs b/engine/crates/xerj-api/src/stub.rs
index 6f4d1b4cd..b1006d562 100644
--- a/engine/crates/xerj-api/src/stub.rs
+++ b/engine/crates/xerj-api/src/stub.rs
@@ -38,9 +38,9 @@ use serde_json::json;
/// envelope so existing clients log it predictably.
///
/// * `feature` — short human label, e.g. `"EQL search"` or
-/// `"Watcher (scheduled queries)"`.
+/// `"Watcher (scheduled queries)"`.
/// * `milestone` — the version this is planned for, e.g. `"v0.7"`
-/// or `"v1.x"` for un-scheduled features.
+/// or `"v1.x"` for un-scheduled features.
/// * `note` — one-sentence detail; printed in `error.reason`.
///
/// Returns a 501 with a JSON body and a `Retry-After: 0` header
diff --git a/engine/crates/xerj-cluster/src/coordinator.rs b/engine/crates/xerj-cluster/src/coordinator.rs
index 3a0f3e462..1b2c72da8 100644
--- a/engine/crates/xerj-cluster/src/coordinator.rs
+++ b/engine/crates/xerj-cluster/src/coordinator.rs
@@ -170,13 +170,9 @@ impl SearchCoordinator {
// Local search — bypass network
debug!(node = %self.local_node_id, index, "Serving search locally");
let response = match &self.local_searcher {
- Some(searcher) => searcher.search_local(
- &request_id,
- index,
- query_json,
- size + from,
- 0,
- ),
+ Some(searcher) => {
+ searcher.search_local(&request_id, index, query_json, size + from, 0)
+ }
None => {
// No local searcher configured — return empty shard response.
warn!(
@@ -257,9 +253,7 @@ impl SearchCoordinator {
// Forward to remote shard owner
debug!(
target = node,
- index,
- doc_id,
- "Forwarding index request to remote node"
+ index, doc_id, "Forwarding index request to remote node"
);
let req = SearchMessage::IndexRequest {
index: index.to_string(),
@@ -280,10 +274,7 @@ impl SearchCoordinator {
SearchMessage::Error { message, .. } => {
Err(anyhow!("remote index error: {}", message))
}
- other => Err(anyhow!(
- "unexpected response to IndexRequest: {:?}",
- other
- )),
+ other => Err(anyhow!("unexpected response to IndexRequest: {:?}", other)),
}
}
// Local node owns the shard (or no assignment — handle locally).
@@ -449,17 +440,9 @@ mod tests {
);
let transport_ref = Arc::clone(&transport);
- let coordinator = SearchCoordinator::new(
- Arc::new(router),
- transport_ref,
- "gateway",
- None,
- );
+ let coordinator = SearchCoordinator::new(Arc::new(router), transport_ref, "gateway", None);
- coordinator
- .search("logs", "{}", 10, 0)
- .await
- .unwrap();
+ coordinator.search("logs", "{}", 10, 0).await.unwrap();
let calls = transport.calls();
assert_eq!(calls.len(), 3, "exactly one call per node");
@@ -482,33 +465,18 @@ mod tests {
let transport = MockSearchTransport::new()
.with_response(
"node-a",
- make_response(
- vec![("p1", 3.0), ("p3", 1.5), ("p5", 0.8)],
- 100,
- 10,
- ),
+ make_response(vec![("p1", 3.0), ("p3", 1.5), ("p5", 0.8)], 100, 10),
)
.with_response(
"node-b",
- make_response(
- vec![("p2", 2.5), ("p4", 1.2), ("p6", 0.5)],
- 80,
- 8,
- ),
+ make_response(vec![("p2", 2.5), ("p4", 1.2), ("p6", 0.5)], 80, 8),
);
- let coordinator = SearchCoordinator::new(
- Arc::new(router),
- Arc::new(transport),
- "gateway",
- None,
- );
+ let coordinator =
+ SearchCoordinator::new(Arc::new(router), Arc::new(transport), "gateway", None);
// Page 1: size=2, from=0
- let page1 = coordinator
- .search("products", "{}", 2, 0)
- .await
- .unwrap();
+ let page1 = coordinator.search("products", "{}", 2, 0).await.unwrap();
assert_eq!(page1.hits[0].id, "p1"); // 3.0
assert_eq!(page1.hits[1].id, "p2"); // 2.5
assert_eq!(page1.total, 180);
@@ -518,36 +486,21 @@ mod tests {
let transport2 = MockSearchTransport::new()
.with_response(
"node-a",
- make_response(
- vec![("p1", 3.0), ("p3", 1.5), ("p5", 0.8)],
- 100,
- 10,
- ),
+ make_response(vec![("p1", 3.0), ("p3", 1.5), ("p5", 0.8)], 100, 10),
)
.with_response(
"node-b",
- make_response(
- vec![("p2", 2.5), ("p4", 1.2), ("p6", 0.5)],
- 80,
- 8,
- ),
+ make_response(vec![("p2", 2.5), ("p4", 1.2), ("p6", 0.5)], 80, 8),
);
let mut router2 = ShardRouter::new(2);
router2.assign("products", 0, "node-a");
router2.assign("products", 1, "node-b");
- let coordinator2 = SearchCoordinator::new(
- Arc::new(router2),
- Arc::new(transport2),
- "gateway",
- None,
- );
+ let coordinator2 =
+ SearchCoordinator::new(Arc::new(router2), Arc::new(transport2), "gateway", None);
- let page2 = coordinator2
- .search("products", "{}", 2, 2)
- .await
- .unwrap();
+ let page2 = coordinator2.search("products", "{}", 2, 2).await.unwrap();
assert_eq!(page2.hits[0].id, "p3"); // 1.5
assert_eq!(page2.hits[1].id, "p4"); // 1.2
}
@@ -561,12 +514,7 @@ mod tests {
let transport = Arc::new(MockSearchTransport::new());
let transport_ref = Arc::clone(&transport);
- let coordinator = SearchCoordinator::new(
- Arc::new(router),
- transport_ref,
- "node-0",
- None,
- );
+ let coordinator = SearchCoordinator::new(Arc::new(router), transport_ref, "node-0", None);
let result = coordinator
.search("empty-index", "{}", 10, 0)
@@ -575,7 +523,10 @@ mod tests {
assert_eq!(result.hits.len(), 0);
assert_eq!(result.total, 0);
- assert!(transport.calls().is_empty(), "no transport calls for empty index");
+ assert!(
+ transport.calls().is_empty(),
+ "no transport calls for empty index"
+ );
}
// ── Index routing test ────────────────────────────────────────────────────
@@ -616,12 +567,8 @@ mod tests {
let transport = MockSearchTransport::new().with_response("node-remote", remote_resp);
- let coordinator = SearchCoordinator::new(
- Arc::new(router),
- Arc::new(transport),
- "node-local",
- None,
- );
+ let coordinator =
+ SearchCoordinator::new(Arc::new(router), Arc::new(transport), "node-local", None);
let resp = coordinator
.route_index("orders", "doc-xyz", r#"{"amount":99}"#)
diff --git a/engine/crates/xerj-cluster/src/lib.rs b/engine/crates/xerj-cluster/src/lib.rs
index 7a90bec50..2100a0439 100644
--- a/engine/crates/xerj-cluster/src/lib.rs
+++ b/engine/crates/xerj-cluster/src/lib.rs
@@ -60,14 +60,18 @@ pub mod transport;
// ── Convenience re-exports ────────────────────────────────────────────────────
-pub use coordinator::{IndexResponse, LocalSearcher, MergedSearchResult, SearchCoordinator, SearchTransport};
+pub use coordinator::{
+ IndexResponse, LocalSearcher, MergedSearchResult, SearchCoordinator, SearchTransport,
+};
pub use metadata::{ClusterMetadata, IndexMetadata, NodeInfo, NodeState};
pub use node::{ClusterNode, ClusterTransport};
pub use raft::{ClusterCommand, LogEntry, RaftMessage, RaftNode, RaftState};
pub use regions::{Region, RegionManager, RegionMove};
-pub use replication::{ReplicationEntry, ReplicationMode, ReplicationOp, ReplicationResult, WalReplicator};
-pub use router::ShardRouter;
+pub use replication::{
+ ReplicationEntry, ReplicationMode, ReplicationOp, ReplicationResult, WalReplicator,
+};
pub use router::jump_hash;
+pub use router::ShardRouter;
pub use runner::{ClusterRunner, ClusterRunnerBuilder};
-pub use search::{SearchHit, SearchMessage, merge_search_responses};
+pub use search::{merge_search_responses, SearchHit, SearchMessage};
pub use transport::TcpTransport;
diff --git a/engine/crates/xerj-cluster/src/metadata.rs b/engine/crates/xerj-cluster/src/metadata.rs
index d806cee2e..c095a8aea 100644
--- a/engine/crates/xerj-cluster/src/metadata.rs
+++ b/engine/crates/xerj-cluster/src/metadata.rs
@@ -118,10 +118,17 @@ impl ClusterMetadata {
ClusterCommand::DeleteIndex { name } => {
self.apply_delete_index(name);
}
- ClusterCommand::UpdateMapping { index, mapping_json } => {
+ ClusterCommand::UpdateMapping {
+ index,
+ mapping_json,
+ } => {
self.apply_update_mapping(index, mapping_json);
}
- ClusterCommand::AssignShard { index, shard, node_id } => {
+ ClusterCommand::AssignShard {
+ index,
+ shard,
+ node_id,
+ } => {
self.apply_assign_shard(index, *shard, node_id);
}
ClusterCommand::AddNode { node_id, address } => {
@@ -168,8 +175,7 @@ impl ClusterMetadata {
return;
}
// Clean up shard assignments for this index
- self.shard_assignments
- .retain(|(idx, _), _| idx != name);
+ self.shard_assignments.retain(|(idx, _), _| idx != name);
info!(index = %name, "Applying DeleteIndex");
}
@@ -266,12 +272,16 @@ mod tests {
},
);
assert_eq!(
- meta.indices["logs"].schema_json,
- r#"{"properties":{}}"#,
+ meta.indices["logs"].schema_json, r#"{"properties":{}}"#,
"duplicate create should not overwrite schema"
);
- meta.apply(3, &ClusterCommand::DeleteIndex { name: "logs".to_string() });
+ meta.apply(
+ 3,
+ &ClusterCommand::DeleteIndex {
+ name: "logs".to_string(),
+ },
+ );
assert!(!meta.indices.contains_key("logs"));
}
@@ -288,7 +298,12 @@ mod tests {
);
assert_eq!(meta.nodes["n1"].state, NodeState::Active);
- meta.apply(2, &ClusterCommand::RemoveNode { node_id: "n1".to_string() });
+ meta.apply(
+ 2,
+ &ClusterCommand::RemoveNode {
+ node_id: "n1".to_string(),
+ },
+ );
assert_eq!(meta.nodes["n1"].state, NodeState::Down);
assert!(
@@ -367,7 +382,10 @@ mod tests {
meta.apply(entry.index, &entry.command);
}
- assert!(meta.indices.contains_key("events"), "CreateIndex should be applied");
+ assert!(
+ meta.indices.contains_key("events"),
+ "CreateIndex should be applied"
+ );
assert!(meta.nodes.contains_key("n2"), "AddNode should be applied");
}
}
diff --git a/engine/crates/xerj-cluster/src/node.rs b/engine/crates/xerj-cluster/src/node.rs
index c0f4f693b..6f66605b1 100644
--- a/engine/crates/xerj-cluster/src/node.rs
+++ b/engine/crates/xerj-cluster/src/node.rs
@@ -46,11 +46,7 @@ pub struct ClusterNode {
impl ClusterNode {
/// Create a new cluster node with the given identity, peers, and transport.
- pub fn new(
- id: String,
- peers: Vec<String>,
- transport: Box<dyn ClusterTransport>,
- ) -> Self {
+ pub fn new(id: String, peers: Vec<String>, transport: Box<dyn ClusterTransport>) -> Self {
ClusterNode {
raft: RaftNode::new(id, peers),
metadata: ClusterMetadata::new(),
@@ -99,11 +95,7 @@ impl ClusterNode {
info!(node = %self.raft.id, "Starting cluster node run loop");
loop {
// Try to receive a message within the tick interval
- let recv_result = tokio::time::timeout(
- tick_interval,
- self.transport.recv(),
- )
- .await;
+ let recv_result = tokio::time::timeout(tick_interval, self.transport.recv()).await;
match recv_result {
Ok(Ok((from, msg))) => {
@@ -199,27 +191,22 @@ pub mod in_memory {
/// Register a node and return its dedicated receiver channel.
pub async fn register(&self, node_id: &str) -> Receiver {
let (tx, rx) = mpsc::unbounded_channel();
- self.senders
- .lock()
- .await
- .insert(node_id.to_string(), tx);
+ self.senders.lock().await.insert(node_id.to_string(), tx);
rx
}
/// Send a message directly (used by [`InMemoryTransport`]).
- pub async fn send_to(
- &self,
- from: &str,
- to: &str,
- msg: RaftMessage,
- ) -> Result<()> {
+ pub async fn send_to(&self, from: &str, to: &str, msg: RaftMessage) -> Result<()> {
let senders = self.senders.lock().await;
if let Some(tx) = senders.get(to) {
tx.send((from.to_string(), msg))
.map_err(|e| anyhow::anyhow!("channel closed: {}", e))?;
} else {
// Node not registered — simulate network partition
- debug!(from, to, "InMemoryBus: target node not found (partitioned?)");
+ debug!(
+ from,
+ to, "InMemoryBus: target node not found (partitioned?)"
+ );
}
Ok(())
}
@@ -271,9 +258,7 @@ mod tests {
#[allow(unused_imports)]
use std::time::Duration;
- async fn make_cluster(
- ids: &[&str],
- ) -> Vec<ClusterNode> {
+ async fn make_cluster(ids: &[&str]) -> Vec<ClusterNode> {
let bus = InMemoryBus::new();
let id_list: Vec<String> = ids.iter().map(|s| s.to_string()).collect();
@@ -285,11 +270,7 @@ mod tests {
.cloned()
.collect();
let transport = InMemoryTransport::new(id.to_string(), bus.clone()).await;
- let node = ClusterNode::new(
- id.to_string(),
- peers,
- Box::new(transport),
- );
+ let node = ClusterNode::new(id.to_string(), peers, Box::new(transport));
nodes.push(node);
}
nodes
diff --git a/engine/crates/xerj-cluster/src/raft.rs b/engine/crates/xerj-cluster/src/raft.rs
index 7055bbaf0..c44d9da95 100644
--- a/engine/crates/xerj-cluster/src/raft.rs
+++ b/engine/crates/xerj-cluster/src/raft.rs
@@ -322,7 +322,12 @@ impl RaftNode {
last_log_index,
last_log_term,
} => {
- out.extend(self.handle_request_vote(term, candidate_id, last_log_index, last_log_term));
+ out.extend(self.handle_request_vote(
+ term,
+ candidate_id,
+ last_log_index,
+ last_log_term,
+ ));
}
RaftMessage::RequestVoteResponse {
from,
@@ -427,8 +432,7 @@ impl RaftNode {
/// send AppendEntries to all peers right now.
pub fn force_heartbeat(&mut self) -> Vec<OutboundMessage> {
if self.state == RaftState::Leader {
- self.last_heartbeat_sent =
- Instant::now() - Duration::from_millis(1000);
+ self.last_heartbeat_sent = Instant::now() - Duration::from_millis(1000);
self.send_heartbeats()
} else {
vec![]
@@ -766,8 +770,8 @@ impl RaftNode {
// a) haven't voted yet (or already voted for this candidate) this term
// b) candidate's log is at least as up-to-date as ours (§5.4)
if term >= self.current_term {
- let can_vote = self.voted_for.is_none()
- || self.voted_for.as_deref() == Some(&candidate_id);
+ let can_vote =
+ self.voted_for.is_none() || self.voted_for.as_deref() == Some(&candidate_id);
// "Up-to-date" check: candidate's last log term > ours, OR
// same last log term and candidate's log is at least as long.
@@ -888,12 +892,14 @@ mod tests {
let mut node = RaftNode::new("n1".to_string(), vec![]);
// Force an immediate election by resetting the heartbeat timer
- node.last_heartbeat =
- Instant::now() - Duration::from_millis(500);
+ node.last_heartbeat = Instant::now() - Duration::from_millis(500);
let msgs = node.tick();
// Single-node cluster: should become leader immediately, no messages needed
- assert!(node.is_leader(), "single node must become leader after timeout");
+ assert!(
+ node.is_leader(),
+ "single node must become leader after timeout"
+ );
// No peers, so no messages to send (or only heartbeat to self which is empty)
assert!(
msgs.is_empty() || msgs.iter().all(|m| m.to == "n1"),
diff --git a/engine/crates/xerj-cluster/src/raft_log.rs b/engine/crates/xerj-cluster/src/raft_log.rs
index 4e20b5bf7..199d8962e 100644
--- a/engine/crates/xerj-cluster/src/raft_log.rs
+++ b/engine/crates/xerj-cluster/src/raft_log.rs
@@ -54,7 +54,7 @@ use crc32fast::Hasher as Crc32;
/// Magic for the raft log file format. Distinct from the WAL magic so
/// we never confuse one for the other on recovery.
-const MAGIC: u32 = 0x5A_52_4C_32; // "ZRL2"
+const MAGIC: u32 = 0x5A52_4C32; // "ZRL2" (ASCII Z R L 2)
const LOG_FILE: &str = "raft.log";
const COMMIT_FILE: &str = "commit.meta";
@@ -68,17 +68,6 @@ pub enum EntryKind {
Snapshot = 2,
}
-impl EntryKind {
- fn from_u8(b: u8) -> Result<Self> {
- match b {
- 0 => Ok(Self::ClusterCommand),
- 1 => Ok(Self::ConfigChange),
- 2 => Ok(Self::Snapshot),
- _ => Err(anyhow!("unknown raft entry kind {b}")),
- }
- }
-}
-
/// A single persisted log entry.
///
/// The payload is the `serde_json::to_vec` form of a
@@ -257,7 +246,8 @@ impl FileRaftLog {
self.writer.write_all(payload)?;
self.writer.write_u32::<LittleEndian>(crc)?;
- self.seek_table.insert(index, (term, header_offset, payload_len));
+ self.seek_table
+ .insert(index, (term, header_offset, payload_len));
if index > self.last_index {
self.last_index = index;
}
@@ -293,8 +283,8 @@ impl FileRaftLog {
// Open a fresh read handle so we don't fight with the writer's
// buffered position.
let log_path = self.dir.join(LOG_FILE);
- let mut file = File::open(&log_path)
- .with_context(|| format!("reopening {log_path:?} for read"))?;
+ let mut file =
+ File::open(&log_path).with_context(|| format!("reopening {log_path:?} for read"))?;
// Header is: magic(4) + entry_len(8) + term(8) + index(8) + kind(1) + payload_len(8)
// = 37 bytes. Then the payload.
file.seek(SeekFrom::Start(offset + 37))
diff --git a/engine/crates/xerj-cluster/src/regions.rs b/engine/crates/xerj-cluster/src/regions.rs
index 8ba946af0..a70fb6e1b 100644
--- a/engine/crates/xerj-cluster/src/regions.rs
+++ b/engine/crates/xerj-cluster/src/regions.rs
@@ -136,8 +136,7 @@ impl RegionManager {
self.regions
.iter()
.filter(|r| {
- r.size_bytes > self.split_threshold_bytes
- || r.doc_count > self.split_threshold_docs
+ r.size_bytes > self.split_threshold_bytes || r.doc_count > self.split_threshold_docs
})
.collect()
}
@@ -313,7 +312,11 @@ impl RegionManager {
let rb = self.regions[pos_b].clone();
if ra.index != rb.index {
- bail!("cannot merge regions from different indices: {} vs {}", ra.index, rb.index);
+ bail!(
+ "cannot merge regions from different indices: {} vs {}",
+ ra.index,
+ rb.index
+ );
}
// Verify adjacency.
@@ -366,7 +369,11 @@ impl RegionManager {
};
// Remove old regions (remove higher index first to keep positions stable).
- let (hi, lo) = if pos_a > pos_b { (pos_a, pos_b) } else { (pos_b, pos_a) };
+ let (hi, lo) = if pos_a > pos_b {
+ (pos_a, pos_b)
+ } else {
+ (pos_b, pos_a)
+ };
self.regions.remove(hi);
self.regions.remove(lo);
self.regions.push(merged.clone());
@@ -399,9 +406,7 @@ impl RegionManager {
let region_loads: HashMap<String, Vec<u64>> = {
let mut m: HashMap<String, Vec<u64>> = HashMap::new();
for r in &self.regions {
- m.entry(r.leader_node.clone())
- .or_default()
- .push(r.id);
+ m.entry(r.leader_node.clone()).or_default().push(r.id);
}
m
};
@@ -409,14 +414,8 @@ impl RegionManager {
loop {
// Find most-loaded and least-loaded nodes.
- let max_node = loads
- .iter()
- .max_by_key(|(_, v)| *v)
- .map(|(k, _)| k.clone());
- let min_node = loads
- .iter()
- .min_by_key(|(_, v)| *v)
- .map(|(k, _)| k.clone());
+ let max_node = loads.iter().max_by_key(|(_, v)| *v).map(|(k, _)| k.clone());
+ let min_node = loads.iter().min_by_key(|(_, v)| *v).map(|(k, _)| k.clone());
let (max_node, min_node) = match (max_node, min_node) {
(Some(a), Some(b)) if a != b => (a, b),
@@ -443,15 +442,17 @@ impl RegionManager {
};
// Simulate the load transfer.
- *loads.get_mut(&max_node).unwrap() =
- max_load.saturating_sub(region.size_bytes);
+ *loads.get_mut(&max_node).unwrap() = max_load.saturating_sub(region.size_bytes);
*loads.entry(min_node.clone()).or_insert(0) += region.size_bytes;
// Update the working region-owner map.
if let Some(ids) = node_regions.get_mut(&max_node) {
ids.retain(|&id| id != region_id);
}
- node_regions.entry(min_node.clone()).or_default().push(region_id);
+ node_regions
+ .entry(min_node.clone())
+ .or_default()
+ .push(region_id);
moves.push(RegionMove {
region_id,
@@ -471,12 +472,7 @@ impl RegionManager {
/// Update a region's statistics after an indexing or deletion operation.
///
/// `size_delta` and `doc_delta` may be negative (deletions).
- pub fn update_region_stats(
- &mut self,
- region_id: u64,
- size_delta: i64,
- doc_delta: i64,
- ) {
+ pub fn update_region_stats(&mut self, region_id: u64, size_delta: i64, doc_delta: i64) {
if let Some(r) = self.regions.iter_mut().find(|r| r.id == region_id) {
r.size_bytes = (r.size_bytes as i64 + size_delta).max(0) as u64;
r.doc_count = (r.doc_count as i64 + doc_delta).max(0) as u64;
@@ -668,7 +664,8 @@ mod tests {
// Regions are contiguous: each start_key == previous end_key.
for i in 1..created.len() {
assert_eq!(
- created[i].start_key, created[i - 1].end_key,
+ created[i].start_key,
+ created[i - 1].end_key,
"region {i} start_key should equal region {} end_key",
i - 1
);
@@ -680,7 +677,7 @@ mod tests {
*counts.entry(r.leader_node.as_str()).or_default() += 1;
}
assert_eq!(counts.len(), 3);
- for (_, count) in &counts {
+ for count in counts.values() {
assert_eq!(*count, 2);
}
}
@@ -739,13 +736,10 @@ mod tests {
mgr.update_region_stats(r.id, 10 * 1024 * 1024, 1000);
}
- let loads: HashMap<String, u64> = mgr
- .regions()
- .iter()
- .fold(HashMap::new(), |mut m, r| {
- *m.entry(r.leader_node.clone()).or_default() += r.size_bytes;
- m
- });
+ let loads: HashMap<String, u64> = mgr.regions().iter().fold(HashMap::new(), |mut m, r| {
+ *m.entry(r.leader_node.clone()).or_default() += r.size_bytes;
+ m
+ });
let moves = mgr.plan_rebalance(&loads);
assert!(
@@ -760,7 +754,7 @@ mod tests {
/// the routed doc_id (when the region map covers the same key space).
#[test]
fn test_jump_hash_routes_to_region() {
- use crate::router::{ShardRouter, jump_hash};
+ use crate::router::{jump_hash, ShardRouter};
use xxhash_rust::xxh3::xxh3_64;
let mut router = ShardRouter::new(4);
@@ -792,7 +786,10 @@ mod tests {
r.start_key.clone()
};
let found = mgr.region_for_key("docs", &probe_key);
- assert!(found.is_some(), "probe key {probe_key:?} should be covered by a region");
+ assert!(
+ found.is_some(),
+ "probe key {probe_key:?} should be covered by a region"
+ );
}
}
diff --git a/engine/crates/xerj-cluster/src/replication.rs b/engine/crates/xerj-cluster/src/replication.rs
index bc0f47dde..2438e7c14 100644
--- a/engine/crates/xerj-cluster/src/replication.rs
+++ b/engine/crates/xerj-cluster/src/replication.rs
@@ -87,7 +87,10 @@ pub struct ReplicationResult {
impl ReplicationResult {
/// Construct a successful result with the given ACK count.
pub fn success(ack_count: usize, duration_ms: u64) -> Self {
- ReplicationResult { ack_count, duration_ms }
+ ReplicationResult {
+ ack_count,
+ duration_ms,
+ }
}
}
@@ -117,7 +120,12 @@ impl WalReplicator {
router: Arc<ShardRouter>,
local_node_id: String,
) -> Self {
- WalReplicator { mode, transport, router, local_node_id }
+ WalReplicator {
+ mode,
+ transport,
+ router,
+ local_node_id,
+ }
}
/// Replicate a WAL entry to replica nodes according to the configured mode.
@@ -188,11 +196,7 @@ impl WalReplicator {
/// (i.e. the transport layer accepts the frame without error). A production
/// system would exchange proper ACK messages; here we model success as
/// transport-level delivery.
- async fn replicate_and_wait(
- &self,
- replicas: &[String],
- entry: &ReplicationEntry,
- ) -> usize {
+ async fn replicate_and_wait(&self, replicas: &[String], entry: &ReplicationEntry) -> usize {
if replicas.is_empty() {
return 0;
}
@@ -218,9 +222,8 @@ impl WalReplicator {
// Wait for all replicas to respond (or fail).
let mut acks = 0usize;
for rx in receivers {
- match rx.await {
- Ok(true) => acks += 1,
- _ => {}
+ if let Ok(true) = rx.await {
+ acks += 1
}
}
acks
@@ -349,8 +352,7 @@ mod tests {
for entry in &entries {
let json = serde_json::to_string(entry).expect("serialize");
- let decoded: ReplicationEntry =
- serde_json::from_str(&json).expect("deserialize");
+ let decoded: ReplicationEntry = serde_json::from_str(&json).expect("deserialize");
assert_eq!(entry.index, decoded.index);
assert_eq!(entry.shard, decoded.shard);
assert_eq!(entry.seq_no, decoded.seq_no);
@@ -379,8 +381,7 @@ mod tests {
use crate::node::in_memory::{InMemoryBus, InMemoryTransport};
let bus = InMemoryBus::new();
- let transport_primary =
- InMemoryTransport::new("primary".to_string(), bus.clone()).await;
+ let transport_primary = InMemoryTransport::new("primary".to_string(), bus.clone()).await;
let _r1 = InMemoryTransport::new("replica-1".to_string(), bus.clone()).await;
let router = make_router_with_replicas("idx", "primary", &["replica-1"]);
@@ -393,7 +394,10 @@ mod tests {
);
let entry = make_entry("idx", 0, 1);
- let result = replicator.replicate(entry).await.expect("async should succeed");
+ let result = replicator
+ .replicate(entry)
+ .await
+ .expect("async should succeed");
assert_eq!(result.ack_count, 0, "async mode never counts ACKs");
}
@@ -411,7 +415,10 @@ mod tests {
);
let entry = make_entry("idx", 0, 5);
- let result = replicator.replicate(entry).await.expect("sync should succeed");
+ let result = replicator
+ .replicate(entry)
+ .await
+ .expect("sync should succeed");
assert_eq!(result.ack_count, 2, "both replicas must ACK");
}
@@ -429,10 +436,15 @@ mod tests {
"primary".to_string(),
);
- let err = replicator.replicate(make_entry("idx", 0, 7)).await
+ let err = replicator
+ .replicate(make_entry("idx", 0, 7))
+ .await
.expect_err("should fail with insufficient replicas");
assert!(
- matches!(err, ReplicationError::InsufficientReplicas { needed: 2, got: 1 }),
+ matches!(
+ err,
+ ReplicationError::InsufficientReplicas { needed: 2, got: 1 }
+ ),
"unexpected error: {err}"
);
}
@@ -457,7 +469,9 @@ mod tests {
"primary".to_string(),
);
- let result = replicator.replicate(make_entry("idx", 0, 10)).await
+ let result = replicator
+ .replicate(make_entry("idx", 0, 10))
+ .await
.expect("quorum should succeed with 2/3 replicas");
assert_eq!(result.ack_count, 2, "2 of 3 replicas must ACK for majority");
}
diff --git a/engine/crates/xerj-cluster/src/router.rs b/engine/crates/xerj-cluster/src/router.rs
index a542a8914..4f215a00d 100644
--- a/engine/crates/xerj-cluster/src/router.rs
+++ b/engine/crates/xerj-cluster/src/router.rs
@@ -96,10 +96,8 @@ impl ShardRouter {
let mut targets: Vec<String> = Vec::new();
for ((idx, _shard), node_id) in &self.routing_table {
- if idx == index {
- if seen.insert(node_id.as_str()) {
- targets.push(node_id.clone());
- }
+ if idx == index && seen.insert(node_id.as_str()) {
+ targets.push(node_id.clone());
}
}
diff --git a/engine/crates/xerj-cluster/src/runner.rs b/engine/crates/xerj-cluster/src/runner.rs
index 30b373e58..78c484126 100644
--- a/engine/crates/xerj-cluster/src/runner.rs
+++ b/engine/crates/xerj-cluster/src/runner.rs
@@ -159,11 +159,7 @@ impl ClusterRunnerBuilder {
self
}
- pub fn build(
- self,
- node: ClusterNode,
- shutdown: watch::Receiver<bool>,
- ) -> ClusterRunner {
+ pub fn build(self, node: ClusterNode, shutdown: watch::Receiver<bool>) -> ClusterRunner {
ClusterRunner::new(node, self.tick_interval, shutdown)
}
}
diff --git a/engine/crates/xerj-cluster/src/search.rs b/engine/crates/xerj-cluster/src/search.rs
index f19570f9c..86c30a610 100644
--- a/engine/crates/xerj-cluster/src/search.rs
+++ b/engine/crates/xerj-cluster/src/search.rs
@@ -22,7 +22,6 @@ use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum SearchMessage {
// ── Search ────────────────────────────────────────────────────────────────
-
/// Gateway → Search node: execute query on local shards.
SearchRequest {
/// Unique identifier for correlating requests and responses.
@@ -50,7 +49,6 @@ pub enum SearchMessage {
},
// ── Indexing ──────────────────────────────────────────────────────────────
-
/// Gateway → Node: forward a document for indexing on the target shard.
IndexRequest {
/// Index to write into.
@@ -72,7 +70,6 @@ pub enum SearchMessage {
},
// ── Errors ────────────────────────────────────────────────────────────────
-
/// Any node → requester: signals that a request could not be fulfilled.
Error {
/// The `request_id` from the original request, if available.
diff --git a/engine/crates/xerj-cluster/src/transport.rs b/engine/crates/xerj-cluster/src/transport.rs
index 346835d7a..fc6d4bf06 100644
--- a/engine/crates/xerj-cluster/src/transport.rs
+++ b/engine/crates/xerj-cluster/src/transport.rs
@@ -26,8 +26,14 @@ use crate::raft::RaftMessage;
async fn write_frame(stream: &mut TcpStream, msg: &RaftMessage) -> Result<()> {
let payload = serde_json::to_vec(msg).context("serialize RaftMessage")?;
let len = payload.len() as u32;
- stream.write_all(&len.to_be_bytes()).await.context("write frame length")?;
- stream.write_all(&payload).await.context("write frame payload")?;
+ stream
+ .write_all(&len.to_be_bytes())
+ .await
+ .context("write frame length")?;
+ stream
+ .write_all(&payload)
+ .await
+ .context("write frame payload")?;
stream.flush().await.context("flush frame")?;
Ok(())
}
@@ -35,7 +41,10 @@ async fn write_frame(stream: &mut TcpStream, msg: &RaftMessage) -> Result<()> {
/// Read a single length-prefixed JSON frame from the given stream.
async fn read_frame(stream: &mut TcpStream) -> Result<RaftMessage> {
let mut len_buf = [0u8; 4];
- stream.read_exact(&mut len_buf).await.context("read frame length")?;
+ stream
+ .read_exact(&mut len_buf)
+ .await
+ .context("read frame length")?;
let len = u32::from_be_bytes(len_buf) as usize;
// Guard against absurdly large frames (e.g. 10 MiB max).
@@ -44,7 +53,10 @@ async fn read_frame(stream: &mut TcpStream) -> Result<RaftMessage> {
}
let mut payload = vec![0u8; len];
- stream.read_exact(&mut payload).await.context("read frame payload")?;
+ stream
+ .read_exact(&mut payload)
+ .await
+ .context("read frame payload")?;
let msg: RaftMessage = serde_json::from_slice(&payload).context("deserialize RaftMessage")?;
Ok(msg)
}
@@ -180,24 +192,26 @@ async fn handle_connection(
) -> Result<()> {
// Read sender header: [4-byte len][node_id bytes]
let mut hlen_buf = [0u8; 4];
- stream.read_exact(&mut hlen_buf).await.context("read sender header length")?;
+ stream
+ .read_exact(&mut hlen_buf)
+ .await
+ .context("read sender header length")?;
let hlen = u32::from_be_bytes(hlen_buf) as usize;
if hlen > 256 {
anyhow::bail!("sender header too large: {} bytes", hlen);
}
let mut hbuf = vec![0u8; hlen];
- stream.read_exact(&mut hbuf).await.context("read sender header")?;
+ stream
+ .read_exact(&mut hbuf)
+ .await
+ .context("read sender header")?;
let from = String::from_utf8(hbuf).context("decode sender node_id")?;
// Read all message frames from this connection.
- loop {
- match read_frame(&mut stream).await {
- Ok(msg) => {
- if tx.send((from.clone(), msg)).await.is_err() {
- break; // receiver dropped
- }
- }
- Err(_) => break, // EOF or parse error
+ // Loop exits on EOF or parse error (read_frame returns Err).
+ while let Ok(msg) = read_frame(&mut stream).await {
+ if tx.send((from.clone(), msg)).await.is_err() {
+ break; // receiver dropped
}
}
diff --git a/engine/crates/xerj-cluster/tests/region_tests.rs b/engine/crates/xerj-cluster/tests/region_tests.rs
index 86074ff70..ab3dc8c42 100644
--- a/engine/crates/xerj-cluster/tests/region_tests.rs
+++ b/engine/crates/xerj-cluster/tests/region_tests.rs
@@ -3,7 +3,7 @@
use std::collections::HashMap;
use xerj_cluster::regions::{Region, RegionManager};
-use xerj_cluster::router::{ShardRouter, jump_hash};
+use xerj_cluster::router::{jump_hash, ShardRouter};
// ── Helpers ───────────────────────────────────────────────────────────────────
@@ -24,7 +24,11 @@ fn test_region_split() {
// Push it over the size threshold.
mgr.update_region_stats(id, 600, 10); // 600 bytes > 500 threshold
- assert_eq!(mgr.regions_to_split().len(), 1, "one region should need splitting");
+ assert_eq!(
+ mgr.regions_to_split().len(),
+ 1,
+ "one region should need splitting"
+ );
let (left, right) = mgr.split_region(id).expect("split should succeed");
@@ -35,8 +39,14 @@ fn test_region_split() {
);
// Two new regions now exist.
- assert!(mgr.regions().iter().any(|r| r.id == left.id), "left half should exist");
- assert!(mgr.regions().iter().any(|r| r.id == right.id), "right half should exist");
+ assert!(
+ mgr.regions().iter().any(|r| r.id == left.id),
+ "left half should exist"
+ );
+ assert!(
+ mgr.regions().iter().any(|r| r.id == right.id),
+ "right half should exist"
+ );
// Key ranges are contiguous.
assert_eq!(
@@ -45,7 +55,11 @@ fn test_region_split() {
);
// Stats are approximately halved.
- assert_eq!(left.size_bytes + right.size_bytes, 600, "stats should sum to original");
+ assert_eq!(
+ left.size_bytes + right.size_bytes,
+ 600,
+ "stats should sum to original"
+ );
assert_eq!(left.doc_count + right.doc_count, 10);
}
@@ -97,14 +111,23 @@ fn test_region_merge() {
let merged = mgr.merge_regions(id_a, id_b).expect("merge should succeed");
// Original regions removed.
- assert!(mgr.regions().iter().all(|r| r.id != id_a), "region a should be removed");
- assert!(mgr.regions().iter().all(|r| r.id != id_b), "region b should be removed");
+ assert!(
+ mgr.regions().iter().all(|r| r.id != id_a),
+ "region a should be removed"
+ );
+ assert!(
+ mgr.regions().iter().all(|r| r.id != id_b),
+ "region b should be removed"
+ );
// Merged region exists.
assert!(mgr.regions().iter().any(|r| r.id == merged.id));
// The merged region spans the full key range.
- assert_eq!(merged.start_key, "", "merged start should be left-unbounded");
+ assert_eq!(
+ merged.start_key, "",
+ "merged start should be left-unbounded"
+ );
assert_eq!(merged.end_key, "", "merged end should be right-unbounded");
// Stats are summed (both were 0 here).
@@ -121,7 +144,10 @@ fn test_region_merge_non_adjacent_fails() {
let id_last = regions[2].id;
let result = mgr.merge_regions(id_first, id_last);
- assert!(result.is_err(), "merging non-adjacent regions should return Err");
+ assert!(
+ result.is_err(),
+ "merging non-adjacent regions should return Err"
+ );
}
/// Merging accumulates stats from both regions.
@@ -153,7 +179,10 @@ fn test_initial_region_creation() {
assert_eq!(mgr.regions().len(), 6);
// First region is left-unbounded.
- assert_eq!(created[0].start_key, "", "first region start should be empty");
+ assert_eq!(
+ created[0].start_key, "",
+ "first region start should be empty"
+ );
// Last region is right-unbounded.
assert_eq!(created[5].end_key, "", "last region end should be empty");
@@ -233,7 +262,10 @@ fn test_rebalance_plan() {
assert!(!moves.is_empty(), "should plan at least one move");
for mv in &moves {
- assert_eq!(mv.from_node, "n1", "moves should come from the overloaded node");
+ assert_eq!(
+ mv.from_node, "n1",
+ "moves should come from the overloaded node"
+ );
assert_eq!(mv.to_node, "n2", "moves should go to the underloaded node");
}
}
diff --git a/engine/crates/xerj-cluster/tests/replication_tests.rs b/engine/crates/xerj-cluster/tests/replication_tests.rs
index d5f386bf5..b11928787 100644
--- a/engine/crates/xerj-cluster/tests/replication_tests.rs
+++ b/engine/crates/xerj-cluster/tests/replication_tests.rs
@@ -33,7 +33,9 @@ impl ClusterTransport for PartialTransport {
if self.reachable.contains(to) {
Ok(())
} else {
- Err(anyhow::anyhow!("node {to} is unreachable (simulated failure)"))
+ Err(anyhow::anyhow!(
+ "node {to} is unreachable (simulated failure)"
+ ))
}
}
@@ -56,11 +58,7 @@ fn make_entry(index: &str, shard: u32, seq_no: u64) -> ReplicationEntry {
}
}
-fn router_with_replicas(
- index: &str,
- primary: &str,
- replicas: &[&str],
-) -> Arc<ShardRouter> {
+fn router_with_replicas(index: &str, primary: &str, replicas: &[&str]) -> Arc<ShardRouter> {
let mut router = ShardRouter::new(8);
router.assign(index, 0, primary);
for rep in replicas {
@@ -125,8 +123,7 @@ async fn test_sync_replication() {
#[tokio::test]
async fn test_sync_replication_insufficient_replicas() {
// Only replica-1 is reachable; replica-2 will return an error.
- let transport: Arc<dyn ClusterTransport> =
- Arc::new(PartialTransport::new(&["replica-1"]));
+ let transport: Arc<dyn ClusterTransport> = Arc::new(PartialTransport::new(&["replica-1"]));
let router = router_with_replicas("idx", "primary", &["replica-1", "replica-2"]);
let replicator = WalReplicator::new(
@@ -142,7 +139,10 @@ async fn test_sync_replication_insufficient_replicas() {
.expect_err("should fail with insufficient replicas");
assert!(
- matches!(err, ReplicationError::InsufficientReplicas { needed: 2, got: 1 }),
+ matches!(
+ err,
+ ReplicationError::InsufficientReplicas { needed: 2, got: 1 }
+ ),
"unexpected error: {err}"
);
}
@@ -183,8 +183,7 @@ async fn test_quorum_replication() {
#[tokio::test]
async fn test_quorum_replication_fails_below_majority() {
// Only 1 of 3 replicas is reachable.
- let transport: Arc<dyn ClusterTransport> =
- Arc::new(PartialTransport::new(&["replica-1"]));
+ let transport: Arc<dyn ClusterTransport> = Arc::new(PartialTransport::new(&["replica-1"]));
let mut router = ShardRouter::new(8);
router.assign("idx", 0, "primary");
@@ -248,8 +247,7 @@ fn test_replication_entry_serialization() {
for entry in &entries {
let json = serde_json::to_string(entry).expect("serialize entry");
- let decoded: ReplicationEntry =
- serde_json::from_str(&json).expect("deserialize entry");
+ let decoded: ReplicationEntry = serde_json::from_str(&json).expect("deserialize entry");
assert_eq!(entry.index, decoded.index, "index mismatch");
assert_eq!(entry.shard, decoded.shard, "shard mismatch");
@@ -257,8 +255,14 @@ fn test_replication_entry_serialization() {
match (&entry.operation, &decoded.operation) {
(
- ReplicationOp::Index { doc_id: a, source_json: sa },
- ReplicationOp::Index { doc_id: b, source_json: sb },
+ ReplicationOp::Index {
+ doc_id: a,
+ source_json: sa,
+ },
+ ReplicationOp::Index {
+ doc_id: b,
+ source_json: sb,
+ },
) => {
assert_eq!(a, b, "doc_id mismatch in Index");
assert_eq!(sa, sb, "source_json mismatch");
@@ -310,6 +314,9 @@ async fn test_replication_no_replicas() {
"primary".to_string(),
);
let result = replicator.replicate(make_entry("solo", 0, 1)).await;
- assert!(result.is_ok(), "no-replica {mode:?} should succeed: {result:?}");
+ assert!(
+ result.is_ok(),
+ "no-replica {mode:?} should succeed: {result:?}"
+ );
}
}
diff --git a/engine/crates/xerj-cluster/tests/transport_tests.rs b/engine/crates/xerj-cluster/tests/transport_tests.rs
index db11a7937..68323fd3d 100644
--- a/engine/crates/xerj-cluster/tests/transport_tests.rs
+++ b/engine/crates/xerj-cluster/tests/transport_tests.rs
@@ -7,12 +7,12 @@ use std::time::Duration;
use tokio::sync::watch;
use xerj_cluster::node::{
- ClusterNode,
in_memory::{InMemoryBus, InMemoryTransport},
+ ClusterNode,
};
use xerj_cluster::raft::{ClusterCommand, RaftMessage};
use xerj_cluster::runner::ClusterRunner;
-use xerj_cluster::search::{SearchHit, SearchMessage, merge_search_responses};
+use xerj_cluster::search::{merge_search_responses, SearchHit, SearchMessage};
use xerj_cluster::transport::TcpTransport;
// ── Helper: find a free port ──────────────────────────────────────────────────
@@ -75,7 +75,9 @@ async fn test_tcp_transport_send_recv() {
assert_eq!(from, "node-a");
match received {
- RaftMessage::RequestVote { term, candidate_id, .. } => {
+ RaftMessage::RequestVote {
+ term, candidate_id, ..
+ } => {
assert_eq!(term, 1);
assert_eq!(candidate_id, "node-a");
}
@@ -124,7 +126,9 @@ async fn test_cluster_runner_election() {
.map(|mut r| {
tokio::spawn(async move {
// Run for at most 2 seconds, then return the runner for inspection.
- tokio::time::timeout(Duration::from_secs(2), r.run()).await.ok();
+ tokio::time::timeout(Duration::from_secs(2), r.run())
+ .await
+ .ok();
r
})
})
@@ -201,7 +205,8 @@ async fn test_distributed_search_message() {
let json = serde_json::to_string(msg).expect("serialize");
let decoded: SearchMessage = serde_json::from_str(&json).expect("deserialize");
assert_eq!(
- msg, &decoded,
+ msg,
+ &decoded,
"round-trip failed for {:?}",
std::mem::discriminant(msg)
);
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index 30aac21c9..f66fc1fc3 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -43,7 +43,7 @@ use crate::error::XerjError;
///
/// Fields are grouped into sub-structs by concern. All fields implement
/// `Default` so that an empty config file (or no file at all) is valid.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Config {
/// Network and data directory settings — 5 settings.
@@ -79,28 +79,9 @@ pub struct Config {
}
// Total: 5+2+3+10+5+3+1+6+2+4+3+4+3 = 51 fields
-
-impl Default for Config {
- fn default() -> Self {
- Self {
- server: ServerConfig::default(),
- auth: AuthConfig::default(),
- tls: TlsConfig::default(),
- storage: StorageConfig::default(),
- merge: MergeConfig::default(),
- compression: CompressionConfig::default(),
- fts: FtsConfig::default(),
- vector: VectorConfig::default(),
- logs: LogsConfig::default(),
- embedding: EmbeddingConfig::default(),
- limits: LimitsConfig::default(),
- indexing: IndexingConfig::default(),
- engine: EngineConfig::default(),
- cluster: ClusterConfig::default(),
- pit: PitConfig::default(),
- }
- }
-}
+// `Default` is derived — every field is a sub-config that implements
+// `Default`, so the derive produces exactly the same all-defaults value
+// the manual impl used to build by hand.
impl Config {
/// Load configuration from a TOML file.
@@ -119,7 +100,11 @@ impl Config {
}
/// Load configuration from a TOML string (useful for testing).
- pub fn from_str(s: &str) -> Result<Self, XerjError> {
+ ///
+ /// Named `from_toml_str` rather than `from_str` to avoid shadowing the
+ /// `std::str::FromStr::from_str` convention (clippy::should_implement_trait):
+ /// this parses TOML specifically and also runs cross-field validation.
+ pub fn from_toml_str(s: &str) -> Result<Self, XerjError> {
let config: Config = toml::from_str(s)?;
config.validate()?;
Ok(config)
@@ -279,7 +264,11 @@ impl Default for AuthConfig {
/// TLS settings.
///
/// **3 settings.**
-#[derive(Debug, Clone, Serialize, Deserialize)]
+///
+/// Defaults are derived: TLS is disabled (`enabled: false`) with empty
+/// cert/key paths so the engine starts out of the box; enable it in
+/// production by setting `cert_path` + `key_path`.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct TlsConfig {
/// Enable TLS on all listeners (default: `false` — enable in production).
@@ -290,18 +279,6 @@ pub struct TlsConfig {
pub key_path: String,
}
-impl Default for TlsConfig {
- fn default() -> Self {
- Self {
- // Default to disabled so the engine starts out of the box.
- // Enable in production by setting cert_path and key_path.
- enabled: false,
- cert_path: String::new(),
- key_path: String::new(),
- }
- }
-}
-
// ─────────────────────────────────────────────────────────────────────────────
/// Write-ahead log, flush, and object-store settings.
@@ -326,7 +303,6 @@ pub struct StorageConfig {
pub flush_interval_secs: u64,
// ── Object-store backend (compute-storage separation) ─────────────────────
-
/// Storage backend: `"local"` or `"s3"` (default: `"local"`).
///
/// When set to `"s3"`, flushed segments are written to the configured S3
@@ -882,12 +858,14 @@ mod tests {
#[test]
fn default_config_is_valid() {
- Config::default().validate().expect("default config should be valid");
+ Config::default()
+ .validate()
+ .expect("default config should be valid");
}
#[test]
fn parse_minimal_toml() {
- let cfg = Config::from_str(
+ let cfg = Config::from_toml_str(
r#"
[server]
rest_port = 9000
@@ -901,7 +879,7 @@ mod tests {
#[test]
fn duplicate_ports_rejected() {
- let result = Config::from_str(
+ let result = Config::from_toml_str(
r#"
[server]
rest_port = 9200
@@ -913,7 +891,7 @@ mod tests {
#[test]
fn tls_enabled_requires_paths() {
- let result = Config::from_str(
+ let result = Config::from_toml_str(
r#"
[tls]
enabled = true
@@ -926,7 +904,7 @@ mod tests {
#[test]
fn tls_disabled_no_paths_ok() {
- let cfg = Config::from_str(
+ let cfg = Config::from_toml_str(
r#"
[tls]
enabled = false
diff --git a/engine/crates/xerj-common/src/error.rs b/engine/crates/xerj-common/src/error.rs
index 8a5662274..2e7fe4363 100644
--- a/engine/crates/xerj-common/src/error.rs
+++ b/engine/crates/xerj-common/src/error.rs
@@ -82,13 +82,23 @@ pub enum XerjError {
// ── Optimistic concurrency ─────────────────────────────────────────────
/// Optimistic concurrency control check failed (if_seq_no / if_primary_term).
- #[error("version conflict on document [{id}]: expected seq_no={expected}, actual seq_no={actual}")]
- VersionConflict { id: String, expected: u64, actual: u64 },
+ #[error(
+ "version conflict on document [{id}]: expected seq_no={expected}, actual seq_no={actual}"
+ )]
+ VersionConflict {
+ id: String,
+ expected: u64,
+ actual: u64,
+ },
// ── Result window ─────────────────────────────────────────────────────
/// The requested from + size exceeds the max_result_window limit.
#[error("result window is too large: from + size > max_result_window={max_result_window}; use search_after for deep pagination")]
- ResultWindowTooLarge { from: usize, size: usize, max_result_window: usize },
+ ResultWindowTooLarge {
+ from: usize,
+ size: usize,
+ max_result_window: usize,
+ },
// ── Index blocks ──────────────────────────────────────────────────────
/// Operation rejected because an index block is set.
@@ -194,11 +204,19 @@ impl XerjError {
}
pub fn version_conflict(id: impl Into<String>, expected: u64, actual: u64) -> Self {
- Self::VersionConflict { id: id.into(), expected, actual }
+ Self::VersionConflict {
+ id: id.into(),
+ expected,
+ actual,
+ }
}
pub fn result_window_too_large(from: usize, size: usize, max_result_window: usize) -> Self {
- Self::ResultWindowTooLarge { from, size, max_result_window }
+ Self::ResultWindowTooLarge {
+ from,
+ size,
+ max_result_window,
+ }
}
pub fn index_blocked(index: impl Into<String>, block_type: impl Into<String>) -> Self {
diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rs
index 3fe5f9406..7964a1eea 100644
--- a/engine/crates/xerj-common/src/lib.rs
+++ b/engine/crates/xerj-common/src/lib.rs
@@ -28,9 +28,7 @@ pub mod types;
// Convenience re-exports at the crate root
pub use config::Config;
pub use error::XerjError;
-pub use types::{
- DocId, Document, FieldConfig, FieldType, IndexName, Schema, SegmentId, SeqNo,
-};
+pub use types::{DocId, Document, FieldConfig, FieldType, IndexName, Schema, SegmentId, SeqNo};
/// Crate-level result alias — uses [`XerjError`] as the error type.
pub type Result<T> = std::result::Result<T, XerjError>;
diff --git a/engine/crates/xerj-common/src/metrics.rs b/engine/crates/xerj-common/src/metrics.rs
index e30f77897..c74486d2b 100644
--- a/engine/crates/xerj-common/src/metrics.rs
+++ b/engine/crates/xerj-common/src/metrics.rs
@@ -295,8 +295,7 @@ impl Metrics {
encoder
.encode(&self.gather(), &mut buf)
.map_err(|e| XerjError::internal(format!("metrics encode: {e}")))?;
- String::from_utf8(buf)
- .map_err(|e| XerjError::internal(format!("metrics utf8: {e}")))
+ String::from_utf8(buf).map_err(|e| XerjError::internal(format!("metrics utf8: {e}")))
}
// ── Ergonomic recording helpers ───────────────────────────────────────────
diff --git a/engine/crates/xerj-common/src/schema.rs b/engine/crates/xerj-common/src/schema.rs
index cc490801f..653944cf3 100644
--- a/engine/crates/xerj-common/src/schema.rs
+++ b/engine/crates/xerj-common/src/schema.rs
@@ -339,9 +339,12 @@ mod tests {
#[test]
fn infer_types_from_json() {
- assert_eq!(infer_field_type(&serde_json::json!(true)), FieldType::Boolean);
+ assert_eq!(
+ infer_field_type(&serde_json::json!(true)),
+ FieldType::Boolean
+ );
assert_eq!(infer_field_type(&serde_json::json!(42)), FieldType::Long);
- assert_eq!(infer_field_type(&serde_json::json!(3.14)), FieldType::Double);
+ assert_eq!(infer_field_type(&serde_json::json!(2.5)), FieldType::Double);
assert_eq!(
infer_field_type(&serde_json::json!("hello world")),
FieldType::Text
@@ -381,7 +384,8 @@ mod tests {
#[test]
fn strict_mapping_rejects_unknown() {
let mut ms = ManagedSchema::strict();
- ms.add_field(FieldConfig::new("known", FieldType::Text)).unwrap();
+ ms.add_field(FieldConfig::new("known", FieldType::Text))
+ .unwrap();
let doc = serde_json::json!({"known": "ok", "unknown": "boom"});
assert!(ms.apply_document(&doc, 500).is_err());
@@ -399,10 +403,12 @@ mod tests {
#[test]
fn schema_compatibility_type_change_rejected() {
let mut old = ManagedSchema::dynamic();
- old.add_field(FieldConfig::new("status", FieldType::Keyword)).unwrap();
+ old.add_field(FieldConfig::new("status", FieldType::Keyword))
+ .unwrap();
let mut new = ManagedSchema::dynamic();
- new.add_field(FieldConfig::new("status", FieldType::Long)).unwrap();
+ new.add_field(FieldConfig::new("status", FieldType::Long))
+ .unwrap();
assert!(old.is_compatible_with(&new).is_err());
}
diff --git a/engine/crates/xerj-common/src/types.rs b/engine/crates/xerj-common/src/types.rs
index 0ffc27e67..671625b67 100644
--- a/engine/crates/xerj-common/src/types.rs
+++ b/engine/crates/xerj-common/src/types.rs
@@ -348,8 +348,7 @@ impl FieldConfig {
/// Returns `true` if this field can be used in sorting and aggregations.
pub fn is_aggregatable(&self) -> bool {
- self.options.doc_values
- && !matches!(self.field_type, FieldType::Text | FieldType::Binary)
+ self.options.doc_values && !matches!(self.field_type, FieldType::Text | FieldType::Binary)
}
}
@@ -512,17 +511,19 @@ mod tests {
#[test]
fn index_name_invalid() {
- assert!(IndexName::new("").is_err()); // empty
- assert!(IndexName::new("MyIndex").is_err()); // uppercase
- assert!(IndexName::new("_private").is_err()); // starts with _
- assert!(IndexName::new("bad name").is_err()); // space
- assert!(IndexName::new("1bad").is_err()); // starts with digit
+ assert!(IndexName::new("").is_err()); // empty
+ assert!(IndexName::new("MyIndex").is_err()); // uppercase
+ assert!(IndexName::new("_private").is_err()); // starts with _
+ assert!(IndexName::new("bad name").is_err()); // space
+ assert!(IndexName::new("1bad").is_err()); // starts with digit
}
#[test]
fn schema_field_add_dedup() {
let mut schema = Schema::empty();
- schema.add_field(FieldConfig::new("title", FieldType::Text)).unwrap();
+ schema
+ .add_field(FieldConfig::new("title", FieldType::Text))
+ .unwrap();
assert_eq!(schema.version, 1);
let result = schema.add_field(FieldConfig::new("title", FieldType::Keyword));
@@ -537,7 +538,10 @@ mod tests {
let idx = IndexName::new("test").unwrap();
let doc = Document::new(idx, source);
- assert_eq!(doc.get_field("user.name"), Some(&serde_json::json!("Alice")));
+ assert_eq!(
+ doc.get_field("user.name"),
+ Some(&serde_json::json!("Alice"))
+ );
assert_eq!(doc.get_field("user.missing"), None);
assert_eq!(doc.get_field("missing"), None);
}
diff --git a/engine/crates/xerj-compress/src/block.rs b/engine/crates/xerj-compress/src/block.rs
index b66128dcb..a5e7fe8a5 100644
--- a/engine/crates/xerj-compress/src/block.rs
+++ b/engine/crates/xerj-compress/src/block.rs
@@ -228,7 +228,8 @@ impl<'a> BlockReader<'a> {
for _ in 0..header.doc_count {
let len = rdr
.read_u32::<LittleEndian>()
- .map_err(|e| XerjError::internal(format!("doc len read: {e}")))? as usize;
+ .map_err(|e| XerjError::internal(format!("doc len read: {e}")))?
+ as usize;
let mut doc = vec![0u8; len];
rdr.read_exact(&mut doc)
.map_err(|e| XerjError::internal(format!("doc data read: {e}")))?;
@@ -281,16 +282,24 @@ mod tests {
}
#[test]
- fn roundtrip_none_exact_block() { roundtrip(CompressionLevel::None, BLOCK_SIZE); }
+ fn roundtrip_none_exact_block() {
+ roundtrip(CompressionLevel::None, BLOCK_SIZE);
+ }
#[test]
- fn roundtrip_lz4_partial_block() { roundtrip(CompressionLevel::Fast, 37); }
+ fn roundtrip_lz4_partial_block() {
+ roundtrip(CompressionLevel::Fast, 37);
+ }
#[test]
- fn roundtrip_lz4_multiple_blocks() { roundtrip(CompressionLevel::Fast, BLOCK_SIZE * 3 + 5); }
+ fn roundtrip_lz4_multiple_blocks() {
+ roundtrip(CompressionLevel::Fast, BLOCK_SIZE * 3 + 5);
+ }
#[test]
- fn roundtrip_zstd_balanced() { roundtrip(CompressionLevel::Balanced, 200); }
+ fn roundtrip_zstd_balanced() {
+ roundtrip(CompressionLevel::Balanced, 200);
+ }
#[test]
fn empty_finish_produces_no_bytes() {
diff --git a/engine/crates/xerj-compress/src/codec.rs b/engine/crates/xerj-compress/src/codec.rs
index b94014f16..11cbe8052 100644
--- a/engine/crates/xerj-compress/src/codec.rs
+++ b/engine/crates/xerj-compress/src/codec.rs
@@ -43,6 +43,11 @@ pub enum CompressionLevel {
impl CompressionLevel {
/// Parse from a string (case-insensitive).
+ ///
+ /// Named `from_str` for config ergonomics; returns `Option` (not the
+ /// `Result` that `std::str::FromStr` requires), so it deliberately does
+ /// not implement that trait.
+ #[allow(clippy::should_implement_trait)]
pub fn from_str(s: &str) -> Option<Self> {
match s.to_ascii_lowercase().as_str() {
"none" => Some(Self::None),
@@ -200,7 +205,10 @@ mod tests {
fn lz4_reduces_size() {
let codec = Lz4Codec;
let compressed = codec.compress(PAYLOAD).unwrap();
- assert!(compressed.len() < PAYLOAD.len(), "LZ4 should compress repetitive data");
+ assert!(
+ compressed.len() < PAYLOAD.len(),
+ "LZ4 should compress repetitive data"
+ );
}
#[test]
diff --git a/engine/crates/xerj-compress/src/dictionary.rs b/engine/crates/xerj-compress/src/dictionary.rs
index e721a8c83..ba5da080b 100644
--- a/engine/crates/xerj-compress/src/dictionary.rs
+++ b/engine/crates/xerj-compress/src/dictionary.rs
@@ -216,7 +216,7 @@ mod tests {
#[test]
fn encode_decode_roundtrip() {
- let samples = vec!["INFO", "ERROR", "WARN", "INFO", "INFO", "DEBUG", "ERROR"];
+ let samples = ["INFO", "ERROR", "WARN", "INFO", "INFO", "DEBUG", "ERROR"];
let mut encoder = DictionaryEncoder::new();
encoder.build_from_sample(samples.iter().copied());
@@ -230,7 +230,7 @@ mod tests {
#[test]
fn serialize_deserialize() {
- let samples = vec!["alpha", "beta", "gamma", "alpha", "beta", "alpha"];
+ let samples = ["alpha", "beta", "gamma", "alpha", "beta", "alpha"];
let mut encoder = DictionaryEncoder::new();
encoder.build_from_sample(samples.iter().copied());
diff --git a/engine/crates/xerj-compress/src/field_codec.rs b/engine/crates/xerj-compress/src/field_codec.rs
index 61eb834fd..9fb8d4875 100644
--- a/engine/crates/xerj-compress/src/field_codec.rs
+++ b/engine/crates/xerj-compress/src/field_codec.rs
@@ -126,9 +126,7 @@ pub enum FieldEncoding {
/// High-cardinality strings — raw storage.
///
/// Falls back to this when no smarter encoding applies.
- RawString {
- values: Vec<String>,
- },
+ RawString { values: Vec<String> },
/// Booleans — bit-packed (8 values per byte).
Bitpacked {
@@ -161,7 +159,7 @@ impl FieldEncoding {
let n_docs: usize = bitmap.first().map(|b| b.len() * 8).unwrap_or(0);
// 4-bit index per document + bitmap overhead
let bitmap_bytes: usize = bitmap.iter().map(|b| b.len()).sum();
- let ids_bytes = (n_docs + 1) / 2; // 4-bit index per doc
+ let ids_bytes = n_docs.div_ceil(2); // 4-bit index per doc
(bitmap_bytes + ids_bytes, n_docs.max(1))
}
FieldEncoding::DeltaTimestamp { deltas, .. } => {
@@ -176,7 +174,11 @@ impl FieldEncoding {
..
} => {
let ids_bytes = template_ids.len() * 2;
- let var_bytes: usize = variables.iter().flat_map(|v| v.iter()).map(|s| s.len() + 1).sum();
+ let var_bytes: usize = variables
+ .iter()
+ .flat_map(|v| v.iter())
+ .map(|s| s.len() + 1)
+ .sum();
(ids_bytes + var_bytes, template_ids.len().max(1))
}
FieldEncoding::Varint { values } => {
@@ -203,9 +205,7 @@ impl FieldEncoding {
/// Total bytes used by the encoded column data.
pub fn total_bytes(&self) -> usize {
match self {
- FieldEncoding::BitsetEnum { bitmap, .. } => {
- bitmap.iter().map(|b| b.len()).sum()
- }
+ FieldEncoding::BitsetEnum { bitmap, .. } => bitmap.iter().map(|b| b.len()).sum(),
FieldEncoding::DeltaTimestamp { deltas, .. } => {
deltas.iter().map(|d| varint_signed_len(*d)).sum()
}
@@ -217,7 +217,11 @@ impl FieldEncoding {
} => {
let tmpl_bytes: usize = templates.iter().map(|t| t.len()).sum();
let ids_bytes = template_ids.len() * 2;
- let var_bytes: usize = variables.iter().flat_map(|v| v.iter()).map(|s| s.len() + 1).sum();
+ let var_bytes: usize = variables
+ .iter()
+ .flat_map(|v| v.iter())
+ .map(|s| s.len() + 1)
+ .sum();
tmpl_bytes + ids_bytes + var_bytes
}
FieldEncoding::Varint { values } => {
@@ -263,14 +267,14 @@ impl FieldEncoding {
// We don't have the original strings at this point, so we use
// a conservative estimate for the raw baseline based on encoding type.
let raw_bpv = match self {
- FieldEncoding::BitsetEnum { .. } => 3.0, // e.g. "200", "GET"
+ FieldEncoding::BitsetEnum { .. } => 3.0, // e.g. "200", "GET"
FieldEncoding::DeltaTimestamp { .. } => 25.0, // typical timestamp string
- FieldEncoding::PackedIp { .. } => 14.0, // "192.168.100.200" avg
+ FieldEncoding::PackedIp { .. } => 14.0, // "192.168.100.200" avg
FieldEncoding::UrlTemplate { .. } => 30.0, // typical URL path
- FieldEncoding::Varint { .. } => 6.0, // content-length as string
- FieldEncoding::Dictionary { .. } => 12.0, // hostname / service name
- FieldEncoding::RawString { .. } => bpv, // already raw
- FieldEncoding::Bitpacked { .. } => 4.5, // "true" / "false"
+ FieldEncoding::Varint { .. } => 6.0, // content-length as string
+ FieldEncoding::Dictionary { .. } => 12.0, // hostname / service name
+ FieldEncoding::RawString { .. } => bpv, // already raw
+ FieldEncoding::Bitpacked { .. } => 4.5, // "true" / "false"
FieldEncoding::FixedPrecision { .. } => 6.0, // "12.34"
};
raw_bpv / bpv
@@ -421,7 +425,7 @@ impl FieldAnalyzer {
.collect();
let n_docs = values.len();
- let bytes_per_bitmap = (n_docs + 7) / 8;
+ let bytes_per_bitmap = n_docs.div_ceil(8);
// One bitmap per dictionary entry
let mut bitmap: Vec<Vec<u8>> = vec![vec![0u8; bytes_per_bitmap]; dict.len()];
@@ -465,10 +469,7 @@ impl FieldAnalyzer {
let base_us = parsed_us[0];
// First pass: absolute → delta
- let deltas_abs: Vec<i64> = parsed_us
- .windows(2)
- .map(|w| w[1] - w[0])
- .collect();
+ let deltas_abs: Vec<i64> = parsed_us.windows(2).map(|w| w[1] - w[0]).collect();
// Second pass: delta → delta-of-delta
let mut deltas: Vec<i64> = Vec::with_capacity(deltas_abs.len());
@@ -488,10 +489,7 @@ impl FieldAnalyzer {
/// Build a [`FieldEncoding::PackedIp`] from a column of IPv4 strings.
pub fn encode_as_packed_ip(&self, values: &[&str]) -> FieldEncoding {
- let packed: Vec<u32> = values
- .iter()
- .map(|v| parse_ipv4(v).unwrap_or(0))
- .collect();
+ let packed: Vec<u32> = values.iter().map(|v| parse_ipv4(v).unwrap_or(0)).collect();
FieldEncoding::PackedIp { values: packed }
}
@@ -556,7 +554,7 @@ impl FieldAnalyzer {
/// Build a [`FieldEncoding::Bitpacked`] from a column of boolean strings.
pub fn encode_as_bitpacked(&self, values: &[&str]) -> FieldEncoding {
let count = values.len();
- let mut bits = vec![0u8; (count + 7) / 8];
+ let mut bits = vec![0u8; count.div_ceil(8)];
for (i, v) in values.iter().enumerate() {
let is_true = *v == "true" || *v == "1";
if is_true {
@@ -622,7 +620,10 @@ enum TimestampLikelihood {
/// Classify a single value for timestamp likelihood without full parsing.
fn detect_timestamp_format_single(v: &str) -> TimestampLikelihood {
// ISO 8601: starts with 4 digits + '-'
- if v.len() >= 10 && v.as_bytes().iter().take(4).all(|b| b.is_ascii_digit()) && v.as_bytes().get(4) == Some(&b'-') {
+ if v.len() >= 10
+ && v.as_bytes().iter().take(4).all(|b| b.is_ascii_digit())
+ && v.as_bytes().get(4) == Some(&b'-')
+ {
return TimestampLikelihood::Likely;
}
// Apache common: starts with '['
@@ -778,7 +779,7 @@ fn parse_nginx_us(v: &str) -> Option<i64> {
/// Days since Unix epoch (1970-01-01) using the proleptic Gregorian calendar.
fn days_since_epoch(year: i64, month: i64, day: i64) -> Option<i64> {
- if !(1 <= month && month <= 12 && 1 <= day && day <= 31) {
+ if !((1..=12).contains(&month) && (1..=31).contains(&day)) {
return None;
}
// Rata Die algorithm
@@ -932,7 +933,10 @@ fn is_uuid_like(s: &str) -> bool {
if bytes.len() != 36 {
return false;
}
- bytes[8] == b'-' && bytes[13] == b'-' && bytes[18] == b'-' && bytes[23] == b'-'
+ bytes[8] == b'-'
+ && bytes[13] == b'-'
+ && bytes[18] == b'-'
+ && bytes[23] == b'-'
&& bytes.iter().enumerate().all(|(i, &b)| {
if i == 8 || i == 13 || i == 18 || i == 23 {
b == b'-'
@@ -1003,7 +1007,10 @@ mod tests {
#[test]
fn test_parse_ipv4() {
- assert_eq!(parse_ipv4("192.168.1.1"), Some((192 << 24) | (168 << 16) | (1 << 8) | 1));
+ assert_eq!(
+ parse_ipv4("192.168.1.1"),
+ Some((192 << 24) | (168 << 16) | (1 << 8) | 1)
+ );
assert_eq!(parse_ipv4("0.0.0.0"), Some(0));
assert_eq!(parse_ipv4("255.255.255.255"), Some(u32::MAX));
assert!(parse_ipv4("not-an-ip").is_none());
@@ -1089,7 +1096,7 @@ mod tests {
let statuses = vec![
"200", "200", "404", "500", "200", "301", "404", "200", "403", "200",
];
- let refs: Vec<&str> = statuses.iter().copied().collect();
+ let refs: Vec<&str> = statuses.to_vec();
let enc = analyzer.analyze("status", &refs);
match &enc {
@@ -1102,7 +1109,10 @@ mod tests {
// doc 0 → byte 0 bit 0
assert!(bmap[0] & 1 != 0);
}
- other => panic!("expected BitsetEnum, got {:?}", std::mem::discriminant(other)),
+ other => panic!(
+ "expected BitsetEnum, got {:?}",
+ std::mem::discriminant(other)
+ ),
}
// bytes_per_value should be well below 1.0 for 10 docs / small cardinality
@@ -1117,8 +1127,8 @@ mod tests {
#[test]
fn test_http_method_encoding() {
let analyzer = FieldAnalyzer::new(1024);
- let methods = vec!["GET", "POST", "GET", "GET", "DELETE", "PUT", "GET", "POST"];
- let refs: Vec<&str> = methods.iter().copied().collect();
+ let methods = ["GET", "POST", "GET", "GET", "DELETE", "PUT", "GET", "POST"];
+ let refs: Vec<&str> = methods.to_vec();
let enc = analyzer.analyze("method", &refs);
assert!(matches!(enc, FieldEncoding::BitsetEnum { .. }));
assert!(enc.bytes_per_value() < 2.0);
@@ -1127,8 +1137,8 @@ mod tests {
#[test]
fn test_log_level_encoding() {
let analyzer = FieldAnalyzer::new(1024);
- let levels = vec!["INFO", "ERROR", "WARN", "INFO", "DEBUG", "INFO", "ERROR"];
- let refs: Vec<&str> = levels.iter().copied().collect();
+ let levels = ["INFO", "ERROR", "WARN", "INFO", "DEBUG", "INFO", "ERROR"];
+ let refs: Vec<&str> = levels.to_vec();
let enc = analyzer.analyze("level", &refs);
assert!(matches!(enc, FieldEncoding::BitsetEnum { .. }));
println!(
@@ -1141,14 +1151,14 @@ mod tests {
#[test]
fn test_timestamp_iso8601() {
let analyzer = FieldAnalyzer::new(1024);
- let timestamps = vec![
+ let timestamps = [
"2024-01-15T12:34:56Z",
"2024-01-15T12:34:57Z",
"2024-01-15T12:34:58Z",
"2024-01-15T12:34:59Z",
"2024-01-15T12:35:00Z",
];
- let refs: Vec<&str> = timestamps.iter().copied().collect();
+ let refs: Vec<&str> = timestamps.to_vec();
let enc = analyzer.analyze("@timestamp", &refs);
match &enc {
@@ -1164,7 +1174,10 @@ mod tests {
assert_eq!(deltas.len(), 4); // N-1 values encoded
println!("timestamp deltas: {:?}", deltas);
}
- other => panic!("expected DeltaTimestamp, got {:?}", std::mem::discriminant(other)),
+ other => panic!(
+ "expected DeltaTimestamp, got {:?}",
+ std::mem::discriminant(other)
+ ),
}
println!(
@@ -1177,27 +1190,33 @@ mod tests {
#[test]
fn test_timestamp_apache() {
let analyzer = FieldAnalyzer::new(1024);
- let timestamps = vec![
+ let timestamps = [
"[10/Jan/2024:12:34:56 +0000]",
"[10/Jan/2024:12:34:57 +0000]",
"[10/Jan/2024:12:35:00 +0000]",
];
- let refs: Vec<&str> = timestamps.iter().copied().collect();
+ let refs: Vec<&str> = timestamps.to_vec();
let enc = analyzer.analyze("@timestamp", &refs);
- assert!(matches!(enc, FieldEncoding::DeltaTimestamp { format: TimestampFormat::ApacheCommon, .. }));
+ assert!(matches!(
+ enc,
+ FieldEncoding::DeltaTimestamp {
+ format: TimestampFormat::ApacheCommon,
+ ..
+ }
+ ));
}
#[test]
fn test_ip_encoding() {
let analyzer = FieldAnalyzer::new(1024);
- let ips = vec![
+ let ips = [
"192.168.1.1",
"10.0.0.1",
"172.16.0.50",
"192.168.1.2",
"8.8.8.8",
];
- let refs: Vec<&str> = ips.iter().copied().collect();
+ let refs: Vec<&str> = ips.to_vec();
let enc = analyzer.analyze("client_ip", &refs);
match &enc {
@@ -1219,7 +1238,7 @@ mod tests {
#[test]
fn test_url_template_encoding() {
let analyzer = FieldAnalyzer::new(1024);
- let paths = vec![
+ let paths = [
"/api/users/1",
"/api/users/2",
"/api/users/3",
@@ -1227,7 +1246,7 @@ mod tests {
"/static/js/app.js",
"/static/css/main.css",
];
- let refs: Vec<&str> = paths.iter().copied().collect();
+ let refs: Vec<&str> = paths.to_vec();
let enc = analyzer.analyze("path", &refs);
match &enc {
@@ -1236,12 +1255,18 @@ mod tests {
template_ids,
variables,
} => {
- assert!(templates.len() < paths.len(), "should deduplicate templates");
+ assert!(
+ templates.len() < paths.len(),
+ "should deduplicate templates"
+ );
assert_eq!(template_ids.len(), paths.len());
assert_eq!(variables.len(), paths.len());
println!("URL templates: {:?}", templates);
}
- other => panic!("expected UrlTemplate, got {:?}", std::mem::discriminant(other)),
+ other => panic!(
+ "expected UrlTemplate, got {:?}",
+ std::mem::discriminant(other)
+ ),
}
println!(
@@ -1254,8 +1279,8 @@ mod tests {
#[test]
fn test_varint_encoding() {
let analyzer = FieldAnalyzer::new(1024);
- let sizes = vec!["1024", "2048", "512", "65536", "128", "256"];
- let refs: Vec<&str> = sizes.iter().copied().collect();
+ let sizes = ["1024", "2048", "512", "65536", "128", "256"];
+ let refs: Vec<&str> = sizes.to_vec();
let enc = analyzer.analyze("content_length", &refs);
assert!(matches!(enc, FieldEncoding::Varint { .. }));
@@ -1270,11 +1295,14 @@ mod tests {
fn test_boolean_encoding() {
let analyzer = FieldAnalyzer::new(1024);
// 2 unique values → BitsetEnum wins (most compact for tiny cardinality)
- let bools: Vec<&str> = (0..64).map(|i| if i % 3 == 0 { "true" } else { "false" }).collect();
+ let bools: Vec<&str> = (0..64)
+ .map(|i| if i % 3 == 0 { "true" } else { "false" })
+ .collect();
let enc = analyzer.analyze("is_active", &bools);
// Either BitsetEnum or Bitpacked is correct — both are compact
assert!(
- matches!(enc, FieldEncoding::Bitpacked { .. }) || matches!(enc, FieldEncoding::BitsetEnum { .. }),
+ matches!(enc, FieldEncoding::Bitpacked { .. })
+ || matches!(enc, FieldEncoding::BitsetEnum { .. }),
"expected compact bool encoding"
);
assert!(enc.bytes_per_value() < 1.0);
@@ -1295,7 +1323,10 @@ mod tests {
assert_eq!(values[0], 34);
assert_eq!(values.len(), 20);
}
- other => panic!("expected FixedPrecision, got {:?}", std::mem::discriminant(other)),
+ other => panic!(
+ "expected FixedPrecision, got {:?}",
+ std::mem::discriminant(other)
+ ),
}
}
@@ -1315,7 +1346,9 @@ mod tests {
fn test_raw_string_fallback() {
let analyzer = FieldAnalyzer::new(1024);
// High cardinality, non-structured strings
- let msgs: Vec<String> = (0..500).map(|i| format!("Unique log message number {i} with some context")).collect();
+ let msgs: Vec<String> = (0..500)
+ .map(|i| format!("Unique log message number {i} with some context"))
+ .collect();
let refs: Vec<&str> = msgs.iter().map(String::as_str).collect();
let enc = analyzer.analyze("message", &refs);
assert!(matches!(enc, FieldEncoding::RawString { .. }));
diff --git a/engine/crates/xerj-compress/src/lib.rs b/engine/crates/xerj-compress/src/lib.rs
index 507b062f0..b9616494a 100644
--- a/engine/crates/xerj-compress/src/lib.rs
+++ b/engine/crates/xerj-compress/src/lib.rs
@@ -13,8 +13,8 @@ pub mod codec;
pub mod dictionary;
pub mod field_codec;
-pub use codec::{get_codec, Codec, CompressionLevel, Lz4Codec, NoneCodec, ZstdCodec};
pub use block::{BlockReader, BlockWriter};
+pub use codec::{get_codec, Codec, CompressionLevel, Lz4Codec, NoneCodec, ZstdCodec};
pub use dictionary::{DictionaryDecoder, DictionaryEncoder};
pub use field_codec::{FieldAnalyzer, FieldEncoding, TimestampFormat};
diff --git a/engine/crates/xerj-console-api/build.rs b/engine/crates/xerj-console-api/build.rs
index 8ca1b1c37..bf0e7a7be 100644
--- a/engine/crates/xerj-console-api/build.rs
+++ b/engine/crates/xerj-console-api/build.rs
@@ -24,9 +24,9 @@ use std::path::{Path, PathBuf};
// works from the repo-root layout and from an engine-worktree layout
// (engine/ checked out separately, with the repo root alongside).
const UX_RELS: &[&str] = &[
- "../../../xerj-ux", // engine/crates/api → repo root → xerj-ux
- "../../../../xerj-ux", // worktree → parent → xerj-ux (sibling)
- "../../xerj-ux", // direct sibling layout
+ "../../../xerj-ux", // engine/crates/api → repo root → xerj-ux
+ "../../../../xerj-ux", // worktree → parent → xerj-ux (sibling)
+ "../../xerj-ux", // direct sibling layout
];
const ALLOWED_EXTS: &[&str] = &["html", "css", "js", "svg", "ico", "woff", "woff2", "png"];
const MAX_FILE_BYTES: u64 = 4 * 1024 * 1024;
@@ -43,7 +43,9 @@ fn main() {
.find_map(|p| fs::canonicalize(&p).ok())
.or_else(|| {
// Last-ditch: env var override for CI / monorepo packaging.
- env::var("XERJ_CONSOLE_UX_DIR").ok().and_then(|p| fs::canonicalize(p).ok())
+ env::var("XERJ_CONSOLE_UX_DIR")
+ .ok()
+ .and_then(|p| fs::canonicalize(p).ok())
});
let ux_root = match ux_root {
Some(p) => p,
@@ -127,7 +129,8 @@ fn write_assets(entries: &[(String, PathBuf)], _ux_root: &Path) {
// (url_path, bytes, content_type) tuples. Order is sorted-by-path\n\
// so the runtime can binary-search by path.\n\
pub static XERJ_CONSOLE_ASSETS: &[(&str, &[u8], &str)] = &["
- ).unwrap();
+ )
+ .unwrap();
for (url_path, src_path) in entries {
let ct = content_type_for(url_path);
// Use absolute path so include_bytes! doesn't depend on the
@@ -140,7 +143,8 @@ fn write_assets(entries: &[(String, PathBuf)], _ux_root: &Path) {
url_path,
src_path.display(),
ct,
- ).unwrap();
+ )
+ .unwrap();
}
writeln!(f, "];").unwrap();
}
@@ -149,14 +153,14 @@ fn content_type_for(path: &str) -> &'static str {
let ext = path.rsplit('.').next().unwrap_or("");
match ext.to_ascii_lowercase().as_str() {
"html" => "text/html; charset=utf-8",
- "css" => "text/css; charset=utf-8",
- "js" => "application/javascript; charset=utf-8",
- "svg" => "image/svg+xml",
- "ico" => "image/x-icon",
+ "css" => "text/css; charset=utf-8",
+ "js" => "application/javascript; charset=utf-8",
+ "svg" => "image/svg+xml",
+ "ico" => "image/x-icon",
"woff" => "font/woff",
- "woff2"=> "font/woff2",
- "png" => "image/png",
+ "woff2" => "font/woff2",
+ "png" => "image/png",
"json" => "application/json; charset=utf-8",
- _ => "application/octet-stream",
+ _ => "application/octet-stream",
}
}
diff --git a/engine/crates/xerj-console-api/src/auth/login.rs b/engine/crates/xerj-console-api/src/auth/login.rs
index e1d1d1492..51901094f 100644
--- a/engine/crates/xerj-console-api/src/auth/login.rs
+++ b/engine/crates/xerj-console-api/src/auth/login.rs
@@ -2,9 +2,9 @@
//!
//! - `POST /auth/login/begin { email }`
//! → returns the WebAuthn `RequestChallengeResponse` JSON.
-//! Even when the email is unknown we return a *fake* challenge with
-//! no allow-credentials so the response shape is identical — this
-//! denies an attacker the email-enumeration oracle.
+//! Even when the email is unknown we return a *fake* challenge with
+//! no allow-credentials so the response shape is identical — this
+//! denies an attacker the email-enumeration oracle.
//! - `POST /auth/login/finish { challenge_id, credential }`
//! → validates the assertion, mints a session cookie.
//!
@@ -25,7 +25,7 @@ use crate::auth::{audit, rate_limit, sessions, store, webauthn_setup};
use crate::error::{ConsoleApiError, ConsoleResult};
use crate::indices;
use crate::response::ok;
-use crate::state::{ChallengeKind, PendingChallenge, ConsoleState};
+use crate::state::{ChallengeKind, ConsoleState, PendingChallenge};
use crate::time::{now_epoch_ms, now_iso};
const CHALLENGE_TTL_MS: i64 = 5 * 60 * 1000;
@@ -64,13 +64,11 @@ pub async fn begin(
// Collect this user's passkeys (empty list when user is None).
let passkeys: Vec<Passkey> = match &user {
None => Vec::new(),
- Some(u) => {
- store::list_passkeys_for_user(&state.engine, &u.id)
- .await?
- .into_iter()
- .filter_map(|p| serde_json::from_value(p.blob).ok())
- .collect()
- }
+ Some(u) => store::list_passkeys_for_user(&state.engine, &u.id)
+ .await?
+ .into_iter()
+ .filter_map(|p| serde_json::from_value(p.blob).ok())
+ .collect(),
};
let webauthn = webauthn_setup::build(&state)?;
@@ -153,7 +151,9 @@ pub async fn finish(
None => {
// Should be impossible given the challenge state used the
// same credential set, but stay defensive.
- return Err(ConsoleApiError::Unauthorized("authentication failed".into()));
+ return Err(ConsoleApiError::Unauthorized(
+ "authentication failed".into(),
+ ));
}
};
@@ -203,7 +203,11 @@ pub async fn finish(
}),
None,
);
- Ok((axum_extra::extract::cookie::CookieJar::new().add(cookie), body).into_response())
+ Ok((
+ axum_extra::extract::cookie::CookieJar::new().add(cookie),
+ body,
+ )
+ .into_response())
}
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-console-api/src/auth/magic.rs b/engine/crates/xerj-console-api/src/auth/magic.rs
index c0992a506..14e69c356 100644
--- a/engine/crates/xerj-console-api/src/auth/magic.rs
+++ b/engine/crates/xerj-console-api/src/auth/magic.rs
@@ -19,7 +19,7 @@ use crate::bootstrap::sha256_hex;
use crate::error::{ConsoleApiError, ConsoleResult};
use crate::indices;
use crate::response::ok;
-use crate::state::{EnrollmentSession, ConsoleState};
+use crate::state::{ConsoleState, EnrollmentSession};
use crate::time::{now_epoch_ms, now_iso, parse_iso};
const ENROLL_TTL_MS: i64 = 30 * 60 * 1000;
@@ -66,64 +66,69 @@ pub async fn redeem(
// Single-use.
if link.used_at.is_some() {
audit_redeem_failed(&state, "already-used", &ip);
- return Err(ConsoleApiError::Unauthorized("invalid or expired link".into()));
+ return Err(ConsoleApiError::Unauthorized(
+ "invalid or expired link".into(),
+ ));
}
// Expiry.
if let Some(exp) = parse_iso(&link.expires_at) {
if now_epoch_ms() > exp.timestamp_millis() {
audit_redeem_failed(&state, "expired", &ip);
- return Err(ConsoleApiError::Unauthorized("invalid or expired link".into()));
+ return Err(ConsoleApiError::Unauthorized(
+ "invalid or expired link".into(),
+ ));
}
}
// Resolve target user. Bootstrap links have no user_id yet — we
// synthesise one. Invite links carry the user_id we already
// provisioned in pending state.
- let (user_id, email) = match link.purpose.as_str() {
- "bootstrap" => {
- // Make sure no active user has snuck in between mint and
- // redeem (race window when two operators open the same
- // banner).
- let active = store::count_active_users(&state.engine).await?;
- if active > 0 {
- audit_redeem_failed(&state, "bootstrap-already-claimed", &ip);
- return Err(ConsoleApiError::Conflict(
- "this server has already been claimed; ask your admin for an invite".into(),
- ));
+ let (user_id, email) =
+ match link.purpose.as_str() {
+ "bootstrap" => {
+ // Make sure no active user has snuck in between mint and
+ // redeem (race window when two operators open the same
+ // banner).
+ let active = store::count_active_users(&state.engine).await?;
+ if active > 0 {
+ audit_redeem_failed(&state, "bootstrap-already-claimed", &ip);
+ return Err(ConsoleApiError::Conflict(
+ "this server has already been claimed; ask your admin for an invite".into(),
+ ));
+ }
+ // Provision a placeholder user. The SPA fills in the email
+ // and display name during the passkey enrollment flow.
+ let synthetic_id = uuid::Uuid::new_v4().to_string();
+ (synthetic_id, link.email.clone())
}
- // Provision a placeholder user. The SPA fills in the email
- // and display name during the passkey enrollment flow.
- let synthetic_id = uuid::Uuid::new_v4().to_string();
- (synthetic_id, link.email.clone())
- }
- "invite" => {
- let uid = link
- .user_id
- .clone()
- .ok_or_else(|| ConsoleApiError::Internal("invite link without user_id".into()))?;
- // Make sure the invitee row still exists (admin may have
- // deleted them between mint and redeem).
- let user = store::get_user(&state.engine, &uid).await?;
- if user.is_none() {
- audit_redeem_failed(&state, "invitee-gone", &ip);
- return Err(ConsoleApiError::Unauthorized("invalid or expired link".into()));
+ "invite" => {
+ let uid = link.user_id.clone().ok_or_else(|| {
+ ConsoleApiError::Internal("invite link without user_id".into())
+ })?;
+ // Make sure the invitee row still exists (admin may have
+ // deleted them between mint and redeem).
+ let user = store::get_user(&state.engine, &uid).await?;
+ if user.is_none() {
+ audit_redeem_failed(&state, "invitee-gone", &ip);
+ return Err(ConsoleApiError::Unauthorized(
+ "invalid or expired link".into(),
+ ));
+ }
+ (uid, link.email.clone())
}
- (uid, link.email.clone())
- }
- "recovery" => {
- // Reserved for v1.1 — same shape as invite.
- let uid = link
- .user_id
- .clone()
- .ok_or_else(|| ConsoleApiError::Internal("recovery link without user_id".into()))?;
- (uid, link.email.clone())
- }
- other => {
- return Err(ConsoleApiError::Internal(format!(
- "unknown magic-link purpose: {other}"
- )));
- }
- };
+ "recovery" => {
+ // Reserved for v1.1 — same shape as invite.
+ let uid = link.user_id.clone().ok_or_else(|| {
+ ConsoleApiError::Internal("recovery link without user_id".into())
+ })?;
+ (uid, link.email.clone())
+ }
+ other => {
+ return Err(ConsoleApiError::Internal(format!(
+ "unknown magic-link purpose: {other}"
+ )));
+ }
+ };
// Mark used.
store::mark_magic_link_used(&state.engine, &token_hash, &now_iso()).await?;
diff --git a/engine/crates/xerj-console-api/src/auth/passkey.rs b/engine/crates/xerj-console-api/src/auth/passkey.rs
index 3253478a4..23f28a26b 100644
--- a/engine/crates/xerj-console-api/src/auth/passkey.rs
+++ b/engine/crates/xerj-console-api/src/auth/passkey.rs
@@ -4,10 +4,10 @@
//!
//! - `POST /auth/passkey/begin { enrollment_session_id, email?, display_name? }`
//! → returns the WebAuthn `CreationChallengeResponse` JSON the SPA
-//! hands to `navigator.credentials.create({ publicKey })`.
+//! hands to `navigator.credentials.create({ publicKey })`.
//! - `POST /auth/passkey/finish { enrollment_session_id, name, credential }`
//! → validates the attestation, persists the credential, marks the
-//! user `active`, and mints a session cookie.
+//! user `active`, and mints a session cookie.
//!
//! The enrollment session itself lives only in
//! `ConsoleState.enrollment_sessions` — it is consumed once at finish-time
@@ -28,7 +28,7 @@ use crate::auth::{audit, sessions, store, webauthn_setup};
use crate::error::{ConsoleApiError, ConsoleResult};
use crate::indices;
use crate::response::ok;
-use crate::state::{ChallengeKind, PendingChallenge, ConsoleState};
+use crate::state::{ChallengeKind, ConsoleState, PendingChallenge};
use crate::time::{now_epoch_ms, now_iso};
const CHALLENGE_TTL_MS: i64 = 5 * 60 * 1000;
@@ -62,7 +62,9 @@ pub async fn begin(
.ok_or_else(|| ConsoleApiError::Unauthorized("unknown enrollment session".into()))?;
if now_epoch_ms() > enroll.expires_at_ms {
- state.enrollment_sessions.remove(&body.enrollment_session_id);
+ state
+ .enrollment_sessions
+ .remove(&body.enrollment_session_id);
return Err(ConsoleApiError::Unauthorized(
"enrollment session expired".into(),
));
@@ -107,8 +109,7 @@ pub async fn begin(
.collect();
let webauthn = webauthn_setup::build(&state)?;
- let user_unique_id = uuid::Uuid::parse_str(&enroll.user_id)
- .unwrap_or_else(|_| Uuid::new_v4()); // synthetic placeholders are uuids; legacy ids fall back.
+ let user_unique_id = uuid::Uuid::parse_str(&enroll.user_id).unwrap_or_else(|_| Uuid::new_v4()); // synthetic placeholders are uuids; legacy ids fall back.
let display_name = body.display_name.clone().unwrap_or_else(|| email.clone());
let (creation_options, reg_state) = webauthn
@@ -116,7 +117,11 @@ pub async fn begin(
user_unique_id,
&email,
&display_name,
- if exclude.is_empty() { None } else { Some(exclude) },
+ if exclude.is_empty() {
+ None
+ } else {
+ Some(exclude)
+ },
)
.map_err(|e| ConsoleApiError::Internal(format!("start registration: {e}")))?;
@@ -196,8 +201,7 @@ pub async fn finish(
// Persist the credential.
let cred_id_bytes: &[u8] = passkey.cred_id().as_ref();
- let credential_id_b64 =
- base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(cred_id_bytes);
+ let credential_id_b64 = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(cred_id_bytes);
let pk_blob = serde_json::to_value(&passkey)?;
let pk = store::StoredPasskey {
id: credential_id_b64.clone(),
@@ -258,14 +262,8 @@ pub async fn finish(
)
.await;
- let (_session, signed) = sessions::mint_session(
- &state,
- &user.id,
- "passkey",
- Some(ip.clone()),
- ua,
- )
- .await?;
+ let (_session, signed) =
+ sessions::mint_session(&state, &user.id, "passkey", Some(ip.clone()), ua).await?;
audit::record(
&state.engine,
@@ -293,7 +291,11 @@ pub async fn finish(
}),
None,
);
- Ok((axum_extra::extract::cookie::CookieJar::new().add(cookie), body).into_response())
+ Ok((
+ axum_extra::extract::cookie::CookieJar::new().add(cookie),
+ body,
+ )
+ .into_response())
}
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-console-api/src/auth/rate_limit.rs b/engine/crates/xerj-console-api/src/auth/rate_limit.rs
index 447c04a70..137c40dbc 100644
--- a/engine/crates/xerj-console-api/src/auth/rate_limit.rs
+++ b/engine/crates/xerj-console-api/src/auth/rate_limit.rs
@@ -13,7 +13,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
use axum::http::request::Parts;
use crate::error::{ConsoleApiError, ConsoleResult};
-use crate::state::{RateWindow, ConsoleState};
+use crate::state::{ConsoleState, RateWindow};
use crate::time::now_epoch_ms;
const PER_MINUTE: u32 = 10;
@@ -26,7 +26,11 @@ const WINDOW_HOUR_MS: i64 = 3_600_000;
/// "unknown" (so a misconfigured deployment still rate-limits, though
/// per-host instead of per-IP).
pub fn caller_ip(parts: &Parts) -> String {
- if let Some(v) = parts.headers.get("x-forwarded-for").and_then(|h| h.to_str().ok()) {
+ if let Some(v) = parts
+ .headers
+ .get("x-forwarded-for")
+ .and_then(|h| h.to_str().ok())
+ {
if let Some(first) = v.split(',').next() {
let trimmed = first.trim();
if !trimmed.is_empty() {
@@ -98,7 +102,10 @@ pub fn charge(state: &ConsoleState, ip: &str, endpoint: &str) -> ConsoleResult<(
}
state.auth_rate_counters.insert(key_hour, win_val);
- if INSERT_COUNTER.fetch_add(1, Ordering::Relaxed) % 50 == 0 {
+ if INSERT_COUNTER
+ .fetch_add(1, Ordering::Relaxed)
+ .is_multiple_of(50)
+ {
prune(state, now_ms);
}
Ok(())
diff --git a/engine/crates/xerj-console-api/src/auth/sessions.rs b/engine/crates/xerj-console-api/src/auth/sessions.rs
index a86d778af..8018444e1 100644
--- a/engine/crates/xerj-console-api/src/auth/sessions.rs
+++ b/engine/crates/xerj-console-api/src/auth/sessions.rs
@@ -178,10 +178,7 @@ impl FromRequestParts<ConsoleState> for AuthSession {
bumped.last_seen_at = now_iso();
let _ = store::put_session(&state.engine, &bumped).await;
- Ok(AuthSession {
- session_id,
- user,
- })
+ Ok(AuthSession { session_id, user })
}
}
diff --git a/engine/crates/xerj-console-api/src/auth/store.rs b/engine/crates/xerj-console-api/src/auth/store.rs
index f676aa762..cfbfec380 100644
--- a/engine/crates/xerj-console-api/src/auth/store.rs
+++ b/engine/crates/xerj-console-api/src/auth/store.rs
@@ -62,10 +62,7 @@ pub async fn get_user(engine: &Engine, user_id: &str) -> ConsoleResult<Option<Us
search_one(&idx, &body).await
}
-pub async fn find_user_by_email(
- engine: &Engine,
- email: &str,
-) -> ConsoleResult<Option<User>> {
+pub async fn find_user_by_email(engine: &Engine, email: &str) -> ConsoleResult<Option<User>> {
let idx = engine.get_index(indices::USERS)?;
let body = json!({
"query": { "term": { "email": email } },
@@ -115,10 +112,7 @@ pub struct MagicLink {
pub used_at: Option<String>,
}
-pub async fn get_magic_link(
- engine: &Engine,
- token_hash: &str,
-) -> ConsoleResult<Option<MagicLink>> {
+pub async fn get_magic_link(engine: &Engine, token_hash: &str) -> ConsoleResult<Option<MagicLink>> {
let idx = engine.get_index(indices::MAGIC_LINKS)?;
let body = json!({
"query": { "ids": { "values": [token_hash] } },
@@ -146,9 +140,8 @@ pub async fn mark_magic_link_used(
) -> ConsoleResult<()> {
let idx = engine.get_index(indices::MAGIC_LINKS)?;
let existing = get_magic_link(engine, token_hash).await?;
- let mut link = existing.ok_or_else(|| {
- ConsoleApiError::NotFound("magic link not found at consume time".into())
- })?;
+ let mut link = existing
+ .ok_or_else(|| ConsoleApiError::NotFound("magic link not found at consume time".into()))?;
link.used_at = Some(used_at_iso.to_string());
let doc = serde_json::to_value(&link)?;
let _ = idx.delete_document(token_hash).await;
@@ -245,10 +238,7 @@ pub async fn put_session(engine: &Engine, sess: &Session) -> ConsoleResult<()> {
Ok(())
}
-pub async fn get_session(
- engine: &Engine,
- session_id: &str,
-) -> ConsoleResult<Option<Session>> {
+pub async fn get_session(engine: &Engine, session_id: &str) -> ConsoleResult<Option<Session>> {
let idx = engine.get_index(indices::SESSIONS)?;
let body = json!({
"query": { "ids": { "values": [session_id] } },
@@ -293,10 +283,7 @@ pub async fn put_api_token(engine: &Engine, token: &ApiToken) -> ConsoleResult<(
Ok(())
}
-pub async fn get_api_token(
- engine: &Engine,
- token_hash: &str,
-) -> ConsoleResult<Option<ApiToken>> {
+pub async fn get_api_token(engine: &Engine, token_hash: &str) -> ConsoleResult<Option<ApiToken>> {
let idx = engine.get_index(indices::API_TOKENS)?;
let body = json!({
"query": { "ids": { "values": [token_hash] } },
diff --git a/engine/crates/xerj-console-api/src/bootstrap.rs b/engine/crates/xerj-console-api/src/bootstrap.rs
index 12abff936..9e57d43d0 100644
--- a/engine/crates/xerj-console-api/src/bootstrap.rs
+++ b/engine/crates/xerj-console-api/src/bootstrap.rs
@@ -76,8 +76,9 @@ const MASTER_KEY_ENV: &str = "XERJ_CONSOLE_KEY";
fn load_or_init_master_key(data_dir: &Path) -> ConsoleResult<[u8; 32]> {
// Env var wins — covers Kubernetes secret mounts.
if let Ok(hex) = std::env::var(MASTER_KEY_ENV) {
- let bytes = hex_decode(hex.trim())
- .ok_or_else(|| ConsoleApiError::Internal(format!("{MASTER_KEY_ENV}: not 64 hex chars")))?;
+ let bytes = hex_decode(hex.trim()).ok_or_else(|| {
+ ConsoleApiError::Internal(format!("{MASTER_KEY_ENV}: not 64 hex chars"))
+ })?;
if bytes.len() != 32 {
return Err(ConsoleApiError::Internal(format!(
"{MASTER_KEY_ENV}: expected 32 bytes (64 hex), got {}",
@@ -263,7 +264,7 @@ fn hex_encode(bytes: &[u8]) -> String {
}
fn hex_decode(s: &str) -> Option<Vec<u8>> {
- if s.len() % 2 != 0 {
+ if !s.len().is_multiple_of(2) {
return None;
}
let mut out = Vec::with_capacity(s.len() / 2);
diff --git a/engine/crates/xerj-console-api/src/dashboards.rs b/engine/crates/xerj-console-api/src/dashboards.rs
index f5ed34edb..dcd70bf66 100644
--- a/engine/crates/xerj-console-api/src/dashboards.rs
+++ b/engine/crates/xerj-console-api/src/dashboards.rs
@@ -57,7 +57,9 @@ pub struct Dashboard {
pub deleted_at: Option<String>,
}
-fn default_org() -> String { "default".to_string() }
+fn default_org() -> String {
+ "default".to_string()
+}
#[derive(Debug, Deserialize)]
pub struct CreateBody {
@@ -78,7 +80,9 @@ pub struct CreateBody {
pub time_default: Option<String>,
}
-fn default_visibility() -> String { "private".to_string() }
+fn default_visibility() -> String {
+ "private".to_string()
+}
#[derive(Debug, Deserialize, Default)]
pub struct PatchBody {
@@ -102,10 +106,7 @@ pub struct PatchBody {
// LIST
// ─────────────────────────────────────────────────────────────────────────────
-pub async fn list(
- State(state): State<ConsoleState>,
- sess: AuthSession,
-) -> ConsoleResult<Response> {
+pub async fn list(State(state): State<ConsoleState>, sess: AuthSession) -> ConsoleResult<Response> {
let idx = state.engine.get_index(indices::DASHBOARDS)?;
// Pull everything; filter by owner / visibility in code. The
// `.xerj_dashboards` index is small (≤ low thousands) so the
@@ -128,14 +129,15 @@ pub async fn list(
if d.deleted_at.is_some() {
continue;
}
- let visible = d.owner == me
- || d.visibility == "shared"
- || d.visibility == "default";
+ let visible = d.owner == me || d.visibility == "shared" || d.visibility == "default";
if visible {
items.push(serde_json::to_value(&d)?);
}
}
- Ok(ok(json!({ "dashboards": items, "total": items.len() }), None))
+ Ok(ok(
+ json!({ "dashboards": items, "total": items.len() }),
+ None,
+ ))
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -250,16 +252,28 @@ pub async fn patch(
enforce_write(&sess, &dash)?;
enforce_etag(&headers, dash.version)?;
- if let Some(name) = body.name { dash.name = name; }
+ if let Some(name) = body.name {
+ dash.name = name;
+ }
if let Some(visibility) = body.visibility {
enforce_can_write_visibility(&sess, &visibility)?;
dash.visibility = visibility;
}
- if let Some(section) = body.section { dash.section = Some(section); }
- if let Some(group) = body.group { dash.group = Some(group); }
- if let Some(panels) = body.panels { dash.panels = panels; }
- if let Some(filters) = body.filters_default { dash.filters_default = filters; }
- if let Some(td) = body.time_default { dash.time_default = Some(td); }
+ if let Some(section) = body.section {
+ dash.section = Some(section);
+ }
+ if let Some(group) = body.group {
+ dash.group = Some(group);
+ }
+ if let Some(panels) = body.panels {
+ dash.panels = panels;
+ }
+ if let Some(filters) = body.filters_default {
+ dash.filters_default = filters;
+ }
+ if let Some(td) = body.time_default {
+ dash.time_default = Some(td);
+ }
dash.version += 1;
dash.updated_at = now_iso();
@@ -314,12 +328,15 @@ async fn read_required(state: &ConsoleState, id: &str) -> ConsoleResult<Dashboar
async fn write_doc(state: &ConsoleState, dash: &Dashboard) -> ConsoleResult<()> {
let idx = state.engine.get_index(indices::DASHBOARDS)?;
let _ = idx.delete_document(&dash.id).await;
- idx.create_document(dash.id.clone(), serde_json::to_value(dash)?).await?;
+ idx.create_document(dash.id.clone(), serde_json::to_value(dash)?)
+ .await?;
Ok(())
}
fn enforce_read(sess: &AuthSession, dash: &Dashboard) -> ConsoleResult<()> {
- if dash.owner == sess.user.id { return Ok(()); }
+ if dash.owner == sess.user.id {
+ return Ok(());
+ }
match dash.visibility.as_str() {
"shared" | "default" => Ok(()),
_ => Err(ConsoleApiError::NotFound("dashboard".into())),
@@ -377,9 +394,7 @@ fn enforce_etag(headers: &HeaderMap, current_version: u64) -> ConsoleResult<()>
return Ok(());
};
let trimmed = if_match.trim();
- let unwrapped = trimmed
- .trim_start_matches("W/")
- .trim_matches('"');
+ let unwrapped = trimmed.trim_start_matches("W/").trim_matches('"');
match unwrapped.parse::<u64>() {
Ok(v) if v == current_version => Ok(()),
_ => Err(ConsoleApiError::Conflict(format!(
diff --git a/engine/crates/xerj-console-api/src/data_sources.rs b/engine/crates/xerj-console-api/src/data_sources.rs
index 711d75034..0ced13b6d 100644
--- a/engine/crates/xerj-console-api/src/data_sources.rs
+++ b/engine/crates/xerj-console-api/src/data_sources.rs
@@ -44,7 +44,9 @@ pub struct Connection {
pub etag: String,
}
-fn default_true() -> bool { true }
+fn default_true() -> bool {
+ true
+}
const BUILTIN_ID: &str = "built-in";
@@ -78,7 +80,11 @@ pub async fn list_connections(
// it). External adapters add real probing in a later commit.
let mut out: Vec<Value> = Vec::with_capacity(conns.len());
for c in conns {
- let status = if c.id == BUILTIN_ID { "green" } else { "unknown" };
+ let status = if c.id == BUILTIN_ID {
+ "green"
+ } else {
+ "unknown"
+ };
let mut v = serde_json::to_value(&c)?;
v["status"] = Value::String(status.into());
v["last_checked_at"] = Value::String(now_iso());
@@ -130,7 +136,9 @@ pub async fn list_indices(
}));
}
items.sort_by(|a, b| {
- a["name"].as_str().unwrap_or("")
+ a["name"]
+ .as_str()
+ .unwrap_or("")
.cmp(b["name"].as_str().unwrap_or(""))
});
Ok(ok(json!({ "indices": items, "total": items.len() }), None))
@@ -170,10 +178,7 @@ pub async fn list_fields(
})
})
.collect();
- Ok(ok(
- json!({ "fields": fields, "total": fields.len() }),
- None,
- ))
+ Ok(ok(json!({ "fields": fields, "total": fields.len() }), None))
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -204,6 +209,7 @@ async fn ensure_builtin(state: &ConsoleState) -> ConsoleResult<()> {
etag: "1".into(),
};
let _ = idx.delete_document(BUILTIN_ID).await;
- idx.create_document(BUILTIN_ID.into(), serde_json::to_value(&conn)?).await?;
+ idx.create_document(BUILTIN_ID.into(), serde_json::to_value(&conn)?)
+ .await?;
Ok(())
}
diff --git a/engine/crates/xerj-console-api/src/prefs.rs b/engine/crates/xerj-console-api/src/prefs.rs
index b88654be7..c353d4c23 100644
--- a/engine/crates/xerj-console-api/src/prefs.rs
+++ b/engine/crates/xerj-console-api/src/prefs.rs
@@ -23,10 +23,7 @@ const FALLBACK_PREFS: &str = r#"{
"mobile": false
}"#;
-pub async fn get(
- State(state): State<ConsoleState>,
- sess: AuthSession,
-) -> ConsoleResult<Response> {
+pub async fn get(State(state): State<ConsoleState>, sess: AuthSession) -> ConsoleResult<Response> {
let idx = state.engine.get_index(indices::PREFS)?;
let body = json!({
"query": { "ids": { "values": [&sess.user.id] } },
@@ -37,8 +34,7 @@ pub async fn get(
let r = idx.search(&req).await?;
let prefs = match r.hits.into_iter().next() {
Some(h) => h.source,
- None => serde_json::from_str(FALLBACK_PREFS)
- .expect("fallback prefs are valid json"),
+ None => serde_json::from_str(FALLBACK_PREFS).expect("fallback prefs are valid json"),
};
Ok(ok(prefs, None))
}
@@ -54,13 +50,13 @@ pub async fn put(
));
}
// Strip any client-set "updated_at" so the server is authoritative.
- body.as_object_mut().unwrap().insert(
- "updated_at".into(),
- Value::String(now_iso()),
- );
+ body.as_object_mut()
+ .unwrap()
+ .insert("updated_at".into(), Value::String(now_iso()));
let idx = state.engine.get_index(indices::PREFS)?;
let _ = idx.delete_document(&sess.user.id).await;
- idx.create_document(sess.user.id.clone(), body.clone()).await?;
+ idx.create_document(sess.user.id.clone(), body.clone())
+ .await?;
Ok(ok(body, None))
}
diff --git a/engine/crates/xerj-console-api/src/router.rs b/engine/crates/xerj-console-api/src/router.rs
index 56497a0c4..2ba0a82cc 100644
--- a/engine/crates/xerj-console-api/src/router.rs
+++ b/engine/crates/xerj-console-api/src/router.rs
@@ -30,9 +30,7 @@ use axum::{
Router,
};
-use crate::{
- auth, cluster, dashboards, data_sources, prefs, spa, state::ConsoleState, views,
-};
+use crate::{auth, cluster, dashboards, data_sources, prefs, spa, state::ConsoleState, views};
/// Build the full Xerj Console router. Mount at the root of an axum Router
/// (it owns the `/_xerj-console/*` prefix internally):
@@ -71,9 +69,15 @@ pub fn xerj_console_router(state: ConsoleState) -> Router {
post(auth::login::finish),
)
// ── API: session-protected ─────────────────────────────────────────
- .route("/_xerj-console/api/v1/auth/logout", post(auth::login::logout))
+ .route(
+ "/_xerj-console/api/v1/auth/logout",
+ post(auth::login::logout),
+ )
.route("/_xerj-console/api/v1/me", get(auth::me::me))
- .route("/_xerj-console/api/v1/auth/passkeys", get(auth::me::list_passkeys))
+ .route(
+ "/_xerj-console/api/v1/auth/passkeys",
+ get(auth::me::list_passkeys),
+ )
.route(
"/_xerj-console/api/v1/auth/passkeys/:id",
delete(auth::me::delete_passkey),
@@ -92,7 +96,10 @@ pub fn xerj_console_router(state: ConsoleState) -> Router {
.put(dashboards::replace)
.delete(dashboards::delete),
)
- .route("/_xerj-console/api/v1/dashboards/:id", patch(dashboards::patch))
+ .route(
+ "/_xerj-console/api/v1/dashboards/:id",
+ patch(dashboards::patch),
+ )
.route(
"/_xerj-console/api/v1/views",
get(views::list).post(views::create),
diff --git a/engine/crates/xerj-console-api/src/spa.rs b/engine/crates/xerj-console-api/src/spa.rs
index 966b68eb1..7a0a5667b 100644
--- a/engine/crates/xerj-console-api/src/spa.rs
+++ b/engine/crates/xerj-console-api/src/spa.rs
@@ -46,7 +46,11 @@ pub async fn xerj_console_index() -> impl IntoResponse {
/// Serve a named asset under `/_xerj-console/{path*}`.
pub async fn xerj_console_asset(Path(rest): Path<String>) -> impl IntoResponse {
// Empty rest path = same as `/_xerj-console/` — serve index.
- let path = if rest.is_empty() { "index.html".to_string() } else { rest };
+ let path = if rest.is_empty() {
+ "index.html".to_string()
+ } else {
+ rest
+ };
serve(&path, false)
}
@@ -81,16 +85,14 @@ fn serve(path: &str, no_cache: bool) -> Response {
return (
StatusCode::NOT_FOUND,
format!("xerj-console asset not found: {path}"),
- ).into_response();
+ )
+ .into_response();
}
};
serve_asset(asset, no_cache)
}
-fn serve_asset(
- asset: &(&'static str, &'static [u8], &'static str),
- no_cache: bool,
-) -> Response {
+fn serve_asset(asset: &(&'static str, &'static [u8], &'static str), no_cache: bool) -> Response {
let cache = if no_cache {
"no-cache"
} else {
diff --git a/engine/crates/xerj-console-api/src/state.rs b/engine/crates/xerj-console-api/src/state.rs
index 863b799ad..1a836b6ed 100644
--- a/engine/crates/xerj-console-api/src/state.rs
+++ b/engine/crates/xerj-console-api/src/state.rs
@@ -133,7 +133,13 @@ impl ConsoleState {
master_key: [u8; 32],
cluster_mode: ClusterMode,
) -> Self {
- Self::new_with_rp(engine, node_id, master_key, cluster_mode, RpConfig::default())
+ Self::new_with_rp(
+ engine,
+ node_id,
+ master_key,
+ cluster_mode,
+ RpConfig::default(),
+ )
}
pub fn new_with_rp(
diff --git a/engine/crates/xerj-console-api/src/views.rs b/engine/crates/xerj-console-api/src/views.rs
index 5edf4f61f..5cd31bfd2 100644
--- a/engine/crates/xerj-console-api/src/views.rs
+++ b/engine/crates/xerj-console-api/src/views.rs
@@ -35,7 +35,9 @@ pub struct View {
pub updated_at: String,
}
-fn default_org() -> String { "default".to_string() }
+fn default_org() -> String {
+ "default".to_string()
+}
#[derive(Debug, Deserialize)]
pub struct ListQuery {
@@ -90,7 +92,8 @@ pub async fn create(
updated_at: now_iso(),
};
let idx = state.engine.get_index(indices::VIEWS)?;
- idx.create_document(id.clone(), serde_json::to_value(&view)?).await?;
+ idx.create_document(id.clone(), serde_json::to_value(&view)?)
+ .await?;
let location = format!("/_xerj-console/api/v1/views/{id}");
Ok(created(view, &location, None))
}
diff --git a/engine/crates/xerj-console-api/tests/phase1_boot_and_cluster_info.rs b/engine/crates/xerj-console-api/tests/phase1_boot_and_cluster_info.rs
index 6f8eb5c98..08ed3ce4b 100644
--- a/engine/crates/xerj-console-api/tests/phase1_boot_and_cluster_info.rs
+++ b/engine/crates/xerj-console-api/tests/phase1_boot_and_cluster_info.rs
@@ -14,8 +14,8 @@ use serde_json::Value;
use tempfile::TempDir;
use tower::ServiceExt; // for `.oneshot`
use xerj_common::config::Config;
-use xerj_engine::Engine;
use xerj_console_api::{state::ClusterMode, xerj_console_router, ConsoleState};
+use xerj_engine::Engine;
fn engine_in(dir: &TempDir) -> Engine {
let mut cfg = Config::default();
@@ -108,8 +108,9 @@ async fn cluster_info_returns_raft_mode_when_clustered() {
// ClusterMode::Raft + a non-"local" node id.
let dir = TempDir::new().unwrap();
let engine = engine_in(&dir);
- let outcome =
- xerj_console_api::bootstrap::run(&engine, dir.path(), "http://x").await.unwrap();
+ let outcome = xerj_console_api::bootstrap::run(&engine, dir.path(), "http://x")
+ .await
+ .unwrap();
let state = ConsoleState::new(
engine,
"10.0.0.1:7000".to_string(),
@@ -188,6 +189,9 @@ async fn known_routes_table_matches_registered_routes() {
// Catches the easy mistake of adding a route to known_routes() (used
// by tests) without registering it, or vice versa.
let routes = xerj_console_api::router::known_routes();
- assert!(!routes.is_empty(), "phase 1 must expose at least /cluster/info");
+ assert!(
+ !routes.is_empty(),
+ "phase 1 must expose at least /cluster/info"
+ );
assert!(routes.contains(&"/_xerj-console/api/v1/cluster/info"));
}
diff --git a/engine/crates/xerj-console-api/tests/phase2_auth_flow.rs b/engine/crates/xerj-console-api/tests/phase2_auth_flow.rs
index d544b7a86..945394c59 100644
--- a/engine/crates/xerj-console-api/tests/phase2_auth_flow.rs
+++ b/engine/crates/xerj-console-api/tests/phase2_auth_flow.rs
@@ -24,8 +24,8 @@ use serde_json::{json, Value};
use tempfile::TempDir;
use tower::ServiceExt;
use xerj_common::config::Config;
-use xerj_engine::Engine;
use xerj_console_api::{state::ClusterMode, xerj_console_router, ConsoleState};
+use xerj_engine::Engine;
async fn boot() -> (Engine, Router, TempDir, String) {
let dir = TempDir::new().unwrap();
@@ -102,13 +102,19 @@ async fn magic_redeem_is_single_use() {
let r1 = router
.clone()
- .oneshot(post_json("/_xerj-console/api/v1/auth/magic/redeem", json!({ "token": token })))
+ .oneshot(post_json(
+ "/_xerj-console/api/v1/auth/magic/redeem",
+ json!({ "token": token }),
+ ))
.await
.unwrap();
assert_eq!(r1.status(), 200);
let r2 = router
- .oneshot(post_json("/_xerj-console/api/v1/auth/magic/redeem", json!({ "token": token })))
+ .oneshot(post_json(
+ "/_xerj-console/api/v1/auth/magic/redeem",
+ json!({ "token": token }),
+ ))
.await
.unwrap();
assert_eq!(r2.status(), 401, "second redemption must fail");
@@ -135,7 +141,10 @@ async fn passkey_begin_returns_creation_options_for_valid_enrollment() {
// Redeem to get an enrollment session.
let r = router
.clone()
- .oneshot(post_json("/_xerj-console/api/v1/auth/magic/redeem", json!({ "token": token })))
+ .oneshot(post_json(
+ "/_xerj-console/api/v1/auth/magic/redeem",
+ json!({ "token": token }),
+ ))
.await
.unwrap();
let (_, redeem_body) = body_json(r).await;
@@ -218,9 +227,7 @@ async fn login_begin_rate_limit_kicks_in() {
.uri("/_xerj-console/api/v1/auth/login/begin")
.header("content-type", "application/json")
.header("x-forwarded-for", "9.9.9.9")
- .body(Body::from(
- json!({ "email": "x@y" }).to_string(),
- ))
+ .body(Body::from(json!({ "email": "x@y" }).to_string()))
.unwrap(),
)
.await
@@ -274,7 +281,10 @@ async fn passkey_finish_rejects_bogus_credential() {
let r = router
.clone()
- .oneshot(post_json("/_xerj-console/api/v1/auth/magic/redeem", json!({ "token": token })))
+ .oneshot(post_json(
+ "/_xerj-console/api/v1/auth/magic/redeem",
+ json!({ "token": token }),
+ ))
.await
.unwrap();
let (_, redeem_body) = body_json(r).await;
diff --git a/engine/crates/xerj-console-api/tests/phase3_user_state.rs b/engine/crates/xerj-console-api/tests/phase3_user_state.rs
index ea002b604..31ba98435 100644
--- a/engine/crates/xerj-console-api/tests/phase3_user_state.rs
+++ b/engine/crates/xerj-console-api/tests/phase3_user_state.rs
@@ -14,12 +14,12 @@ use serde_json::{json, Value};
use tempfile::TempDir;
use tower::ServiceExt;
use xerj_common::config::Config;
-use xerj_engine::Engine;
use xerj_console_api::{
auth::{sessions, store},
state::ClusterMode,
xerj_console_router, ConsoleState,
};
+use xerj_engine::Engine;
struct TestApp {
router: Router,
@@ -32,10 +32,9 @@ async fn boot_with_session() -> TestApp {
let mut cfg = Config::default();
cfg.server.data_dir = dir.path().to_str().unwrap().to_string();
let engine = Engine::new(cfg).expect("engine");
- let outcome =
- xerj_console_api::bootstrap::run(&engine, dir.path(), "http://localhost:9200")
- .await
- .unwrap();
+ let outcome = xerj_console_api::bootstrap::run(&engine, dir.path(), "http://localhost:9200")
+ .await
+ .unwrap();
let state = ConsoleState::new(
engine.clone(),
"local".into(),
@@ -55,13 +54,17 @@ async fn boot_with_session() -> TestApp {
last_seen_at: Some(xerj_console_api::time::now_iso()),
};
store::upsert_user(&engine, &user).await.unwrap();
- let (_session, signed) = sessions::mint_session(
- &state, &user.id, "passkey", None, None,
- ).await.unwrap();
+ let (_session, signed) = sessions::mint_session(&state, &user.id, "passkey", None, None)
+ .await
+ .unwrap();
let cookie = format!("xerj_session={signed}");
let router = xerj_console_router(state);
- TestApp { router, cookie, _dir: dir }
+ TestApp {
+ router,
+ cookie,
+ _dir: dir,
+ }
}
async fn body_json(resp: axum::response::Response) -> (axum::http::StatusCode, Value) {
@@ -96,7 +99,11 @@ fn req(method: &str, path: &str, cookie: &str, body: Option<Value>) -> Request<B
#[tokio::test]
async fn prefs_default_when_unset() {
let app = boot_with_session().await;
- let r = app.router.oneshot(req("GET", "/_xerj-console/api/v1/prefs", &app.cookie, None)).await.unwrap();
+ let r = app
+ .router
+ .oneshot(req("GET", "/_xerj-console/api/v1/prefs", &app.cookie, None))
+ .await
+ .unwrap();
let (status, body) = body_json(r).await;
assert_eq!(status, 200);
assert!(body["data"]["theme"].is_string(), "got body: {body}");
@@ -106,15 +113,24 @@ async fn prefs_default_when_unset() {
async fn prefs_round_trip() {
let app = boot_with_session().await;
- let put = req("PUT", "/_xerj-console/api/v1/prefs", &app.cookie, Some(json!({
- "theme": "light",
- "time": "1H",
- "cluster": "REMOTE"
- })));
+ let put = req(
+ "PUT",
+ "/_xerj-console/api/v1/prefs",
+ &app.cookie,
+ Some(json!({
+ "theme": "light",
+ "time": "1H",
+ "cluster": "REMOTE"
+ })),
+ );
let r = app.router.clone().oneshot(put).await.unwrap();
assert_eq!(r.status(), 200);
- let r = app.router.oneshot(req("GET", "/_xerj-console/api/v1/prefs", &app.cookie, None)).await.unwrap();
+ let r = app
+ .router
+ .oneshot(req("GET", "/_xerj-console/api/v1/prefs", &app.cookie, None))
+ .await
+ .unwrap();
let (_, body) = body_json(r).await;
assert_eq!(body["data"]["theme"], "light");
assert_eq!(body["data"]["time"], "1H");
@@ -129,10 +145,17 @@ async fn dashboards_create_get_list_patch_delete() {
let app = boot_with_session().await;
// CREATE
- let r = app.router.clone().oneshot(req(
- "POST", "/_xerj-console/api/v1/dashboards", &app.cookie,
- Some(json!({ "name": "AI Overview", "visibility": "private" })),
- )).await.unwrap();
+ let r = app
+ .router
+ .clone()
+ .oneshot(req(
+ "POST",
+ "/_xerj-console/api/v1/dashboards",
+ &app.cookie,
+ Some(json!({ "name": "AI Overview", "visibility": "private" })),
+ ))
+ .await
+ .unwrap();
let (status, body) = body_json(r).await;
assert_eq!(status, 201, "create: {body}");
let dash_id = body["data"]["id"].as_str().unwrap().to_string();
@@ -140,25 +163,44 @@ async fn dashboards_create_get_list_patch_delete() {
assert_eq!(body["data"]["version"], 1);
// GET ONE
- let r = app.router.clone().oneshot(req(
- "GET", &format!("/_xerj-console/api/v1/dashboards/{dash_id}"), &app.cookie, None,
- )).await.unwrap();
+ let r = app
+ .router
+ .clone()
+ .oneshot(req(
+ "GET",
+ &format!("/_xerj-console/api/v1/dashboards/{dash_id}"),
+ &app.cookie,
+ None,
+ ))
+ .await
+ .unwrap();
let (status, body) = body_json(r).await;
assert_eq!(status, 200);
assert_eq!(body["data"]["id"], dash_id);
// LIST
- let r = app.router.clone().oneshot(req(
- "GET", "/_xerj-console/api/v1/dashboards", &app.cookie, None,
- )).await.unwrap();
+ let r = app
+ .router
+ .clone()
+ .oneshot(req(
+ "GET",
+ "/_xerj-console/api/v1/dashboards",
+ &app.cookie,
+ None,
+ ))
+ .await
+ .unwrap();
let (_, body) = body_json(r).await;
let arr = body["data"]["dashboards"].as_array().unwrap();
- assert!(arr.iter().any(|d| d["id"] == dash_id), "missing in list: {body}");
+ assert!(
+ arr.iter().any(|d| d["id"] == dash_id),
+ "missing in list: {body}"
+ );
// PATCH (rename) — exercise the etag check path with a correct etag
let patch = Request::builder()
.method("PATCH")
- .uri(&format!("/_xerj-console/api/v1/dashboards/{dash_id}"))
+ .uri(format!("/_xerj-console/api/v1/dashboards/{dash_id}"))
.header("cookie", &app.cookie)
.header("content-type", "application/json")
.header("if-match", "W/\"1\"")
@@ -173,7 +215,7 @@ async fn dashboards_create_get_list_patch_delete() {
// PATCH with stale etag → 409
let stale = Request::builder()
.method("PATCH")
- .uri(&format!("/_xerj-console/api/v1/dashboards/{dash_id}"))
+ .uri(format!("/_xerj-console/api/v1/dashboards/{dash_id}"))
.header("cookie", &app.cookie)
.header("content-type", "application/json")
.header("if-match", "W/\"1\"")
@@ -183,34 +225,63 @@ async fn dashboards_create_get_list_patch_delete() {
assert_eq!(r.status(), 409, "stale etag must 409");
// DELETE
- let r = app.router.clone().oneshot(req(
- "DELETE", &format!("/_xerj-console/api/v1/dashboards/{dash_id}"), &app.cookie, None,
- )).await.unwrap();
+ let r = app
+ .router
+ .clone()
+ .oneshot(req(
+ "DELETE",
+ &format!("/_xerj-console/api/v1/dashboards/{dash_id}"),
+ &app.cookie,
+ None,
+ ))
+ .await
+ .unwrap();
assert_eq!(r.status(), 204);
// GET after delete → 404
- let r = app.router.oneshot(req(
- "GET", &format!("/_xerj-console/api/v1/dashboards/{dash_id}"), &app.cookie, None,
- )).await.unwrap();
+ let r = app
+ .router
+ .oneshot(req(
+ "GET",
+ &format!("/_xerj-console/api/v1/dashboards/{dash_id}"),
+ &app.cookie,
+ None,
+ ))
+ .await
+ .unwrap();
assert_eq!(r.status(), 404);
}
#[tokio::test]
async fn dashboards_unauth_returns_401() {
let app = boot_with_session().await;
- let r = app.router.oneshot(
- Request::builder().method("GET").uri("/_xerj-console/api/v1/dashboards").body(Body::empty()).unwrap()
- ).await.unwrap();
+ let r = app
+ .router
+ .oneshot(
+ Request::builder()
+ .method("GET")
+ .uri("/_xerj-console/api/v1/dashboards")
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
assert_eq!(r.status(), 401);
}
#[tokio::test]
async fn dashboard_create_rejects_unknown_visibility() {
let app = boot_with_session().await;
- let r = app.router.oneshot(req(
- "POST", "/_xerj-console/api/v1/dashboards", &app.cookie,
- Some(json!({ "name": "x", "visibility": "totally-not-real" })),
- )).await.unwrap();
+ let r = app
+ .router
+ .oneshot(req(
+ "POST",
+ "/_xerj-console/api/v1/dashboards",
+ &app.cookie,
+ Some(json!({ "name": "x", "visibility": "totally-not-real" })),
+ ))
+ .await
+ .unwrap();
assert_eq!(r.status(), 400);
}
@@ -223,37 +294,66 @@ async fn views_round_trip() {
let app = boot_with_session().await;
// Need a dashboard first
- let r = app.router.clone().oneshot(req(
- "POST", "/_xerj-console/api/v1/dashboards", &app.cookie,
- Some(json!({ "name": "D", "visibility": "private" })),
- )).await.unwrap();
+ let r = app
+ .router
+ .clone()
+ .oneshot(req(
+ "POST",
+ "/_xerj-console/api/v1/dashboards",
+ &app.cookie,
+ Some(json!({ "name": "D", "visibility": "private" })),
+ ))
+ .await
+ .unwrap();
let (_, body) = body_json(r).await;
let dash_id = body["data"]["id"].as_str().unwrap().to_string();
// Create view
- let r = app.router.clone().oneshot(req(
- "POST", "/_xerj-console/api/v1/views", &app.cookie,
- Some(json!({
- "dashboard_id": dash_id, "name": "Last 24h",
- "time": { "from": "now-24h", "to": "now" }
- })),
- )).await.unwrap();
+ let r = app
+ .router
+ .clone()
+ .oneshot(req(
+ "POST",
+ "/_xerj-console/api/v1/views",
+ &app.cookie,
+ Some(json!({
+ "dashboard_id": dash_id, "name": "Last 24h",
+ "time": { "from": "now-24h", "to": "now" }
+ })),
+ ))
+ .await
+ .unwrap();
let (status, body) = body_json(r).await;
assert_eq!(status, 201, "{body}");
let view_id = body["data"]["id"].as_str().unwrap().to_string();
// List filtered by dashboard
- let r = app.router.clone().oneshot(req(
- "GET", &format!("/_xerj-console/api/v1/views?dashboard={dash_id}"), &app.cookie, None,
- )).await.unwrap();
+ let r = app
+ .router
+ .clone()
+ .oneshot(req(
+ "GET",
+ &format!("/_xerj-console/api/v1/views?dashboard={dash_id}"),
+ &app.cookie,
+ None,
+ ))
+ .await
+ .unwrap();
let (_, body) = body_json(r).await;
assert_eq!(body["data"]["total"], 1);
assert_eq!(body["data"]["views"][0]["id"], view_id);
// Delete
- let r = app.router.oneshot(req(
- "DELETE", &format!("/_xerj-console/api/v1/views/{view_id}"), &app.cookie, None,
- )).await.unwrap();
+ let r = app
+ .router
+ .oneshot(req(
+ "DELETE",
+ &format!("/_xerj-console/api/v1/views/{view_id}"),
+ &app.cookie,
+ None,
+ ))
+ .await
+ .unwrap();
assert_eq!(r.status(), 204);
}
@@ -264,9 +364,16 @@ async fn views_round_trip() {
#[tokio::test]
async fn data_sources_lists_built_in_connection() {
let app = boot_with_session().await;
- let r = app.router.oneshot(req(
- "GET", "/_xerj-console/api/v1/data-sources/connections", &app.cookie, None,
- )).await.unwrap();
+ let r = app
+ .router
+ .oneshot(req(
+ "GET",
+ "/_xerj-console/api/v1/data-sources/connections",
+ &app.cookie,
+ None,
+ ))
+ .await
+ .unwrap();
let (status, body) = body_json(r).await;
assert_eq!(status, 200);
let conns = body["data"]["connections"].as_array().unwrap();
@@ -280,9 +387,16 @@ async fn data_sources_lists_built_in_connection() {
#[tokio::test]
async fn data_sources_list_indices_for_built_in() {
let app = boot_with_session().await;
- let r = app.router.oneshot(req(
- "GET", "/_xerj-console/api/v1/data-sources/connections/built-in/indices", &app.cookie, None,
- )).await.unwrap();
+ let r = app
+ .router
+ .oneshot(req(
+ "GET",
+ "/_xerj-console/api/v1/data-sources/connections/built-in/indices",
+ &app.cookie,
+ None,
+ ))
+ .await
+ .unwrap();
let (status, body) = body_json(r).await;
assert_eq!(status, 200);
assert!(body["data"]["indices"].is_array());
@@ -293,9 +407,16 @@ async fn data_sources_list_indices_for_built_in() {
#[tokio::test]
async fn data_sources_unknown_connection_returns_501() {
let app = boot_with_session().await;
- let r = app.router.oneshot(req(
- "GET", "/_xerj-console/api/v1/data-sources/connections/elasticsearch-prod/indices", &app.cookie, None,
- )).await.unwrap();
+ let r = app
+ .router
+ .oneshot(req(
+ "GET",
+ "/_xerj-console/api/v1/data-sources/connections/elasticsearch-prod/indices",
+ &app.cookie,
+ None,
+ ))
+ .await
+ .unwrap();
assert_eq!(r.status(), 501);
}
diff --git a/engine/crates/xerj-console-api/tests/router_precedence.rs b/engine/crates/xerj-console-api/tests/router_precedence.rs
index 8afcaa25b..292ebc5cd 100644
--- a/engine/crates/xerj-console-api/tests/router_precedence.rs
+++ b/engine/crates/xerj-console-api/tests/router_precedence.rs
@@ -11,8 +11,8 @@ use http_body_util::BodyExt;
use tempfile::TempDir;
use tower::ServiceExt;
use xerj_common::config::Config;
-use xerj_engine::Engine;
use xerj_console_api::{state::ClusterMode, xerj_console_router, ConsoleState};
+use xerj_engine::Engine;
fn boot() -> (axum::Router, TempDir) {
let dir = TempDir::new().unwrap();
diff --git a/engine/crates/xerj-console-api/tests/spa_assets.rs b/engine/crates/xerj-console-api/tests/spa_assets.rs
index 7ec38cd1f..e93ffcd88 100644
--- a/engine/crates/xerj-console-api/tests/spa_assets.rs
+++ b/engine/crates/xerj-console-api/tests/spa_assets.rs
@@ -11,8 +11,8 @@ use http_body_util::BodyExt;
use tempfile::TempDir;
use tower::ServiceExt;
use xerj_common::config::Config;
-use xerj_engine::Engine;
use xerj_console_api::{state::ClusterMode, xerj_console_router, ConsoleState};
+use xerj_engine::Engine;
fn boot() -> (axum::Router, TempDir) {
let dir = TempDir::new().unwrap();
@@ -31,12 +31,7 @@ fn boot() -> (axum::Router, TempDir) {
async fn fetch(router: axum::Router, path: &str) -> (u16, String) {
let resp = router
- .oneshot(
- Request::builder()
- .uri(path)
- .body(Body::empty())
- .unwrap(),
- )
+ .oneshot(Request::builder().uri(path).body(Body::empty()).unwrap())
.await
.unwrap();
let status = resp.status().as_u16();
@@ -49,7 +44,10 @@ async fn setup_html_is_bundled_and_served() {
let (router, _dir) = boot();
// /_xerj-console/setup (no extension) should fall back to setup.html.
let (status, body) = fetch(router, "/_xerj-console/setup").await;
- assert_eq!(status, 200, "expected 200 for /_xerj-console/setup, body={body}");
+ assert_eq!(
+ status, 200,
+ "expected 200 for /_xerj-console/setup, body={body}"
+ );
assert!(
body.contains("XERJ CONSOLE") && body.contains("setup"),
"setup page must contain XERJ CONSOLE banner — got: {}",
diff --git a/engine/crates/xerj-engine/benches/search_bench.rs b/engine/crates/xerj-engine/benches/search_bench.rs
index 2dd39b53e..827f596f0 100644
--- a/engine/crates/xerj-engine/benches/search_bench.rs
+++ b/engine/crates/xerj-engine/benches/search_bench.rs
@@ -145,9 +145,7 @@ fn bench_bm25_search(c: &mut Criterion) {
b.iter(|| {
let idx = Arc::clone(&idx_ref);
let req = request.clone();
- rt_ref.block_on(async move {
- black_box(idx.search(&req).await.expect("search"))
- })
+ rt_ref.block_on(async move { black_box(idx.search(&req).await.expect("search")) })
});
});
}
@@ -302,9 +300,7 @@ fn bench_aggregations(c: &mut Criterion) {
b.iter(|| {
let idx = Arc::clone(&idx_ref);
let req = request.clone();
- rt_ref.block_on(async move {
- black_box(idx.search(&req).await.expect("search"))
- })
+ rt_ref.block_on(async move { black_box(idx.search(&req).await.expect("search")) })
});
});
}
diff --git a/engine/crates/xerj-engine/examples/alloc_count.rs b/engine/crates/xerj-engine/examples/alloc_count.rs
index 435418fc9..dd779f2ce 100644
--- a/engine/crates/xerj-engine/examples/alloc_count.rs
+++ b/engine/crates/xerj-engine/examples/alloc_count.rs
@@ -51,8 +51,12 @@ fn main() {
let reg = AnalyzerRegistry::default();
let analyzer = reg.get_analyzer("standard").unwrap();
let schema = Schema::default();
- let mut inputs: Vec<(String, Arc<Value>, Vec<(String, Vec<xerj_fts::analyzer::Token>)>)> =
- Vec::with_capacity(n_docs);
+ type DocInput = (
+ String,
+ Arc<Value>,
+ Vec<(String, Vec<xerj_fts::analyzer::Token>)>,
+ );
+ let mut inputs: Vec<DocInput> = Vec::with_capacity(n_docs);
for i in 0..n_docs {
let line = &raw[i % raw.len()];
let v: Value = serde_json::from_str(line).unwrap();
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index 8e3fec3fd..b8acfb396 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -52,19 +52,29 @@ struct FieldCache<'d> {
impl<'d> FieldCache<'d> {
fn new(docs: &'d [Value]) -> Self {
- Self { docs, numeric: HashMap::new(), strings: HashMap::new() }
+ Self {
+ docs,
+ numeric: HashMap::new(),
+ strings: HashMap::new(),
+ }
}
fn get_numeric(&mut self, field: &str) -> &[Option<f64>] {
- self.numeric
- .entry(field.to_string())
- .or_insert_with(|| self.docs.iter().map(|d| extract_numeric(d, field)).collect())
+ self.numeric.entry(field.to_string()).or_insert_with(|| {
+ self.docs
+ .iter()
+ .map(|d| extract_numeric(d, field))
+ .collect()
+ })
}
fn get_strings(&mut self, field: &str) -> &[Vec<String>] {
- self.strings
- .entry(field.to_string())
- .or_insert_with(|| self.docs.iter().map(|d| extract_field_values(d, field)).collect())
+ self.strings.entry(field.to_string()).or_insert_with(|| {
+ self.docs
+ .iter()
+ .map(|d| extract_field_values(d, field))
+ .collect()
+ })
}
}
@@ -99,8 +109,7 @@ pub fn run_aggs(aggs_def: &Value, docs: &[Value]) -> Value {
// Set once at Index construction via `set_max_buckets`; defaults to the same
// 65 536 ES uses if never set. Atomic + Relaxed: read in hot agg loops, write
// once at startup, no ordering needed.
-static MAX_BUCKETS: std::sync::atomic::AtomicUsize =
- std::sync::atomic::AtomicUsize::new(65_536);
+static MAX_BUCKETS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(65_536);
/// Override the per-aggregation bucket cap. Call once at Index startup with
/// `config.limits.max_buckets`. Subsequent calls overwrite the previous value.
@@ -121,12 +130,12 @@ thread_local! {
/// layer. Set by the search handler via `set_outer_query_terms`
/// before calling `run_aggs_with_all`; cleared afterwards.
pub static OUTER_QUERY_TERMS: std::cell::RefCell<Vec<String>> =
- std::cell::RefCell::new(Vec::new());
+ const { std::cell::RefCell::new(Vec::new()) };
/// (field, term) pairs extracted from the outer Match/Term queries.
/// Used by run_top_hits to build per-hit _explanation output when
/// the caller sets `explain: true` on top_hits.
pub static OUTER_QUERY_FIELD_TERMS: std::cell::RefCell<Vec<(String, String)>> =
- std::cell::RefCell::new(Vec::new());
+ const { std::cell::RefCell::new(Vec::new()) };
}
pub fn set_outer_query_terms(terms: Vec<String>) {
@@ -165,7 +174,8 @@ pub fn run_aggs_with_all(aggs_def: &Value, docs: &[Value], all_docs: &[Value]) -
let mut result = Map::new();
for (agg_name, agg_body) in obj {
- let mut agg_result = execute_agg_with_all_cached(agg_name, agg_body, docs, all_docs, &mut cache);
+ let mut agg_result =
+ execute_agg_with_all_cached(agg_name, agg_body, docs, all_docs, &mut cache);
if let Some(meta) = agg_body.get("meta") {
if let Some(res_obj) = agg_result.as_object_mut() {
res_obj.insert("meta".into(), meta.clone());
@@ -188,7 +198,11 @@ pub fn run_aggs_with_all(aggs_def: &Value, docs: &[Value], all_docs: &[Value]) -
pub(crate) fn resolve_sibling_pipelines(result: &mut Map<String, Value>) {
let pipeline_keys: Vec<String> = result
.iter()
- .filter(|(_, v)| v.get("__pipeline__").and_then(Value::as_bool).unwrap_or(false))
+ .filter(|(_, v)| {
+ v.get("__pipeline__")
+ .and_then(Value::as_bool)
+ .unwrap_or(false)
+ })
.map(|(k, _)| k.clone())
.collect();
@@ -197,11 +211,14 @@ pub(crate) fn resolve_sibling_pipelines(result: &mut Map<String, Value>) {
let spec = &result[&key];
let t = spec["type"].as_str().unwrap_or("").to_string();
let bp = spec["buckets_path"].as_str().unwrap_or("").to_string();
- let format = spec.get("format").and_then(Value::as_str).map(str::to_string);
+ let format = spec
+ .get("format")
+ .and_then(Value::as_str)
+ .map(str::to_string);
let extra = spec.clone();
(t, bp, extra, format)
};
- let mut resolved = resolve_pipeline_agg_full(&agg_type, &buckets_path, &extra, &result);
+ let mut resolved = resolve_pipeline_agg_full(&agg_type, &buckets_path, &extra, result);
if let Some(fmt) = format {
apply_pipeline_value_format(&mut resolved, &fmt);
}
@@ -240,40 +257,64 @@ fn resolve_pipeline_agg_full(
resolve_moving_avg_full(buckets_path, window, siblings)
}
"bucket_selector" => {
- let script = spec.get("script")
- .and_then(|v| v.as_str().or_else(|| v.get("source").and_then(|s| s.as_str())))
+ let script = spec
+ .get("script")
+ .and_then(|v| {
+ v.as_str()
+ .or_else(|| v.get("source").and_then(|s| s.as_str()))
+ })
.unwrap_or("")
.to_string();
resolve_bucket_selector(buckets_path, &script, siblings)
}
"bucket_sort" => {
let sort = spec.get("sort").cloned();
- let size = spec.get("size").and_then(|v| v.as_u64()).map(|n| n as usize);
+ let size = spec
+ .get("size")
+ .and_then(|v| v.as_u64())
+ .map(|n| n as usize);
resolve_bucket_sort(buckets_path, sort.as_ref(), size, siblings)
}
"bucket_script" => {
let script_val = spec.get("script");
let script = script_val
- .and_then(|v| v.as_str().or_else(|| v.get("source").and_then(|s| s.as_str())))
+ .and_then(|v| {
+ v.as_str()
+ .or_else(|| v.get("source").and_then(|s| s.as_str()))
+ })
.unwrap_or("")
.to_string();
// Script-provided constant params (e.g. `script.params.extra = 1`).
let extra_params: HashMap<String, f64> = script_val
.and_then(|v| v.get("params"))
.and_then(|v| v.as_object())
- .map(|o| o.iter().filter_map(|(k, v)| v.as_f64().map(|f| (k.clone(), f))).collect())
+ .map(|o| {
+ o.iter()
+ .filter_map(|(k, v)| v.as_f64().map(|f| (k.clone(), f)))
+ .collect()
+ })
.unwrap_or_default();
- let bp_full = spec.get("buckets_path_full").cloned().unwrap_or(Value::Null);
+ let bp_full = spec
+ .get("buckets_path_full")
+ .cloned()
+ .unwrap_or(Value::Null);
resolve_bucket_script(&bp_full, &script, &extra_params, siblings)
}
_ => {
- let gap_policy = spec.get("gap_policy").and_then(Value::as_str).unwrap_or("skip");
+ let gap_policy = spec
+ .get("gap_policy")
+ .and_then(Value::as_str)
+ .unwrap_or("skip");
resolve_pipeline_agg_with_gap(agg_type, buckets_path, siblings, gap_policy)
}
}
}
-fn resolve_moving_avg_full(buckets_path: &str, window: usize, siblings: &Map<String, Value>) -> Value {
+fn resolve_moving_avg_full(
+ buckets_path: &str,
+ window: usize,
+ siblings: &Map<String, Value>,
+) -> Value {
let (sibling_name, metric_name) = split_buckets_path(buckets_path);
let sibling = match siblings.get(sibling_name) {
Some(v) => v,
@@ -283,7 +324,11 @@ fn resolve_moving_avg_full(buckets_path: &str, window: usize, siblings: &Map<Str
resolve_moving_avg(&bucket_values, window)
}
-fn resolve_bucket_selector(buckets_path: &str, script: &str, siblings: &Map<String, Value>) -> Value {
+fn resolve_bucket_selector(
+ buckets_path: &str,
+ script: &str,
+ siblings: &Map<String, Value>,
+) -> Value {
// Parse a simple comparison from the script string:
// "params.count > 10" → threshold = 10, operator = ">"
// We look for the sibling bucket array and filter by the parsed condition.
@@ -301,26 +346,31 @@ fn resolve_bucket_selector(buckets_path: &str, script: &str, siblings: &Map<Stri
None => return json!({"buckets": []}),
};
- let filtered: Vec<&Value> = buckets.iter().filter(|b| {
- let val = if let Some(m) = metric_name {
- b.get(m).and_then(|mv| mv.get("value")).and_then(Value::as_f64)
- } else {
- b.get("doc_count").and_then(Value::as_f64)
- };
- if let Some(v) = val {
- match operator {
- ">" => v > threshold,
- ">=" => v >= threshold,
- "<" => v < threshold,
- "<=" => v <= threshold,
- "==" | "=" => (v - threshold).abs() < f64::EPSILON,
- "!=" => (v - threshold).abs() >= f64::EPSILON,
- _ => true,
+ let filtered: Vec<&Value> = buckets
+ .iter()
+ .filter(|b| {
+ let val = if let Some(m) = metric_name {
+ b.get(m)
+ .and_then(|mv| mv.get("value"))
+ .and_then(Value::as_f64)
+ } else {
+ b.get("doc_count").and_then(Value::as_f64)
+ };
+ if let Some(v) = val {
+ match operator {
+ ">" => v > threshold,
+ ">=" => v >= threshold,
+ "<" => v < threshold,
+ "<=" => v <= threshold,
+ "==" | "=" => (v - threshold).abs() < f64::EPSILON,
+ "!=" => (v - threshold).abs() >= f64::EPSILON,
+ _ => true,
+ }
+ } else {
+ false
}
- } else {
- false
- }
- }).collect();
+ })
+ .collect();
json!({"buckets": filtered})
}
@@ -338,7 +388,7 @@ fn resolve_bucket_sort(
};
let mut buckets: Vec<Value> = match sibling.get("buckets").and_then(Value::as_array) {
- Some(b) => b.iter().cloned().collect(),
+ Some(b) => b.to_vec(),
None => return json!({"buckets": []}),
};
@@ -349,8 +399,11 @@ fn resolve_bucket_sort(
for (sort_field, sort_opts) in sort_obj {
let descending = match sort_opts {
Value::String(s) => s == "desc",
- Value::Object(o) => o.get("order")
- .and_then(Value::as_str).map(|v| v == "desc").unwrap_or(false),
+ Value::Object(o) => o
+ .get("order")
+ .and_then(Value::as_str)
+ .map(|v| v == "desc")
+ .unwrap_or(false),
_ => false,
};
let sf = sort_field.clone();
@@ -358,7 +411,11 @@ fn resolve_bucket_sort(
let av = extract_bucket_value(a, &sf);
let bv = extract_bucket_value(b, &sf);
let cmp = av.partial_cmp(&bv).unwrap_or(std::cmp::Ordering::Equal);
- if descending { cmp.reverse() } else { cmp }
+ if descending {
+ cmp.reverse()
+ } else {
+ cmp
+ }
});
}
}
@@ -378,7 +435,10 @@ fn resolve_bucket_sort(
/// they operate on the PARENT's bucket list (dropping/reordering entries).
/// Call this at the tail of each multi-bucket agg, after all buckets have
/// been built and populated with sub-agg metrics.
-pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Option<&Value>) -> Vec<Value> {
+pub(crate) fn apply_bucket_pipeline_ops(
+ mut buckets: Vec<Value>,
+ sub_aggs: Option<&Value>,
+) -> Vec<Value> {
let sub_obj = match sub_aggs.and_then(Value::as_object) {
Some(o) => o,
None => return buckets,
@@ -388,12 +448,17 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
let mut ops: Vec<(String, String, Value)> = Vec::new();
let mut parent_ops: Vec<(String, String, Value)> = Vec::new();
for (sub_name, sub_spec) in sub_obj {
- let spec_obj = match sub_spec.as_object() { Some(o) => o, None => continue };
+ let spec_obj = match sub_spec.as_object() {
+ Some(o) => o,
+ None => continue,
+ };
for (agg_type, params) in spec_obj {
if agg_type == "bucket_sort" || agg_type == "bucket_selector" {
ops.push((sub_name.clone(), agg_type.clone(), params.clone()));
- } else if matches!(agg_type.as_str(),
- "serial_diff" | "derivative" | "moving_avg" | "moving_fn" | "cumulative_sum") {
+ } else if matches!(
+ agg_type.as_str(),
+ "serial_diff" | "derivative" | "moving_avg" | "moving_fn" | "cumulative_sum"
+ ) {
parent_ops.push((sub_name.clone(), agg_type.clone(), params.clone()));
}
}
@@ -403,11 +468,15 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
// moving_fn, cumulative_sum): each consumes the metric series across
// the parent's buckets and writes one `{value: N}` per bucket.
for (sub_name, agg_type, spec) in &parent_ops {
- let bp = spec.get("buckets_path").and_then(Value::as_str).unwrap_or("");
- let fmt = spec.get("format").and_then(Value::as_str).map(str::to_string);
- let values: Vec<Option<f64>> = buckets.iter()
- .map(|b| resolve_metric_path(b, bp))
- .collect();
+ let bp = spec
+ .get("buckets_path")
+ .and_then(Value::as_str)
+ .unwrap_or("");
+ let fmt = spec
+ .get("format")
+ .and_then(Value::as_str)
+ .map(str::to_string);
+ let values: Vec<Option<f64>> = buckets.iter().map(|b| resolve_metric_path(b, bp)).collect();
let lag = spec.get("lag").and_then(Value::as_u64).unwrap_or(1) as usize;
let window = spec.get("window").and_then(Value::as_u64).unwrap_or(5) as usize;
for (i, bucket) in buckets.iter_mut().enumerate() {
@@ -418,7 +487,9 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
(Some(a), Some(b)) => Some(a - b),
_ => None,
}
- } else { None }
+ } else {
+ None
+ }
}
"derivative" => {
if i >= 1 {
@@ -426,13 +497,18 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
(Some(a), Some(b)) => Some(a - b),
_ => None,
}
- } else { None }
+ } else {
+ None
+ }
}
"moving_avg" | "moving_fn" => {
// `shift` advances the window by N positions. Defaults
// to 0 (window covers the buckets *before* the current).
let shift = spec.get("shift").and_then(Value::as_i64).unwrap_or(0);
- let gap_policy = spec.get("gap_policy").and_then(Value::as_str).unwrap_or("skip");
+ let gap_policy = spec
+ .get("gap_policy")
+ .and_then(Value::as_str)
+ .unwrap_or("skip");
let i_signed = i as i64;
let win_end_signed = i_signed + shift;
if win_end_signed <= 0 {
@@ -448,7 +524,10 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
let win_end = (win_end_signed as usize).min(values.len());
let start = win_end.saturating_sub(window);
let wnd: Vec<f64> = match gap_policy {
- "insert_zeros" => values[start..win_end].iter().map(|v| v.unwrap_or(0.0)).collect(),
+ "insert_zeros" => values[start..win_end]
+ .iter()
+ .map(|v| v.unwrap_or(0.0))
+ .collect(),
"keep_values" | "keep_value" => {
// keep_values: treat the source stream
// as a dense vec of non-null values —
@@ -456,7 +535,8 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
// window, but it also EMITS a value at
// gap buckets (carried over from the
// prior non-gap bucket's output).
- let dense: Vec<f64> = values[..win_end].iter().filter_map(|v| *v).collect();
+ let dense: Vec<f64> =
+ values[..win_end].iter().filter_map(|v| *v).collect();
let take_from = dense.len().saturating_sub(window);
dense[take_from..].to_vec()
}
@@ -470,18 +550,25 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
// position N doesn't shrink the window
// used by bucket N+1 — it pulls an older
// value in instead.
- let dense: Vec<f64> = values[..win_end].iter().filter_map(|v| *v).collect();
+ let dense: Vec<f64> =
+ values[..win_end].iter().filter_map(|v| *v).collect();
let take_from = dense.len().saturating_sub(window);
dense[take_from..].to_vec()
}
};
- if wnd.is_empty() { None } else {
+ if wnd.is_empty() {
+ None
+ } else {
// moving_fn allows a `script` field naming the
// reducer function. Recognised forms: min, max,
// sum, stdDev, linearWeightedAvg, ewma (alpha
// default 0.3), unweightedAvg (default: avg).
- let op = spec.get("script")
- .and_then(|v| v.as_str().or_else(|| v.get("source").and_then(Value::as_str)))
+ let op = spec
+ .get("script")
+ .and_then(|v| {
+ v.as_str()
+ .or_else(|| v.get("source").and_then(Value::as_str))
+ })
.map(|s| s.to_ascii_lowercase())
.unwrap_or_default();
// Handle simple `fn(values) OP fn(values)`
@@ -500,26 +587,39 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
None
}
};
- let combined: Option<f64> = if op.contains(") - ") || op.contains(")-") {
+ let combined: Option<f64> = if op.contains(") - ") || op.contains(")-")
+ {
let idx = op.find(") - ").unwrap_or_else(|| op.find(")-").unwrap());
let (l, r) = op.split_at(idx);
let (a, b) = (eval_side(l), eval_side(r));
- match (a, b) { (Some(x), Some(y)) => Some(x - y), _ => None }
+ match (a, b) {
+ (Some(x), Some(y)) => Some(x - y),
+ _ => None,
+ }
} else if op.contains(") + ") || op.contains(")+") {
let idx = op.find(") + ").unwrap_or_else(|| op.find(")+").unwrap());
let (l, r) = op.split_at(idx);
let (a, b) = (eval_side(l), eval_side(r));
- match (a, b) { (Some(x), Some(y)) => Some(x + y), _ => None }
+ match (a, b) {
+ (Some(x), Some(y)) => Some(x + y),
+ _ => None,
+ }
} else if op.contains(") * ") || op.contains(")*") {
let idx = op.find(") * ").unwrap_or_else(|| op.find(")*").unwrap());
let (l, r) = op.split_at(idx);
let (a, b) = (eval_side(l), eval_side(r));
- match (a, b) { (Some(x), Some(y)) => Some(x * y), _ => None }
+ match (a, b) {
+ (Some(x), Some(y)) => Some(x * y),
+ _ => None,
+ }
} else if op.contains(") / ") || op.contains(")/") {
let idx = op.find(") / ").unwrap_or_else(|| op.find(")/").unwrap());
let (l, r) = op.split_at(idx);
let (a, b) = (eval_side(l), eval_side(r));
- match (a, b) { (Some(x), Some(y)) if y != 0.0 => Some(x / y), _ => None }
+ match (a, b) {
+ (Some(x), Some(y)) if y != 0.0 => Some(x / y),
+ _ => None,
+ }
} else {
None
};
@@ -535,13 +635,16 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
Some(wnd.iter().sum::<f64>())
} else if op.contains("stddev") {
let mean = wnd.iter().sum::<f64>() / wnd.len() as f64;
- let var = wnd.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / wnd.len() as f64;
+ let var = wnd.iter().map(|x| (x - mean).powi(2)).sum::<f64>()
+ / wnd.len() as f64;
Some(var.sqrt())
} else if op.contains("linearweighted") {
// Weight i+1 across window [v_0,...v_{n-1}].
let n = wnd.len() as f64;
let total_w = n * (n + 1.0) / 2.0;
- let weighted: f64 = wnd.iter().enumerate()
+ let weighted: f64 = wnd
+ .iter()
+ .enumerate()
.map(|(idx, v)| v * (idx as f64 + 1.0))
.sum();
Some(weighted / total_w)
@@ -575,7 +678,9 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
};
let result = match computed {
Some(v) if v.is_finite() => {
- let num = serde_json::Number::from_f64(v).map(Value::Number).unwrap_or(Value::Null);
+ let num = serde_json::Number::from_f64(v)
+ .map(Value::Number)
+ .unwrap_or(Value::Null);
let mut m = serde_json::Map::new();
m.insert("value".to_string(), num);
if let Some(ref f) = fmt {
@@ -598,34 +703,59 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
}
}
- if ops.is_empty() { return buckets; }
+ if ops.is_empty() {
+ return buckets;
+ }
// Apply bucket_selector: drop buckets whose script evaluates to false.
for (_, agg_type, spec) in &ops {
- if agg_type != "bucket_selector" { continue; }
- let bp_obj = spec.get("buckets_path").and_then(Value::as_object).cloned().unwrap_or_default();
- let script_str = spec.get("script")
- .and_then(|v| v.as_str().or_else(|| v.get("source").and_then(Value::as_str)))
+ if agg_type != "bucket_selector" {
+ continue;
+ }
+ let bp_obj = spec
+ .get("buckets_path")
+ .and_then(Value::as_object)
+ .cloned()
+ .unwrap_or_default();
+ let script_str = spec
+ .get("script")
+ .and_then(|v| {
+ v.as_str()
+ .or_else(|| v.get("source").and_then(Value::as_str))
+ })
.unwrap_or("")
.to_string();
- let extra_params: HashMap<String, f64> = spec.get("script")
+ let extra_params: HashMap<String, f64> = spec
+ .get("script")
.and_then(|v| v.get("params"))
.and_then(|v| v.as_object())
- .map(|o| o.iter().filter_map(|(k, v)| v.as_f64().map(|f| (k.clone(), f))).collect())
+ .map(|o| {
+ o.iter()
+ .filter_map(|(k, v)| v.as_f64().map(|f| (k.clone(), f)))
+ .collect()
+ })
.unwrap_or_default();
- let gap_policy = spec.get("gap_policy").and_then(Value::as_str).unwrap_or("skip");
+ let gap_policy = spec
+ .get("gap_policy")
+ .and_then(Value::as_str)
+ .unwrap_or("skip");
buckets.retain(|bucket| {
let mut params = extra_params.clone();
for (alias, target) in &bp_obj {
- let tstr = match target.as_str() { Some(s) => s, None => return false };
+ let tstr = match target.as_str() {
+ Some(s) => s,
+ None => return false,
+ };
match get_bucket_metric_value(bucket, tstr) {
- Some(v) => { params.insert(alias.clone(), v); }
- None => {
- match gap_policy {
- "insert_zeros" => { params.insert(alias.clone(), 0.0); }
- _ => return false,
- }
+ Some(v) => {
+ params.insert(alias.clone(), v);
}
+ None => match gap_policy {
+ "insert_zeros" => {
+ params.insert(alias.clone(), 0.0);
+ }
+ _ => return false,
+ },
}
}
match eval_script_expr(&script_str, &params) {
@@ -637,11 +767,17 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
// Apply bucket_sort: gap_policy → reorder → from/size.
for (_, agg_type, spec) in &ops {
- if agg_type != "bucket_sort" { continue; }
+ if agg_type != "bucket_sort" {
+ continue;
+ }
// Collect every sort key referenced by this bucket_sort. Gap handling
// runs across the union of these keys.
- let sort_arr = spec.get("sort").and_then(Value::as_array).cloned().unwrap_or_default();
+ let sort_arr = spec
+ .get("sort")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default();
let sort_keys: Vec<String> = sort_arr
.iter()
.filter_map(|item| item.as_object())
@@ -669,23 +805,43 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
}
for sort_item in sort_arr.iter().rev() {
- let sort_obj = match sort_item.as_object() { Some(o) => o, None => continue };
+ let sort_obj = match sort_item.as_object() {
+ Some(o) => o,
+ None => continue,
+ };
for (sort_field, sort_opts) in sort_obj {
let desc = match sort_opts {
Value::String(s) => s == "desc",
- Value::Object(o) => o.get("order")
- .and_then(Value::as_str).map(|v| v == "desc").unwrap_or(false),
+ Value::Object(o) => o
+ .get("order")
+ .and_then(Value::as_str)
+ .map(|v| v == "desc")
+ .unwrap_or(false),
_ => false,
};
let sf = sort_field.clone();
let use_zero_for_missing = gap_policy == "insert_zeros";
buckets.sort_by(|a, b| {
- let av = extract_bucket_value_opt(a, &sf)
- .unwrap_or(if use_zero_for_missing { 0.0 } else if desc { f64::NEG_INFINITY } else { f64::INFINITY });
- let bv = extract_bucket_value_opt(b, &sf)
- .unwrap_or(if use_zero_for_missing { 0.0 } else if desc { f64::NEG_INFINITY } else { f64::INFINITY });
+ let av = extract_bucket_value_opt(a, &sf).unwrap_or(if use_zero_for_missing {
+ 0.0
+ } else if desc {
+ f64::NEG_INFINITY
+ } else {
+ f64::INFINITY
+ });
+ let bv = extract_bucket_value_opt(b, &sf).unwrap_or(if use_zero_for_missing {
+ 0.0
+ } else if desc {
+ f64::NEG_INFINITY
+ } else {
+ f64::INFINITY
+ });
let cmp = av.partial_cmp(&bv).unwrap_or(std::cmp::Ordering::Equal);
- if desc { cmp.reverse() } else { cmp }
+ if desc {
+ cmp.reverse()
+ } else {
+ cmp
+ }
});
}
}
@@ -706,7 +862,9 @@ pub(crate) fn apply_bucket_pipeline_ops(mut buckets: Vec<Value>, sub_aggs: Optio
// not echo them back in the output).
for bucket in buckets.iter_mut() {
if let Some(bo) = bucket.as_object_mut() {
- for (name, _, _) in &ops { bo.remove(name); }
+ for (name, _, _) in &ops {
+ bo.remove(name);
+ }
}
}
@@ -746,7 +904,8 @@ fn get_bucket_metric_value(bucket: &Value, path: &str) -> Option<f64> {
}
if let Some(rest) = inner.strip_prefix("fields.") {
let arr_path = format!("/fields/{}", rest);
- return top_hit.pointer(&arr_path)
+ return top_hit
+ .pointer(&arr_path)
.and_then(|v| v.as_array())
.and_then(|a| a.first())
.and_then(Value::as_f64);
@@ -757,9 +916,11 @@ fn get_bucket_metric_value(bucket: &Value, path: &str) -> Option<f64> {
let (sib, metric) = split_buckets_path(path);
let sib_val = bucket.get(sib)?;
if let Some(m) = metric {
- sib_val
- .get(m)
- .and_then(|x| x.get("value").and_then(Value::as_f64).or_else(|| x.as_f64()))
+ sib_val.get(m).and_then(|x| {
+ x.get("value")
+ .and_then(Value::as_f64)
+ .or_else(|| x.as_f64())
+ })
} else {
sib_val.get("value").and_then(Value::as_f64)
}
@@ -802,10 +963,10 @@ fn parse_simple_script(script: &str) -> (&'static str, f64) {
">=" => ">=",
"<=" => "<=",
"!=" => "!=",
- ">" => ">",
- "<" => "<",
+ ">" => ">",
+ "<" => "<",
"==" => "==",
- "=" => "=",
+ "=" => "=",
_ => ">",
};
return (op_str, threshold);
@@ -834,16 +995,27 @@ fn resolve_bucket_script(
};
let mut params: HashMap<String, f64> = extra_params.clone();
for (alias, target) in bp {
- let t = match target.as_str() { Some(s) => s, None => continue };
+ let t = match target.as_str() {
+ Some(s) => s,
+ None => continue,
+ };
// Walk "sibling>metric" if nested, else just ".value".
let (sib_name, metric) = split_buckets_path(t);
- let sib = match siblings.get(sib_name) { Some(v) => v, None => return json!({"value": Value::Null}) };
+ let sib = match siblings.get(sib_name) {
+ Some(v) => v,
+ None => return json!({"value": Value::Null}),
+ };
let v = if let Some(m) = metric {
- sib.get(m).and_then(|x| x.get("value")).and_then(Value::as_f64)
+ sib.get(m)
+ .and_then(|x| x.get("value"))
+ .and_then(Value::as_f64)
} else {
sib.get("value").and_then(Value::as_f64)
};
- let v = match v { Some(x) => x, None => return json!({"value": Value::Null}) };
+ let v = match v {
+ Some(x) => x,
+ None => return json!({"value": Value::Null}),
+ };
params.insert(alias.clone(), v);
}
match eval_script_expr(script, &params) {
@@ -865,7 +1037,12 @@ fn eval_script_expr(script: &str, params: &HashMap<String, f64>) -> Option<f64>
}
#[derive(Debug, Clone)]
-enum Tok { Num(f64), Op(&'static str), LParen, RParen }
+enum Tok {
+ Num(f64),
+ Op(&'static str),
+ LParen,
+ RParen,
+}
fn tokenize_script(s: &str, params: &HashMap<String, f64>) -> Option<Vec<Tok>> {
let bytes = s.as_bytes();
@@ -873,10 +1050,17 @@ fn tokenize_script(s: &str, params: &HashMap<String, f64>) -> Option<Vec<Tok>> {
let mut i = 0;
while i < bytes.len() {
let c = bytes[i] as char;
- if c.is_whitespace() { i += 1; continue; }
- if c.is_ascii_digit() || (c == '.' && i + 1 < bytes.len() && (bytes[i+1] as char).is_ascii_digit()) {
+ if c.is_whitespace() {
+ i += 1;
+ continue;
+ }
+ if c.is_ascii_digit()
+ || (c == '.' && i + 1 < bytes.len() && (bytes[i + 1] as char).is_ascii_digit())
+ {
let start = i;
- while i < bytes.len() && ((bytes[i] as char).is_ascii_digit() || bytes[i] as char == '.') {
+ while i < bytes.len()
+ && ((bytes[i] as char).is_ascii_digit() || bytes[i] as char == '.')
+ {
i += 1;
}
let n: f64 = s[start..i].parse().ok()?;
@@ -885,7 +1069,11 @@ fn tokenize_script(s: &str, params: &HashMap<String, f64>) -> Option<Vec<Tok>> {
}
if c.is_ascii_alphabetic() || c == '_' {
let start = i;
- while i < bytes.len() && ((bytes[i] as char).is_ascii_alphanumeric() || bytes[i] as char == '_' || bytes[i] as char == '.') {
+ while i < bytes.len()
+ && ((bytes[i] as char).is_ascii_alphanumeric()
+ || bytes[i] as char == '_'
+ || bytes[i] as char == '.')
+ {
i += 1;
}
let ident = &s[start..i];
@@ -901,12 +1089,17 @@ fn tokenize_script(s: &str, params: &HashMap<String, f64>) -> Option<Vec<Tok>> {
}
// Two-character operators first.
if i + 1 < bytes.len() {
- let two = &s[i..i+2];
+ let two = &s[i..i + 2];
match two {
">=" | "<=" | "==" | "!=" | "&&" | "||" => {
let op: &'static str = match two {
- ">=" => ">=", "<=" => "<=", "==" => "==", "!=" => "!=",
- "&&" => "&&", "||" => "||", _ => unreachable!(),
+ ">=" => ">=",
+ "<=" => "<=",
+ "==" => "==",
+ "!=" => "!=",
+ "&&" => "&&",
+ "||" => "||",
+ _ => unreachable!(),
};
toks.push(Tok::Op(op));
i += 2;
@@ -916,15 +1109,33 @@ fn tokenize_script(s: &str, params: &HashMap<String, f64>) -> Option<Vec<Tok>> {
}
}
match c {
- '+' => { toks.push(Tok::Op("+")); }
- '-' => { toks.push(Tok::Op("-")); }
- '*' => { toks.push(Tok::Op("*")); }
- '/' => { toks.push(Tok::Op("/")); }
- '%' => { toks.push(Tok::Op("%")); }
- '<' => { toks.push(Tok::Op("<")); }
- '>' => { toks.push(Tok::Op(">")); }
- '(' => { toks.push(Tok::LParen); }
- ')' => { toks.push(Tok::RParen); }
+ '+' => {
+ toks.push(Tok::Op("+"));
+ }
+ '-' => {
+ toks.push(Tok::Op("-"));
+ }
+ '*' => {
+ toks.push(Tok::Op("*"));
+ }
+ '/' => {
+ toks.push(Tok::Op("/"));
+ }
+ '%' => {
+ toks.push(Tok::Op("%"));
+ }
+ '<' => {
+ toks.push(Tok::Op("<"));
+ }
+ '>' => {
+ toks.push(Tok::Op(">"));
+ }
+ '(' => {
+ toks.push(Tok::LParen);
+ }
+ ')' => {
+ toks.push(Tok::RParen);
+ }
';' => {} // ignore trailing semicolons
_ => return None,
}
@@ -953,8 +1164,11 @@ fn shunting_yard(toks: &[Tok]) -> Option<Vec<Tok>> {
Tok::Num(_) => out.push(t.clone()),
Tok::Op(o) => {
while let Some(Tok::Op(top)) = ops.last() {
- if precedence(top) >= precedence(o) { out.push(ops.pop().unwrap()); }
- else { break; }
+ if precedence(top) >= precedence(o) {
+ out.push(ops.pop().unwrap());
+ } else {
+ break;
+ }
}
ops.push(t.clone());
}
@@ -970,7 +1184,9 @@ fn shunting_yard(toks: &[Tok]) -> Option<Vec<Tok>> {
}
}
while let Some(op) = ops.pop() {
- if matches!(op, Tok::LParen | Tok::RParen) { return None; }
+ if matches!(op, Tok::LParen | Tok::RParen) {
+ return None;
+ }
out.push(op);
}
Some(out)
@@ -985,18 +1201,79 @@ fn evaluate_rpn(rpn: &[Tok]) -> Option<f64> {
let b = stack.pop()?;
let a = stack.pop()?;
let r = match *op {
- "+" => a + b, "-" => a - b,
+ "+" => a + b,
+ "-" => a - b,
"*" => a * b,
- "/" => if b == 0.0 { return None; } else { a / b },
- "%" => if b == 0.0 { return None; } else { a % b },
- "<" => if a < b { 1.0 } else { 0.0 },
- ">" => if a > b { 1.0 } else { 0.0 },
- "<=" => if a <= b { 1.0 } else { 0.0 },
- ">=" => if a >= b { 1.0 } else { 0.0 },
- "==" => if (a - b).abs() < 1e-9 { 1.0 } else { 0.0 },
- "!=" => if (a - b).abs() >= 1e-9 { 1.0 } else { 0.0 },
- "&&" => if a != 0.0 && b != 0.0 { 1.0 } else { 0.0 },
- "||" => if a != 0.0 || b != 0.0 { 1.0 } else { 0.0 },
+ "/" => {
+ if b == 0.0 {
+ return None;
+ } else {
+ a / b
+ }
+ }
+ "%" => {
+ if b == 0.0 {
+ return None;
+ } else {
+ a % b
+ }
+ }
+ "<" => {
+ if a < b {
+ 1.0
+ } else {
+ 0.0
+ }
+ }
+ ">" => {
+ if a > b {
+ 1.0
+ } else {
+ 0.0
+ }
+ }
+ "<=" => {
+ if a <= b {
+ 1.0
+ } else {
+ 0.0
+ }
+ }
+ ">=" => {
+ if a >= b {
+ 1.0
+ } else {
+ 0.0
+ }
+ }
+ "==" => {
+ if (a - b).abs() < 1e-9 {
+ 1.0
+ } else {
+ 0.0
+ }
+ }
+ "!=" => {
+ if (a - b).abs() >= 1e-9 {
+ 1.0
+ } else {
+ 0.0
+ }
+ }
+ "&&" => {
+ if a != 0.0 && b != 0.0 {
+ 1.0
+ } else {
+ 0.0
+ }
+ }
+ "||" => {
+ if a != 0.0 || b != 0.0 {
+ 1.0
+ } else {
+ 0.0
+ }
+ }
_ => return None,
};
stack.push(r);
@@ -1004,7 +1281,11 @@ fn evaluate_rpn(rpn: &[Tok]) -> Option<f64> {
_ => return None,
}
}
- if stack.len() == 1 { Some(stack[0]) } else { None }
+ if stack.len() == 1 {
+ Some(stack[0])
+ } else {
+ None
+ }
}
fn split_buckets_path(buckets_path: &str) -> (&str, Option<&str>) {
@@ -1035,7 +1316,11 @@ fn collect_bucket_values(sibling: &Value, metric_name: Option<&str>) -> Vec<f64>
///
/// `metric_name` supports dotted paths (`stats.sum`) and subscript notation
/// (`percentiles[99.9]`, `extended_stats[std_deviation]`).
-fn collect_bucket_values_with_policy(sibling: &Value, metric_name: Option<&str>, gap_policy: &str) -> Vec<f64> {
+fn collect_bucket_values_with_policy(
+ sibling: &Value,
+ metric_name: Option<&str>,
+ gap_policy: &str,
+) -> Vec<f64> {
let buckets = match sibling.get("buckets").and_then(Value::as_array) {
Some(b) => b,
None => return vec![],
@@ -1084,14 +1369,22 @@ fn resolve_metric_path(bucket: &Value, path: &str) -> Option<f64> {
let key = &path[open + 1..open + close];
// Try both "values.<key>" (percentiles) and direct child.
let node = bucket.get(base)?;
- if let Some(v) = node.get("values").and_then(|vals| vals.get(key)).and_then(Value::as_f64) {
+ if let Some(v) = node
+ .get("values")
+ .and_then(|vals| vals.get(key))
+ .and_then(Value::as_f64)
+ {
return Some(v);
}
// Numeric percentile rendered as number (not string)
if let Ok(n) = key.parse::<f64>() {
if let Some(vals) = node.get("values").and_then(Value::as_object) {
for (k, v) in vals {
- if k.parse::<f64>().ok().map(|x| (x - n).abs() < 1e-6).unwrap_or(false) {
+ if k.parse::<f64>()
+ .ok()
+ .map(|x| (x - n).abs() < 1e-6)
+ .unwrap_or(false)
+ {
return v.as_f64();
}
}
@@ -1123,9 +1416,7 @@ fn resolve_metric_path(bucket: &Value, path: &str) -> Option<f64> {
let k = entry.get("key").and_then(Value::as_f64);
if let Some(k) = k {
if (k - key_num).abs() < 1e-6 {
- return entry
- .get("value")
- .and_then(Value::as_f64);
+ return entry.get("value").and_then(Value::as_f64);
}
}
}
@@ -1167,10 +1458,16 @@ fn resolve_metric_path(bucket: &Value, path: &str) -> Option<f64> {
if let Some(values) = node.get("values") {
match values {
Value::Object(map) if map.len() == 1 => {
- if let Some((_, v)) = map.iter().next() { if let Some(f) = v.as_f64() { return Some(f); } }
+ if let Some((_, v)) = map.iter().next() {
+ if let Some(f) = v.as_f64() {
+ return Some(f);
+ }
+ }
}
Value::Array(arr) if arr.len() == 1 => {
- if let Some(f) = arr[0].get("value").and_then(Value::as_f64) { return Some(f); }
+ if let Some(f) = arr[0].get("value").and_then(Value::as_f64) {
+ return Some(f);
+ }
}
_ => {}
}
@@ -1203,7 +1500,8 @@ fn collect_bucket_keys_matching_value(
};
if let Some(v) = v {
if (v - target_value).abs() < 1e-9 {
- let k = b.get("key_as_string")
+ let k = b
+ .get("key_as_string")
.or_else(|| b.get("key"))
.cloned()
.unwrap_or(Value::Null);
@@ -1214,13 +1512,6 @@ fn collect_bucket_keys_matching_value(
out
}
-/// Resolve a pipeline aggregation against already-computed sibling results.
-///
-/// `buckets_path` format: `"sibling_agg>metric_name"` or just `"sibling_agg"`.
-fn resolve_pipeline_agg(agg_type: &str, buckets_path: &str, siblings: &Map<String, Value>) -> Value {
- resolve_pipeline_agg_with_gap(agg_type, buckets_path, siblings, "skip")
-}
-
fn resolve_pipeline_agg_with_gap(
agg_type: &str,
buckets_path: &str,
@@ -1262,7 +1553,10 @@ fn resolve_pipeline_agg_with_gap(
make_value_result(sum / bucket_values.len() as f64)
}
"max_bucket" => {
- let value = bucket_values.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
+ let value = bucket_values
+ .iter()
+ .cloned()
+ .fold(f64::NEG_INFINITY, f64::max);
let keys = collect_bucket_keys_matching_value(sibling, metric_name, value);
let mut r = make_value_result(value);
if let Some(obj) = r.as_object_mut() {
@@ -1284,7 +1578,10 @@ fn resolve_pipeline_agg_with_gap(
let count = bucket_values.len() as f64;
let sum: f64 = bucket_values.iter().sum();
let min = bucket_values.iter().cloned().fold(f64::INFINITY, f64::min);
- let max = bucket_values.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
+ let max = bucket_values
+ .iter()
+ .cloned()
+ .fold(f64::NEG_INFINITY, f64::max);
let avg = sum / count;
json!({
"count": bucket_values.len(),
@@ -1299,7 +1596,10 @@ fn resolve_pipeline_agg_with_gap(
let n = bucket_values.len() as f64;
let sum: f64 = bucket_values.iter().sum();
let min = bucket_values.iter().cloned().fold(f64::INFINITY, f64::min);
- let max = bucket_values.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
+ let max = bucket_values
+ .iter()
+ .cloned()
+ .fold(f64::NEG_INFINITY, f64::max);
let avg = if n > 0.0 { sum / n } else { 0.0 };
let sum_sq: f64 = bucket_values.iter().map(|v| v * v).sum();
let variance_pop = if n > 0.0 { sum_sq / n - avg * avg } else { 0.0 };
@@ -1310,7 +1610,11 @@ fn resolve_pipeline_agg_with_gap(
};
let std_pop = variance_pop.max(0.0).sqrt();
let std_sample = variance_sample.max(0.0).sqrt();
- let num = |x: f64| serde_json::Number::from_f64(x).map(Value::Number).unwrap_or(Value::Null);
+ let num = |x: f64| {
+ serde_json::Number::from_f64(x)
+ .map(Value::Number)
+ .unwrap_or(Value::Null)
+ };
json!({
"count": bucket_values.len(),
"min": num(min),
@@ -1348,11 +1652,15 @@ fn resolve_pipeline_agg_with_gap(
let val = if n == 0 {
Value::Null
} else if n == 1 {
- serde_json::Number::from_f64(sorted[0]).map(Value::Number).unwrap_or(Value::Null)
+ serde_json::Number::from_f64(sorted[0])
+ .map(Value::Number)
+ .unwrap_or(Value::Null)
} else {
let idx = ((p / 100.0) * (n as f64 - 1.0)).round() as usize;
let idx = idx.min(n - 1);
- serde_json::Number::from_f64(sorted[idx]).map(Value::Number).unwrap_or(Value::Null)
+ serde_json::Number::from_f64(sorted[idx])
+ .map(Value::Number)
+ .unwrap_or(Value::Null)
};
values.insert(format!("{:.1}", p), val);
}
@@ -1360,7 +1668,9 @@ fn resolve_pipeline_agg_with_gap(
}
"serial_diff" => {
if bucket_values.len() >= 2 {
- make_value_result(bucket_values[bucket_values.len() - 1] - bucket_values[bucket_values.len() - 2])
+ make_value_result(
+ bucket_values[bucket_values.len() - 1] - bucket_values[bucket_values.len() - 2],
+ )
} else {
json!({"value": Value::Null})
}
@@ -1368,8 +1678,7 @@ fn resolve_pipeline_agg_with_gap(
"derivative" => {
if bucket_values.len() >= 2 {
make_value_result(
- bucket_values[bucket_values.len() - 1]
- - bucket_values[bucket_values.len() - 2],
+ bucket_values[bucket_values.len() - 1] - bucket_values[bucket_values.len() - 2],
)
} else {
json!({"value": Value::Null})
@@ -1394,7 +1703,9 @@ fn make_value_result(value: f64) -> Value {
/// rendering. Supports `"0.00"`, `"#,##0.00"`, `"0.##"` — the minimal set
/// the YAML tests exercise — via `format_number_pattern`.
fn apply_pipeline_value_format(result: &mut Value, fmt: &str) {
- let Some(obj) = result.as_object_mut() else { return };
+ let Some(obj) = result.as_object_mut() else {
+ return;
+ };
if let Some(Value::Number(n)) = obj.get("value").cloned() {
if let Some(f) = n.as_f64() {
obj.insert(
@@ -1405,9 +1716,16 @@ fn apply_pipeline_value_format(result: &mut Value, fmt: &str) {
}
// `stats_bucket` / `extended_stats_bucket` — format each numeric stat.
for key in [
- "min", "max", "avg", "sum",
- "variance", "variance_population", "variance_sampling",
- "std_deviation", "std_deviation_population", "std_deviation_sampling",
+ "min",
+ "max",
+ "avg",
+ "sum",
+ "variance",
+ "variance_population",
+ "variance_sampling",
+ "std_deviation",
+ "std_deviation_population",
+ "std_deviation_sampling",
"sum_of_squares",
] {
if let Some(Value::Number(n)) = obj.get(key).cloned() {
@@ -1445,7 +1763,10 @@ fn apply_pipeline_value_format(result: &mut Value, fmt: &str) {
out.insert(k, Value::String(format_number_pattern(f, fmt)));
}
}
- obj.insert("std_deviation_bounds_as_string".to_string(), Value::Object(out));
+ obj.insert(
+ "std_deviation_bounds_as_string".to_string(),
+ Value::Object(out),
+ );
}
}
@@ -1559,7 +1880,14 @@ fn holt(values: &[f64], alpha: f64, beta: f64) -> f64 {
/// initial level = mean of the first full period, initial trend =
/// (mean of 2nd period - mean of 1st period) / period, initial seasonal
/// indices = element-wise residual over the first period.
-fn holt_winters(values: &[f64], alpha: f64, beta: f64, gamma: f64, period: usize, multiplicative: bool) -> Option<f64> {
+fn holt_winters(
+ values: &[f64],
+ alpha: f64,
+ beta: f64,
+ gamma: f64,
+ period: usize,
+ multiplicative: bool,
+) -> Option<f64> {
if period == 0 || values.len() < 2 * period {
return Some(0.0);
}
@@ -1594,15 +1922,21 @@ fn holt_winters(values: &[f64], alpha: f64, beta: f64, gamma: f64, period: usize
}
b = beta * (s - last_s) + (1.0 - beta) * last_b;
if multiplicative {
- seasonal[i] = gamma * (vs[i] / (last_s + last_b)) + (1.0 - gamma) * seasonal[i - period];
+ seasonal[i] =
+ gamma * (vs[i] / (last_s + last_b)) + (1.0 - gamma) * seasonal[i - period];
} else {
- seasonal[i] = gamma * (vs[i] - (last_s - last_b)) + (1.0 - gamma) * seasonal[i - period];
+ seasonal[i] =
+ gamma * (vs[i] - (last_s - last_b)) + (1.0 - gamma) * seasonal[i - period];
}
last_s = s;
last_b = b;
}
let idx = n - period;
- let out = if multiplicative { (s + b) * seasonal[idx] } else { s + b + seasonal[idx] };
+ let out = if multiplicative {
+ (s + b) * seasonal[idx]
+ } else {
+ s + b + seasonal[idx]
+ };
Some(out)
}
@@ -1621,7 +1955,10 @@ pub(crate) fn format_number_pattern(value: f64, pattern: &str) -> String {
// substring — chars in {0,#,.,,} — and keep the rest as literal.
let is_fmt = |c: char| matches!(c, '0' | '#' | '.' | ',');
let start = pattern.find(is_fmt).unwrap_or(pattern.len());
- let end = pattern.rfind(is_fmt).map(|i| i + 1).unwrap_or(pattern.len());
+ let end = pattern
+ .rfind(is_fmt)
+ .map(|i| i + 1)
+ .unwrap_or(pattern.len());
let (prefix, rest) = pattern.split_at(start);
let (active_pat, suffix) = if end >= start {
(&rest[..(end - start)], &pattern[end..])
@@ -1636,7 +1973,10 @@ pub(crate) fn format_number_pattern(value: f64, pattern: &str) -> String {
Some(i) => (&active_pat[..i], &active_pat[i + 1..]),
None => (active_pat, ""),
};
- let frac_digits = frac_part_pat.chars().filter(|c| *c == '0' || *c == '#').count();
+ let frac_digits = frac_part_pat
+ .chars()
+ .filter(|c| *c == '0' || *c == '#')
+ .count();
let fixed_frac_digits = frac_part_pat.chars().filter(|c| *c == '0').count();
let int_min_digits = int_part_pat.chars().filter(|c| *c == '0').count();
let use_grouping = int_part_pat.contains(',');
@@ -1692,7 +2032,7 @@ fn resolve_moving_avg(bucket_values: &[f64], window: usize) -> Value {
}
// Return a per-bucket moving average (only the last bucket's value for the pipeline result).
let n = bucket_values.len();
- let start = if n >= window { n - window } else { 0 };
+ let start = n.saturating_sub(window);
let slice = &bucket_values[start..];
let avg = slice.iter().sum::<f64>() / slice.len() as f64;
make_value_result(avg)
@@ -1708,15 +2048,6 @@ fn resolve_cumulative_sum(bucket_values: &[f64]) -> Value {
// ── Dispatch ──────────────────────────────────────────────────────────────────
-fn execute_agg(name: &str, body: &Value, docs: &[Value]) -> Value {
- execute_agg_with_all(name, body, docs, docs)
-}
-
-fn execute_agg_with_all(name: &str, body: &Value, docs: &[Value], all_docs: &[Value]) -> Value {
- let mut cache = FieldCache::new(docs);
- execute_agg_with_all_cached(name, body, docs, all_docs, &mut cache)
-}
-
fn execute_agg_with_all_cached<'d>(
name: &str,
body: &Value,
@@ -1730,9 +2061,7 @@ fn execute_agg_with_all_cached<'d>(
};
// Sub-aggregations live under the "aggs" (or "aggregations") key.
- let sub_aggs = obj
- .get("aggs")
- .or_else(|| obj.get("aggregations"));
+ let sub_aggs = obj.get("aggs").or_else(|| obj.get("aggregations"));
// Find the aggregation type key (first key that is not "aggs"/"aggregations"/"meta").
let type_key = obj
@@ -1765,7 +2094,9 @@ fn execute_agg_with_all_cached<'d>(
"range" => run_range(agg_params, sub_aggs, docs, all_docs),
"missing" => run_missing(agg_params, docs),
"histogram" => run_histogram(agg_params, sub_aggs, docs, all_docs),
- "variable_width_histogram" => run_variable_width_histogram(agg_params, sub_aggs, docs, all_docs),
+ "variable_width_histogram" => {
+ run_variable_width_histogram(agg_params, sub_aggs, docs, all_docs)
+ }
"percentiles" => run_percentiles(agg_params, docs),
"composite" => run_composite(agg_params, sub_aggs, docs, all_docs),
"significant_terms" => run_significant_terms(agg_params, docs, all_docs),
@@ -1816,12 +2147,21 @@ fn execute_agg_with_all_cached<'d>(
// global — run sub-aggs on ALL docs, regardless of query filter.
"global" => run_global(sub_aggs, all_docs),
// Pipeline aggregations — these are computed in a second pass; return placeholder.
- "avg_bucket" | "max_bucket" | "min_bucket" | "sum_bucket" | "derivative"
- | "moving_avg" | "cumulative_sum" | "bucket_selector" | "bucket_sort"
- | "bucket_script" | "percentiles_bucket" | "stats_bucket" | "serial_diff"
- | "moving_fn" | "extended_stats_bucket" => {
- run_pipeline_agg(agg_type, agg_params)
- }
+ "avg_bucket"
+ | "max_bucket"
+ | "min_bucket"
+ | "sum_bucket"
+ | "derivative"
+ | "moving_avg"
+ | "cumulative_sum"
+ | "bucket_selector"
+ | "bucket_sort"
+ | "bucket_script"
+ | "percentiles_bucket"
+ | "stats_bucket"
+ | "serial_diff"
+ | "moving_fn"
+ | "extended_stats_bucket" => run_pipeline_agg(agg_type, agg_params),
// date_range — like range but with date math
"date_range" => run_date_range(agg_params, sub_aggs, docs, all_docs),
// percentile_ranks — given values, return percentile rank of each
@@ -1864,7 +2204,11 @@ pub(crate) fn run_pipeline_agg(agg_type: &str, params: &Value) -> Value {
Value::String(s) => s.clone(),
Value::Object(o) => {
// Take the first value — e.g. {"count": "_count"} → "_count"
- o.values().next().and_then(|v| v.as_str()).unwrap_or("").to_string()
+ o.values()
+ .next()
+ .and_then(|v| v.as_str())
+ .unwrap_or("")
+ .to_string()
}
_ => String::new(),
};
@@ -1922,7 +2266,7 @@ pub(crate) fn run_pipeline_agg(agg_type: &str, params: &Value) -> Value {
/// Arrays are flattened — each element becomes a separate bucket key.
pub(crate) fn extract_field_values(doc: &Value, field: &str) -> Vec<String> {
let val = get_nested_field(doc, field);
- let mut out = flatten_to_strings(&val);
+ let mut out = flatten_to_strings(val);
if out.is_empty() {
// Array-aware dotted-path fallback: for `users.last` where `users`
// is an array of objects, fan out through each element and pull
@@ -1940,13 +2284,17 @@ pub(crate) fn extract_field_values(doc: &Value, field: &str) -> Vec<String> {
/// `users[]` array.
fn extract_array_path_values(doc: &Value, field: &str) -> Vec<String> {
fn walk(val: &Value, parts: &[&str]) -> Vec<String> {
- if parts.is_empty() { return flatten_to_strings(val); }
+ if parts.is_empty() {
+ return flatten_to_strings(val);
+ }
let head = parts[0];
let rest = &parts[1..];
match val {
Value::Array(arr) => {
let mut out = Vec::new();
- for el in arr { out.extend(walk(el, parts)); }
+ for el in arr {
+ out.extend(walk(el, parts));
+ }
out
}
Value::Object(obj) => {
@@ -2021,7 +2369,7 @@ pub(crate) fn get_nested_field<'a>(doc: &'a Value, field: &str) -> &'a Value {
fn flatten_to_strings(val: &Value) -> Vec<String> {
match val {
- Value::Array(arr) => arr.iter().flat_map(|v| flatten_to_strings(v)).collect(),
+ Value::Array(arr) => arr.iter().flat_map(flatten_to_strings).collect(),
Value::Null => vec![],
Value::String(s) => vec![s.clone()],
Value::Number(n) => vec![n.to_string()],
@@ -2057,9 +2405,21 @@ pub(crate) fn extract_numeric(doc: &Value, field: &str) -> Option<f64> {
fn extract_numeric_values(doc: &Value, field: &str) -> Vec<f64> {
fn walk(v: &Value, out: &mut Vec<f64>) {
match v {
- Value::Number(n) => { if let Some(f) = n.as_f64() { out.push(f); } }
- Value::String(s) => { if let Ok(f) = s.parse::<f64>() { out.push(f); } }
- Value::Array(arr) => { for e in arr { walk(e, out); } }
+ Value::Number(n) => {
+ if let Some(f) = n.as_f64() {
+ out.push(f);
+ }
+ }
+ Value::String(s) => {
+ if let Ok(f) = s.parse::<f64>() {
+ out.push(f);
+ }
+ }
+ Value::Array(arr) => {
+ for e in arr {
+ walk(e, out);
+ }
+ }
_ => {}
}
}
@@ -2072,9 +2432,15 @@ fn extract_numeric_values(doc: &Value, field: &str) -> Vec<f64> {
pub(crate) fn extract_date_ms_values(doc: &Value, field: &str) -> Vec<i64> {
fn walk(v: &Value, out: &mut Vec<i64>) {
match v {
- Value::Array(arr) => { for e in arr { walk(e, out); } }
+ Value::Array(arr) => {
+ for e in arr {
+ walk(e, out);
+ }
+ }
other => {
- if let Some(ms) = parse_date_ms(other) { out.push(ms); }
+ if let Some(ms) = parse_date_ms(other) {
+ out.push(ms);
+ }
}
}
}
@@ -2083,11 +2449,6 @@ pub(crate) fn extract_date_ms_values(doc: &Value, field: &str) -> Vec<i64> {
out
}
-/// Extract numeric with a `missing` fallback value (ES `"missing"` param).
-fn extract_numeric_or_missing(doc: &Value, field: &str, missing: Option<f64>) -> Option<f64> {
- extract_numeric(doc, field).or(missing)
-}
-
/// Extract range-typed values as (from_inclusive, to_exclusive) numeric pairs.
/// Handles `{gte, gt, lte, lt}` objects, including arrays of them. Unbounded
/// sides default to `f64::MIN`/`f64::MAX` so callers can clamp with
@@ -2096,17 +2457,23 @@ fn extract_numeric_ranges(doc: &Value, field: &str) -> Vec<(f64, f64)> {
fn walk(v: &Value, out: &mut Vec<(f64, f64)>) {
match v {
Value::Array(arr) => {
- for e in arr { walk(e, out); }
+ for e in arr {
+ walk(e, out);
+ }
}
Value::Object(obj) => {
let num = |k: &str| obj.get(k).and_then(Value::as_f64);
- let from = num("gte").or_else(|| num("gt").map(|g| g + f64::EPSILON))
+ let from = num("gte")
+ .or_else(|| num("gt").map(|g| g + f64::EPSILON))
.unwrap_or(f64::MIN);
- let to = num("lt").or_else(|| num("lte").map(|l| l + f64::EPSILON))
+ let to = num("lt")
+ .or_else(|| num("lte").map(|l| l + f64::EPSILON))
.unwrap_or(f64::MAX);
// Only treat as a range if at least one of the range keys is present.
- if obj.contains_key("gte") || obj.contains_key("gt")
- || obj.contains_key("lte") || obj.contains_key("lt")
+ if obj.contains_key("gte")
+ || obj.contains_key("gt")
+ || obj.contains_key("lte")
+ || obj.contains_key("lt")
{
out.push((from, to));
}
@@ -2124,14 +2491,22 @@ fn extract_date_ranges(doc: &Value, field: &str) -> Vec<(i64, i64)> {
fn walk(v: &Value, out: &mut Vec<(i64, i64)>) {
match v {
Value::Array(arr) => {
- for e in arr { walk(e, out); }
+ for e in arr {
+ walk(e, out);
+ }
}
Value::Object(obj) => {
let ms = |k: &str| obj.get(k).and_then(parse_date_ms);
- let from = ms("gte").or_else(|| ms("gt").map(|g| g + 1)).unwrap_or(i64::MIN);
- let to = ms("lt").or_else(|| ms("lte").map(|l| l + 1)).unwrap_or(i64::MAX);
- if obj.contains_key("gte") || obj.contains_key("gt")
- || obj.contains_key("lte") || obj.contains_key("lt")
+ let from = ms("gte")
+ .or_else(|| ms("gt").map(|g| g + 1))
+ .unwrap_or(i64::MIN);
+ let to = ms("lt")
+ .or_else(|| ms("lte").map(|l| l + 1))
+ .unwrap_or(i64::MAX);
+ if obj.contains_key("gte")
+ || obj.contains_key("gt")
+ || obj.contains_key("lte")
+ || obj.contains_key("lt")
{
out.push((from, to));
}
@@ -2146,7 +2521,12 @@ fn extract_date_ranges(doc: &Value, field: &str) -> Vec<(i64, i64)> {
// ── Terms aggregation ─────────────────────────────────────────────────────────
-fn run_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_terms(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": []}),
@@ -2159,9 +2539,9 @@ fn run_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs:
.map(|v| v as usize);
// size=0 → no truncation; absent → default 10.
let cap: Option<usize> = match size_opt {
- Some(0) => None, // unlimited
- Some(n) => Some(n), // explicit cap
- None => Some(10), // ES default
+ Some(0) => None, // unlimited
+ Some(n) => Some(n), // explicit cap
+ None => Some(10), // ES default
};
// Count occurrences of each term, honoring the `_doc_count` metadata
@@ -2171,14 +2551,12 @@ fn run_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs:
//
// The `missing` param supplies a placeholder key for docs that don't
// have the field — ES treats it as the field value for bucketing.
- let missing_placeholder: Option<String> = params
- .get("missing")
- .and_then(|v| match v {
- Value::String(s) => Some(s.clone()),
- Value::Number(n) => Some(n.to_string()),
- Value::Bool(b) => Some(b.to_string()),
- _ => None,
- });
+ let missing_placeholder: Option<String> = params.get("missing").and_then(|v| match v {
+ Value::String(s) => Some(s.clone()),
+ Value::Number(n) => Some(n.to_string()),
+ Value::Bool(b) => Some(b.to_string()),
+ _ => None,
+ });
let mut counts: HashMap<String, u64> = HashMap::new();
let bucket_cap = max_buckets();
for doc in docs {
@@ -2261,9 +2639,7 @@ fn run_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs:
//
// When the ordering only references `_count` or `_key`, skip the
// pre-compute — simple sort, cheaper.
- let orders_need_sub_agg = orders
- .iter()
- .any(|(k, _)| k != "_count" && k != "_key");
+ let orders_need_sub_agg = orders.iter().any(|(k, _)| k != "_count" && k != "_key");
let min_doc_count: u64 = params
.get("min_doc_count")
@@ -2287,24 +2663,28 @@ fn run_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs:
// of explicit terms, which we match as literal equality.
let include_regex = params.get("include").and_then(Value::as_str);
let exclude_regex = params.get("exclude").and_then(Value::as_str);
- let include_list: Option<Vec<String>> = params
- .get("include")
- .and_then(Value::as_array)
- .map(|a| a.iter().filter_map(|v| match v {
- Value::String(s) => Some(s.clone()),
- Value::Number(n) => Some(n.to_string()),
- Value::Bool(b) => Some(b.to_string()),
- _ => None,
- }).collect());
- let exclude_list: Option<Vec<String>> = params
- .get("exclude")
- .and_then(Value::as_array)
- .map(|a| a.iter().filter_map(|v| match v {
- Value::String(s) => Some(s.clone()),
- Value::Number(n) => Some(n.to_string()),
- Value::Bool(b) => Some(b.to_string()),
- _ => None,
- }).collect());
+ let include_list: Option<Vec<String>> =
+ params.get("include").and_then(Value::as_array).map(|a| {
+ a.iter()
+ .filter_map(|v| match v {
+ Value::String(s) => Some(s.clone()),
+ Value::Number(n) => Some(n.to_string()),
+ Value::Bool(b) => Some(b.to_string()),
+ _ => None,
+ })
+ .collect()
+ });
+ let exclude_list: Option<Vec<String>> =
+ params.get("exclude").and_then(Value::as_array).map(|a| {
+ a.iter()
+ .filter_map(|v| match v {
+ Value::String(s) => Some(s.clone()),
+ Value::Number(n) => Some(n.to_string()),
+ Value::Bool(b) => Some(b.to_string()),
+ _ => None,
+ })
+ .collect()
+ });
// Apply min_doc_count filter before sorting — ES default is 1, tests set
// values as low as 0 (to force empty-term buckets into the output).
@@ -2376,20 +2756,19 @@ fn run_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs:
// Build (key, count, sub_aggs) so we can reuse sub-agg results for
// both ordering and final bucket output.
- let pre_computed: Vec<(String, u64, Option<Value>)> = if orders_need_sub_agg
- || sub_aggs.is_some()
- {
- candidates
- .iter()
- .map(|(k, c)| {
- let bucket_docs = compute_bucket_docs(k);
- let sub_res = sub_aggs.map(|sa| run_aggs_with_all(sa, &bucket_docs, all_docs));
- (k.clone(), *c, sub_res)
- })
- .collect()
- } else {
- candidates.drain(..).map(|(k, c)| (k, c, None)).collect()
- };
+ let pre_computed: Vec<(String, u64, Option<Value>)> =
+ if orders_need_sub_agg || sub_aggs.is_some() {
+ candidates
+ .iter()
+ .map(|(k, c)| {
+ let bucket_docs = compute_bucket_docs(k);
+ let sub_res = sub_aggs.map(|sa| run_aggs_with_all(sa, &bucket_docs, all_docs));
+ (k.clone(), *c, sub_res)
+ })
+ .collect()
+ } else {
+ candidates.drain(..).map(|(k, c)| (k, c, None)).collect()
+ };
// Reorder according to the requested sort keys.
let mut sorted: Vec<(String, u64, Option<Value>)> = pre_computed;
@@ -2470,7 +2849,9 @@ pub(crate) fn typed_term_key(key: &str) -> (Value, Option<String>) {
} else if let Ok(n) = key.parse::<i64>() {
json!(n)
} else if let Ok(f) = key.parse::<f64>() {
- serde_json::Number::from_f64(f).map(Value::Number).unwrap_or(json!(key))
+ serde_json::Number::from_f64(f)
+ .map(Value::Number)
+ .unwrap_or(json!(key))
} else if let Some(epoch_ms) = parse_date_ms(&Value::String(key.to_string())) {
// ES emits `key_as_string` in full ISO-8601 UTC with millis
// regardless of the input granularity (e.g. "2016-05-03" →
@@ -2523,7 +2904,9 @@ fn cmp_terms_by_orders(
/// `<bucket_agg>>.<metric_agg>` (or `>` only), and keyed
/// percentile/percentile_rank lookups.
fn lookup_agg_metric(sub: Option<&Value>, path: &str) -> f64 {
- let Some(root) = sub else { return f64::NEG_INFINITY };
+ let Some(root) = sub else {
+ return f64::NEG_INFINITY;
+ };
// Normalize ES `>` separator (nested bucket-path syntax) to `.`
// for uniform walking. `aggA>aggB` → `aggA.aggB`. Strip leading dot.
let normalized = path.replace('>', ".");
@@ -2542,15 +2925,28 @@ fn lookup_agg_metric(sub: Option<&Value>, path: &str) -> f64 {
cur.as_f64().unwrap_or(f64::NEG_INFINITY)
}
-fn run_ip_prefix(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_ip_prefix(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": []}),
};
- let prefix_length = params.get("prefix_length").and_then(Value::as_u64).unwrap_or(24) as usize;
- let is_ipv6 = params.get("is_ipv6").and_then(Value::as_bool).unwrap_or(false);
+ let prefix_length = params
+ .get("prefix_length")
+ .and_then(Value::as_u64)
+ .unwrap_or(24) as usize;
+ let is_ipv6 = params
+ .get("is_ipv6")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
let append_prefix_length = params
- .get("append_prefix_length").and_then(Value::as_bool).unwrap_or(false);
+ .get("append_prefix_length")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
// Group docs by their prefixed IP key. ES sorts buckets by the binary
// octet representation of the masked address, not by lexicographic
@@ -2568,8 +2964,11 @@ fn run_ip_prefix(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
for v in extract_field_values(doc, field) {
let (binkey, display): (Option<Vec<u8>>, Option<String>) = if is_ipv6 {
// Treat every address as 128-bit. v4 maps to ::ffff:V.V.V.V.
- let addr: Option<std::net::Ipv6Addr> = v.parse().ok()
- .or_else(|| v.parse::<std::net::Ipv4Addr>().ok().map(|v4| v4.to_ipv6_mapped()));
+ let addr: Option<std::net::Ipv6Addr> = v.parse().ok().or_else(|| {
+ v.parse::<std::net::Ipv4Addr>()
+ .ok()
+ .map(|v4| v4.to_ipv6_mapped())
+ });
match addr {
Some(a) => {
let masked = mask_ipv6(a, prefix_length);
@@ -2585,14 +2984,20 @@ fn run_ip_prefix(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
// When the value is IPv6 but is_ipv6=false, ES takes the
// last 4 bytes and treats them as IPv4 (the "incorrect"
// classification case) rather than discarding the doc.
- let ip4: Option<std::net::Ipv4Addr> = v.parse::<std::net::Ipv4Addr>().ok()
- .or_else(|| v.parse::<std::net::Ipv6Addr>().ok().map(|v6| {
- let oct = v6.octets();
- std::net::Ipv4Addr::new(oct[12], oct[13], oct[14], oct[15])
- }));
+ let ip4: Option<std::net::Ipv4Addr> =
+ v.parse::<std::net::Ipv4Addr>().ok().or_else(|| {
+ v.parse::<std::net::Ipv6Addr>().ok().map(|v6| {
+ let oct = v6.octets();
+ std::net::Ipv4Addr::new(oct[12], oct[13], oct[14], oct[15])
+ })
+ });
match ip4 {
Some(ip4) => {
- let mask = if prefix_length >= 32 { !0u32 } else { !0u32 << (32 - prefix_length) };
+ let mask = if prefix_length >= 32 {
+ !0u32
+ } else {
+ !0u32 << (32 - prefix_length)
+ };
let masked = u32::from_be_bytes(ip4.octets()) & mask;
let ip = std::net::Ipv4Addr::from(masked);
let octets = ip.octets();
@@ -2605,8 +3010,14 @@ fn run_ip_prefix(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
}
};
if let (Some(bk), Some(k)) = (binkey, display) {
- let disp = if append_prefix_length { format!("{}/{}", k, prefix_length) } else { k };
- let entry = prefix_docs.entry(bk).or_insert_with(|| (disp.clone(), Vec::new()));
+ let disp = if append_prefix_length {
+ format!("{}/{}", k, prefix_length)
+ } else {
+ k
+ };
+ let entry = prefix_docs
+ .entry(bk)
+ .or_insert_with(|| (disp.clone(), Vec::new()));
entry.1.push(i);
}
}
@@ -2617,26 +3028,31 @@ fn run_ip_prefix(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
entries.sort_by(|a, b| a.0.cmp(&b.0));
let entries: Vec<(String, Vec<usize>)> = entries.into_iter().map(|(_, v)| v).collect();
- let buckets: Vec<Value> = entries.iter().map(|(key, indices)| {
- let bucket_docs: Vec<Value> = indices.iter().map(|&i| docs[i].clone()).collect();
- let mut bucket = json!({
- "key": key,
- "doc_count": indices.len(),
- "is_ipv6": is_ipv6,
- "prefix_length": prefix_length,
- });
- // netmask is only emitted for IPv4.
- if !is_ipv6 {
- bucket["netmask"] = json!(format_netmask_v4(prefix_length));
- }
- if let Some(sub) = sub_aggs {
- let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
- if let (Some(bo), Value::Object(so)) = (bucket.as_object_mut(), sub_result) {
- for (k, v) in so { bo.insert(k, v); }
+ let buckets: Vec<Value> = entries
+ .iter()
+ .map(|(key, indices)| {
+ let bucket_docs: Vec<Value> = indices.iter().map(|&i| docs[i].clone()).collect();
+ let mut bucket = json!({
+ "key": key,
+ "doc_count": indices.len(),
+ "is_ipv6": is_ipv6,
+ "prefix_length": prefix_length,
+ });
+ // netmask is only emitted for IPv4.
+ if !is_ipv6 {
+ bucket["netmask"] = json!(format_netmask_v4(prefix_length));
}
- }
- bucket
- }).collect();
+ if let Some(sub) = sub_aggs {
+ let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
+ if let (Some(bo), Value::Object(so)) = (bucket.as_object_mut(), sub_result) {
+ for (k, v) in so {
+ bo.insert(k, v);
+ }
+ }
+ }
+ bucket
+ })
+ .collect();
json!({"buckets": buckets})
}
@@ -2655,57 +3071,95 @@ fn mask_ipv6(addr: std::net::Ipv6Addr, prefix_length: usize) -> std::net::Ipv6Ad
}
fn format_netmask_v4(prefix_length: usize) -> String {
- let mask = if prefix_length >= 32 { !0u32 } else { !0u32 << (32 - prefix_length) };
+ let mask = if prefix_length >= 32 {
+ !0u32
+ } else {
+ !0u32 << (32 - prefix_length)
+ };
let ip = std::net::Ipv4Addr::from(mask);
ip.to_string()
}
-fn run_rare_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_rare_terms(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": []}),
};
- let max_doc_count = params.get("max_doc_count").and_then(Value::as_u64).unwrap_or(1) as usize;
+ let max_doc_count = params
+ .get("max_doc_count")
+ .and_then(Value::as_u64)
+ .unwrap_or(1) as usize;
// include / exclude — ES accepts array of literals (applies to any
// field type) or regex string (keyword fields only).
- let include_list: Option<Vec<String>> = params.get("include").and_then(Value::as_array)
- .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)
- .or_else(|| v.as_f64().map(|f| f.to_string()))).collect());
- let exclude_list: Option<Vec<String>> = params.get("exclude").and_then(Value::as_array)
- .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)
- .or_else(|| v.as_f64().map(|f| f.to_string()))).collect());
+ let include_list: Option<Vec<String>> =
+ params.get("include").and_then(Value::as_array).map(|a| {
+ a.iter()
+ .filter_map(|v| {
+ v.as_str()
+ .map(String::from)
+ .or_else(|| v.as_f64().map(|f| f.to_string()))
+ })
+ .collect()
+ });
+ let exclude_list: Option<Vec<String>> =
+ params.get("exclude").and_then(Value::as_array).map(|a| {
+ a.iter()
+ .filter_map(|v| {
+ v.as_str()
+ .map(String::from)
+ .or_else(|| v.as_f64().map(|f| f.to_string()))
+ })
+ .collect()
+ });
let mut term_docs: HashMap<String, Vec<usize>> = HashMap::new();
for (i, doc) in docs.iter().enumerate() {
for v in extract_field_values(doc, field) {
- if v.is_empty() { continue; }
+ if v.is_empty() {
+ continue;
+ }
if let Some(ref inc) = include_list {
- if !inc.contains(&v) { continue; }
+ if !inc.contains(&v) {
+ continue;
+ }
}
if let Some(ref exc) = exclude_list {
- if exc.contains(&v) { continue; }
+ if exc.contains(&v) {
+ continue;
+ }
}
term_docs.entry(v).or_default().push(i);
}
}
- let mut entries: Vec<_> = term_docs.into_iter()
+ let mut entries: Vec<_> = term_docs
+ .into_iter()
.filter(|(_, indices)| indices.len() <= max_doc_count)
.collect();
entries.sort_by(|a, b| a.1.len().cmp(&b.1.len()).then(a.0.cmp(&b.0)));
- let buckets: Vec<Value> = entries.iter().map(|(key, indices)| {
- let bucket_docs: Vec<Value> = indices.iter().map(|&i| docs[i].clone()).collect();
- let mut bucket = build_terms_bucket(key, indices.len() as u64);
- if let Some(sub) = sub_aggs {
- let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
- if let (Some(bo), Value::Object(so)) = (bucket.as_object_mut(), sub_result) {
- for (k, v) in so { bo.insert(k, v); }
+ let buckets: Vec<Value> = entries
+ .iter()
+ .map(|(key, indices)| {
+ let bucket_docs: Vec<Value> = indices.iter().map(|&i| docs[i].clone()).collect();
+ let mut bucket = build_terms_bucket(key, indices.len() as u64);
+ if let Some(sub) = sub_aggs {
+ let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
+ if let (Some(bo), Value::Object(so)) = (bucket.as_object_mut(), sub_result) {
+ for (k, v) in so {
+ bo.insert(k, v);
+ }
+ }
}
- }
- bucket
- }).collect();
+ bucket
+ })
+ .collect();
json!({"buckets": buckets})
}
@@ -2722,7 +3176,11 @@ fn run_matrix_stats(params: &Value, docs: &[Value]) -> Value {
let fields: Vec<String> = params
.get("fields")
.and_then(Value::as_array)
- .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
+ .map(|a| {
+ a.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect()
+ })
.unwrap_or_default();
if fields.is_empty() {
return json!({"doc_count": 0, "fields": []});
@@ -2744,10 +3202,18 @@ fn run_matrix_stats(params: &Value, docs: &[Value]) -> Value {
let f32_fields: HashSet<String> = params
.get("__xy_f32_fields__")
.and_then(Value::as_array)
- .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
+ .map(|a| {
+ a.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect()
+ })
.unwrap_or_default();
let cast_if_f32 = |field: &str, v: f64| -> f64 {
- if f32_fields.contains(field) { v as f32 as f64 } else { v }
+ if f32_fields.contains(field) {
+ v as f32 as f64
+ } else {
+ v
+ }
};
// Per-doc value vector (f64 per field) — doc is included only when
@@ -2774,7 +3240,11 @@ fn run_matrix_stats(params: &Value, docs: &[Value]) -> Value {
"median" => {
let mut s = nums.clone();
s.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
- if s.is_empty() { None } else { Some(s[s.len() / 2]) }
+ if s.is_empty() {
+ None
+ } else {
+ Some(s[s.len() / 2])
+ }
}
_ => Some(nums.iter().sum::<f64>() / nums.len() as f64), // "avg" default
}
@@ -2854,6 +3324,9 @@ fn run_matrix_stats(params: &Value, docs: &[Value]) -> Value {
}
}
let denom = if n > 1 { (n - 1) as f64 } else { 1.0 };
+ // cov[i][j] is 2-D matrix indexing; a plain range loop is the clearest
+ // form and an iterator refactor would obscure the normalization math.
+ #[allow(clippy::needless_range_loop)]
for i in 0..k {
for j in 0..k {
cov[i][j] /= denom;
@@ -2861,7 +3334,11 @@ fn run_matrix_stats(params: &Value, docs: &[Value]) -> Value {
}
let corr = |i: usize, j: usize| -> f64 {
let d = std_dev[i] * std_dev[j];
- if d > 0.0 { cov[i][j] / d } else { 0.0 }
+ if d > 0.0 {
+ cov[i][j] / d
+ } else {
+ 0.0
+ }
};
// Build per-field output. ES orders fields by the input order.
@@ -2922,8 +3399,12 @@ fn run_string_stats(params: &Value, docs: &[Value]) -> Value {
for v in extract_field_values(doc, field) {
count += 1;
let l = v.chars().count();
- if l < min_len { min_len = l; }
- if l > max_len { max_len = l; }
+ if l < min_len {
+ min_len = l;
+ }
+ if l > max_len {
+ max_len = l;
+ }
total_len += l;
for c in v.chars() {
*char_counts.entry(c).or_insert(0) += 1;
@@ -2975,7 +3456,9 @@ fn run_string_stats(params: &Value, docs: &[Value]) -> Value {
fn run_boxplot(params: &Value, docs: &[Value]) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
- None => return json!({"min": Value::Null, "max": Value::Null, "q1": Value::Null, "q2": Value::Null, "q3": Value::Null}),
+ None => {
+ return json!({"min": Value::Null, "max": Value::Null, "q1": Value::Null, "q2": Value::Null, "q3": Value::Null})
+ }
};
let mut values: Vec<f64> = Vec::new();
for doc in docs {
@@ -2996,12 +3479,18 @@ fn run_boxplot(params: &Value, docs: &[Value]) -> Value {
}
values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let percentile = |p: f64, vals: &[f64]| -> f64 {
- if vals.is_empty() { return 0.0; }
- if vals.len() == 1 { return vals[0]; }
+ if vals.is_empty() {
+ return 0.0;
+ }
+ if vals.len() == 1 {
+ return vals[0];
+ }
let rank = p / 100.0 * (vals.len() - 1) as f64;
let lo = rank.floor() as usize;
let hi = rank.ceil() as usize;
- if lo == hi { return vals[lo]; }
+ if lo == hi {
+ return vals[lo];
+ }
vals[lo] + (rank - lo as f64) * (vals[hi] - vals[lo])
};
let q1 = percentile(25.0, &values);
@@ -3032,7 +3521,10 @@ fn run_top_metrics(params: &Value, docs: &[Value]) -> Value {
.filter_map(|m| m.get("field").and_then(Value::as_str).map(String::from))
.collect::<Vec<_>>(),
),
- Value::Object(o) => o.get("field").and_then(Value::as_str).map(|s| vec![s.to_string()]),
+ Value::Object(o) => o
+ .get("field")
+ .and_then(Value::as_str)
+ .map(|s| vec![s.to_string()]),
_ => None,
})
.unwrap_or_default();
@@ -3047,7 +3539,11 @@ fn run_top_metrics(params: &Value, docs: &[Value]) -> Value {
if let Some((k, v)) = o.iter().next() {
let desc = match v {
Value::String(s) => s == "desc",
- Value::Object(ob) => ob.get("order").and_then(Value::as_str).map(|s| s == "desc").unwrap_or(true),
+ Value::Object(ob) => ob
+ .get("order")
+ .and_then(Value::as_str)
+ .map(|s| s == "desc")
+ .unwrap_or(true),
_ => true,
};
(k.clone(), desc)
@@ -3060,7 +3556,11 @@ fn run_top_metrics(params: &Value, docs: &[Value]) -> Value {
if let Some((k, v)) = o.iter().next() {
let desc = match v {
Value::String(s) => s == "desc",
- Value::Object(ob) => ob.get("order").and_then(Value::as_str).map(|s| s == "desc").unwrap_or(true),
+ Value::Object(ob) => ob
+ .get("order")
+ .and_then(Value::as_str)
+ .map(|s| s == "desc")
+ .unwrap_or(true),
_ => true,
};
(k.clone(), desc)
@@ -3077,12 +3577,19 @@ fn run_top_metrics(params: &Value, docs: &[Value]) -> Value {
let mut ranked: Vec<(f64, &Value)> = docs
.iter()
.filter_map(|d| {
- extract_numeric_values(d, &sort_field).first().copied().map(|v| (v, d))
+ extract_numeric_values(d, &sort_field)
+ .first()
+ .copied()
+ .map(|v| (v, d))
})
.collect();
ranked.sort_by(|a, b| {
let cmp = a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal);
- if sort_desc { cmp.reverse() } else { cmp }
+ if sort_desc {
+ cmp.reverse()
+ } else {
+ cmp
+ }
});
let out: Vec<Value> = ranked
@@ -3153,15 +3660,12 @@ pub fn render_date_format(
Some("strict_date_optional_time")
| Some("strict_date_optional_time_nanos")
| Some("iso8601")
- | Some("date_optional_time") => {
- dt.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()
- }
+ | Some("date_optional_time") => dt.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(),
Some("basic_date") => dt.format("%Y%m%d").to_string(),
Some("basic_date_time") => dt.format("%Y%m%dT%H%M%S%.3fZ").to_string(),
Some("basic_time") => dt.format("%H%M%S%.3fZ").to_string(),
Some("date") | Some("strict_date") => dt.format("%Y-%m-%d").to_string(),
- Some("date_hour_minute_second")
- | Some("strict_date_hour_minute_second") => {
+ Some("date_hour_minute_second") | Some("strict_date_hour_minute_second") => {
dt.format("%Y-%m-%dT%H:%M:%S").to_string()
}
Some("date_time") | Some("strict_date_time") => {
@@ -3197,28 +3701,45 @@ pub(crate) fn java_to_strftime(pattern: &str) -> String {
out.push(bytes[i] as char);
i += 1;
}
- if i < bytes.len() { i += 1; } // skip closing quote
+ if i < bytes.len() {
+ i += 1;
+ } // skip closing quote
continue;
}
// Match longest-first. Java uses `uuuu` for proleptic year and
// `yyyy` for year-of-era; both render as 4-digit year in modern
// ES so map both to `%Y`. `ZZZZZ` is offset with colon; `XXX`
// similarly emits `+HH:MM`.
- let matched: Option<(usize, &str)> = if bytes[i..].starts_with(b"uuuu") { Some((4, "%Y")) }
- else if bytes[i..].starts_with(b"yyyy") { Some((4, "%Y")) }
- else if bytes[i..].starts_with(b"yy") { Some((2, "%y")) }
- else if bytes[i..].starts_with(b"MM") { Some((2, "%m")) }
- else if bytes[i..].starts_with(b"dd") { Some((2, "%d")) }
- else if bytes[i..].starts_with(b"HH") { Some((2, "%H")) }
- else if bytes[i..].starts_with(b"mm") { Some((2, "%M")) }
- else if bytes[i..].starts_with(b"ss") { Some((2, "%S")) }
- else if bytes[i..].starts_with(b"SSSSSSSSS") { Some((9, "%9f")) }
- else if bytes[i..].starts_with(b"SSSSSS") { Some((6, "%6f")) }
- else if bytes[i..].starts_with(b"SSS") { Some((3, "%3f")) }
- else if bytes[i..].starts_with(b"ZZZZZ") { Some((5, "%:z")) }
- else if bytes[i..].starts_with(b"XXX") { Some((3, "%:z")) }
- else if bytes[i..].starts_with(b"Z") { Some((1, "%z")) }
- else { None };
+ let matched: Option<(usize, &str)> =
+ if bytes[i..].starts_with(b"uuuu") || bytes[i..].starts_with(b"yyyy") {
+ Some((4, "%Y"))
+ } else if bytes[i..].starts_with(b"yy") {
+ Some((2, "%y"))
+ } else if bytes[i..].starts_with(b"MM") {
+ Some((2, "%m"))
+ } else if bytes[i..].starts_with(b"dd") {
+ Some((2, "%d"))
+ } else if bytes[i..].starts_with(b"HH") {
+ Some((2, "%H"))
+ } else if bytes[i..].starts_with(b"mm") {
+ Some((2, "%M"))
+ } else if bytes[i..].starts_with(b"ss") {
+ Some((2, "%S"))
+ } else if bytes[i..].starts_with(b"SSSSSSSSS") {
+ Some((9, "%9f"))
+ } else if bytes[i..].starts_with(b"SSSSSS") {
+ Some((6, "%6f"))
+ } else if bytes[i..].starts_with(b"SSS") {
+ Some((3, "%3f"))
+ } else if bytes[i..].starts_with(b"ZZZZZ") {
+ Some((5, "%:z"))
+ } else if bytes[i..].starts_with(b"XXX") {
+ Some((3, "%:z"))
+ } else if bytes[i..].starts_with(b"Z") {
+ Some((1, "%z"))
+ } else {
+ None
+ };
match matched {
Some((n, s)) => {
out.push_str(s);
@@ -3288,14 +3809,10 @@ pub(crate) fn parse_date_ms(val: &Value) -> Option<i64> {
return Some(dt.timestamp_millis());
}
// Try ISO-8601 without timezone (treat as UTC).
- if let Ok(dt) =
- chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f")
- {
+ if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f") {
return Some(dt.and_utc().timestamp_millis());
}
- if let Ok(dt) =
- chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S")
- {
+ if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S") {
return Some(dt.and_utc().timestamp_millis());
}
if let Ok(dt) = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d") {
@@ -3306,21 +3823,15 @@ pub(crate) fn parse_date_ms(val: &Value) -> Option<i64> {
if let Ok(dt) = chrono::NaiveDate::parse_from_str(s, "%Y/%m/%d") {
return Some(dt.and_hms_opt(0, 0, 0)?.and_utc().timestamp_millis());
}
- if let Ok(dt) =
- chrono::NaiveDateTime::parse_from_str(s, "%Y/%m/%d %H:%M:%S")
- {
+ if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(s, "%Y/%m/%d %H:%M:%S") {
return Some(dt.and_utc().timestamp_millis());
}
// Space-separated form `YYYY-MM-DD HH:MM:SS[.fff]` (ES Java
// pattern `yyyy-MM-dd HH:mm:ss.SSS`).
- if let Ok(dt) =
- chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f")
- {
+ if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") {
return Some(dt.and_utc().timestamp_millis());
}
- if let Ok(dt) =
- chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S")
- {
+ if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") {
return Some(dt.and_utc().timestamp_millis());
}
// Fallback: try parsing as a raw integer string.
@@ -3370,7 +3881,9 @@ fn build_terms_bucket(key: &str, count: u64) -> Value {
} else if let Ok(n) = key.parse::<i64>() {
json!(n)
} else if let Ok(f) = key.parse::<f64>() {
- serde_json::Number::from_f64(f).map(Value::Number).unwrap_or(json!(key))
+ serde_json::Number::from_f64(f)
+ .map(Value::Number)
+ .unwrap_or(json!(key))
} else if let Some(epoch_ms) = parse_date_ms(&Value::String(key.to_string())) {
// ES emits the canonical ISO-8601 UTC form (with millisecond precision),
// regardless of the input form (`2016-05-03` expands to full timestamp).
@@ -3432,13 +3945,20 @@ pub(crate) fn is_calendar_interval(interval: &str) -> bool {
// 23h / 25h days that still bucket correctly.
matches!(
interval.trim(),
- "minute" | "1m"
- | "hour" | "1h"
- | "day" | "1d"
- | "week" | "1w"
- | "month" | "1M"
- | "quarter" | "1q"
- | "year" | "1y"
+ "minute"
+ | "1m"
+ | "hour"
+ | "1h"
+ | "day"
+ | "1d"
+ | "week"
+ | "1w"
+ | "month"
+ | "1M"
+ | "quarter"
+ | "1q"
+ | "year"
+ | "1y"
)
}
@@ -3457,16 +3977,28 @@ pub(crate) fn calendar_bucket_key(ts_ms: i64, interval: &str) -> i64 {
}
"month" | "1M" => {
let start = chrono::NaiveDate::from_ymd_opt(naive.year(), naive.month(), 1).unwrap();
- start.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis()
+ start
+ .and_hms_opt(0, 0, 0)
+ .unwrap()
+ .and_utc()
+ .timestamp_millis()
}
"quarter" | "1q" => {
let q_month = ((naive.month() - 1) / 3) * 3 + 1;
let start = chrono::NaiveDate::from_ymd_opt(naive.year(), q_month, 1).unwrap();
- start.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis()
+ start
+ .and_hms_opt(0, 0, 0)
+ .unwrap()
+ .and_utc()
+ .timestamp_millis()
}
"year" | "1y" => {
let start = chrono::NaiveDate::from_ymd_opt(naive.year(), 1, 1).unwrap();
- start.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis()
+ start
+ .and_hms_opt(0, 0, 0)
+ .unwrap()
+ .and_utc()
+ .timestamp_millis()
}
_ => ts_ms,
}
@@ -3481,19 +4013,39 @@ pub(crate) fn next_calendar_bucket(bucket_ms: i64, interval: &str) -> i64 {
"day" | "1d" => bucket_ms + 86_400_000,
"week" | "1w" => bucket_ms + 7 * 86_400_000,
"month" | "1M" => {
- let (y, m) = if naive.month() == 12 { (naive.year() + 1, 1) } else { (naive.year(), naive.month() + 1) };
+ let (y, m) = if naive.month() == 12 {
+ (naive.year() + 1, 1)
+ } else {
+ (naive.year(), naive.month() + 1)
+ };
let start = chrono::NaiveDate::from_ymd_opt(y, m, 1).unwrap();
- start.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis()
+ start
+ .and_hms_opt(0, 0, 0)
+ .unwrap()
+ .and_utc()
+ .timestamp_millis()
}
"quarter" | "1q" => {
let q_month = ((naive.month() - 1) / 3) * 3 + 1;
- let (y, m) = if q_month + 3 > 12 { (naive.year() + 1, q_month + 3 - 12) } else { (naive.year(), q_month + 3) };
+ let (y, m) = if q_month + 3 > 12 {
+ (naive.year() + 1, q_month + 3 - 12)
+ } else {
+ (naive.year(), q_month + 3)
+ };
let start = chrono::NaiveDate::from_ymd_opt(y, m, 1).unwrap();
- start.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis()
+ start
+ .and_hms_opt(0, 0, 0)
+ .unwrap()
+ .and_utc()
+ .timestamp_millis()
}
"year" | "1y" => {
let start = chrono::NaiveDate::from_ymd_opt(naive.year() + 1, 1, 1).unwrap();
- start.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis()
+ start
+ .and_hms_opt(0, 0, 0)
+ .unwrap()
+ .and_utc()
+ .timestamp_millis()
}
_ => bucket_ms,
}
@@ -3518,9 +4070,15 @@ pub(crate) fn parse_time_zone_offset(tz: &str) -> Option<chrono::FixedOffset> {
let sign: i32 = if bytes[0] == b'+' { 1 } else { -1 };
let rest = &t[1..];
let (h, m) = if let Some(idx) = rest.find(':') {
- (rest[..idx].parse::<i32>().ok()?, rest[idx + 1..].parse::<i32>().ok()?)
+ (
+ rest[..idx].parse::<i32>().ok()?,
+ rest[idx + 1..].parse::<i32>().ok()?,
+ )
} else if rest.len() == 4 {
- (rest[..2].parse::<i32>().ok()?, rest[2..].parse::<i32>().ok()?)
+ (
+ rest[..2].parse::<i32>().ok()?,
+ rest[2..].parse::<i32>().ok()?,
+ )
} else if rest.len() == 2 {
(rest.parse::<i32>().ok()?, 0)
} else {
@@ -3539,7 +4097,7 @@ pub(crate) fn fixed_offset_for_tz_at(tz: &str, ts_ms: i64) -> Option<chrono::Fix
if let Some(off) = parse_time_zone_offset(tz) {
return Some(off);
}
- use chrono::{TimeZone, Offset};
+ use chrono::{Offset, TimeZone};
let zone: chrono_tz::Tz = tz.parse().ok()?;
let dt = chrono::DateTime::from_timestamp_millis(ts_ms)?;
let local = zone.from_utc_datetime(&dt.naive_utc());
@@ -3550,7 +4108,9 @@ pub(crate) fn fixed_offset_for_tz_at(tz: &str, ts_ms: i64) -> Option<chrono::Fix
/// Accepts forms like "+1d", "-1h", "+30m", "+90ms".
pub(crate) fn parse_offset_ms(s: &str) -> Option<i64> {
let t = s.trim();
- if t.is_empty() { return None; }
+ if t.is_empty() {
+ return None;
+ }
let (sign, rest) = match t.as_bytes()[0] {
b'+' => (1i64, &t[1..]),
b'-' => (-1i64, &t[1..]),
@@ -3559,7 +4119,12 @@ pub(crate) fn parse_offset_ms(s: &str) -> Option<i64> {
interval_to_ms(rest).map(|v| sign * v)
}
-fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_date_histogram(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": []}),
@@ -3581,7 +4146,10 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
let use_calendar = is_calendar_interval(interval_str);
- let tz_str = params.get("time_zone").and_then(Value::as_str).unwrap_or("");
+ let tz_str = params
+ .get("time_zone")
+ .and_then(Value::as_str)
+ .unwrap_or("");
// Only set tz_offset when an explicit time_zone was supplied. ES uses
// `...Z` ISO-8601 formatting when no time_zone is set, and `...+HH:MM`
// when one is. IANA zones (Europe/Berlin etc.) are resolved to the
@@ -3593,7 +4161,8 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
Some(off)
} else {
// Need a sample instant to pick the right DST offset.
- let sample_ms = docs.iter()
+ let sample_ms = docs
+ .iter()
.filter_map(|d| extract_date_ms_values(d, field).into_iter().next())
.next()
.unwrap_or(0);
@@ -3607,21 +4176,29 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
// hard_bounds: { min: ISO, max: ISO }. Buckets outside are dropped.
let (hard_min, hard_max) = if let Some(hb) = params.get("hard_bounds") {
- (hb.get("min").and_then(parse_date_ms), hb.get("max").and_then(parse_date_ms))
- } else { (None, None) };
+ (
+ hb.get("min").and_then(parse_date_ms),
+ hb.get("max").and_then(parse_date_ms),
+ )
+ } else {
+ (None, None)
+ };
// extended_bounds: { min, max } forces bucket range to include these.
let (ext_min, ext_max) = if let Some(eb) = params.get("extended_bounds") {
- (eb.get("min").and_then(parse_date_ms), eb.get("max").and_then(parse_date_ms))
- } else { (None, None) };
+ (
+ eb.get("min").and_then(parse_date_ms),
+ eb.get("max").and_then(parse_date_ms),
+ )
+ } else {
+ (None, None)
+ };
let mut buckets: HashMap<i64, Vec<usize>> = HashMap::new();
// `missing` on date_histogram: the value can be an ISO-8601 string or
// an epoch-ms number. Missing-field docs fall into this bucket.
- let missing_ms: Option<i64> = params
- .get("missing")
- .and_then(|v| parse_date_ms(v));
+ let missing_ms: Option<i64> = params.get("missing").and_then(parse_date_ms);
// Bucket-key derivation with time_zone:
// local_ts = utc_ts + tz_offset_seconds*1000
@@ -3632,15 +4209,16 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
// timestamp so buckets straddling a DST transition get the correct
// midnight. For fixed-offset zones this reduces to the single value.
let tz_str_bucket = tz_str.to_string();
- let tz_is_iana = !tz_str_bucket.is_empty()
- && parse_time_zone_offset(&tz_str_bucket).is_none();
+ let tz_is_iana = !tz_str_bucket.is_empty() && parse_time_zone_offset(&tz_str_bucket).is_none();
let shift_at = |ts: i64| -> i64 {
if tz_is_iana {
if let Some(off) = fixed_offset_for_tz_at(&tz_str_bucket, ts) {
return off.local_minus_utc() as i64 * 1000;
}
}
- tz_offset.map(|o| o.local_minus_utc() as i64 * 1000).unwrap_or(0)
+ tz_offset
+ .map(|o| o.local_minus_utc() as i64 * 1000)
+ .unwrap_or(0)
};
let to_local = |ts: i64| ts + shift_at(ts) - offset_ms;
// Map a local-time bucket-start back to UTC. The local time 00:00 of
@@ -3667,7 +4245,9 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
// scalar extraction found nothing.
let ranges = if raws.is_empty() {
extract_date_ranges(doc, field)
- } else { Vec::new() };
+ } else {
+ Vec::new()
+ };
if raws.is_empty() && ranges.is_empty() {
if let Some(m) = missing_ms {
raws.push(m);
@@ -3683,8 +4263,16 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
};
let bucket_key = to_utc(local_bucket);
// Apply hard_bounds (inclusive on both ends — ES semantics).
- if let Some(m) = hard_min { if bucket_key < m { continue; } }
- if let Some(m) = hard_max { if bucket_key > m { continue; } }
+ if let Some(m) = hard_min {
+ if bucket_key < m {
+ continue;
+ }
+ }
+ if let Some(m) = hard_max {
+ if bucket_key > m {
+ continue;
+ }
+ }
if !seen.contains(&bucket_key) {
seen.push(bucket_key);
buckets.entry(bucket_key).or_default().push(i);
@@ -3696,7 +4284,9 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
// the bucket at that key spans beyond max by one interval, so
// we still emit it when the range intersects it.
let end_anchor = to_ms; // compare bucket_key against to_ms via interval overlap
- if !(end_anchor > start_anchor) { continue; }
+ if end_anchor <= start_anchor {
+ continue;
+ }
let local_start = to_local(start_anchor);
let mut local_bucket = if use_calendar {
calendar_bucket_key(local_start, interval_str)
@@ -3706,13 +4296,21 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
let mut guard = 0usize;
loop {
guard += 1;
- if guard > 65_536 { break; }
+ if guard > 65_536 {
+ break;
+ }
let bucket_key = to_utc(local_bucket);
// Stop once the bucket's start is at/after the range's upper bound
// — no more intersection possible.
- if bucket_key >= end_anchor { break; }
+ if bucket_key >= end_anchor {
+ break;
+ }
// Stop once we've walked past the hard_max bucket.
- if let Some(m) = hard_max { if bucket_key > m { break; } }
+ if let Some(m) = hard_max {
+ if bucket_key > m {
+ break;
+ }
+ }
let next_local = if use_calendar {
next_calendar_bucket(local_bucket, interval_str)
} else {
@@ -3720,20 +4318,33 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
};
let next_key = to_utc(next_local);
if next_key > start_anchor {
- if let Some(m) = hard_min { if bucket_key < m { local_bucket = next_local; continue; } }
+ if let Some(m) = hard_min {
+ if bucket_key < m {
+ local_bucket = next_local;
+ continue;
+ }
+ }
if !seen.contains(&bucket_key) {
seen.push(bucket_key);
buckets.entry(bucket_key).or_default().push(i);
}
}
- if next_key <= bucket_key { break; } // safety against infinite loop
+ if next_key <= bucket_key {
+ break;
+ } // safety against infinite loop
local_bucket = next_local;
}
}
}
- let min_doc_count = params.get("min_doc_count").and_then(|v| v.as_u64()).unwrap_or(0);
- let keyed = params.get("keyed").and_then(Value::as_bool).unwrap_or(false);
+ let min_doc_count = params
+ .get("min_doc_count")
+ .and_then(|v| v.as_u64())
+ .unwrap_or(0);
+ let keyed = params
+ .get("keyed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
const MAX_BUCKETS: i64 = 65_536;
@@ -3767,14 +4378,22 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
};
if let Some(m) = ext_min {
let local_m = ext_bound_to_local(m);
- let b = if use_calendar { calendar_bucket_key(local_m, interval_str) } else { local_m.div_euclid(interval_ms) * interval_ms };
+ let b = if use_calendar {
+ calendar_bucket_key(local_m, interval_str)
+ } else {
+ local_m.div_euclid(interval_ms) * interval_ms
+ };
let key = ext_bound_to_utc(b);
range_min = Some(range_min.map_or(key, |k| k.min(key)));
range_max = Some(range_max.map_or(key, |k| k.max(key)));
}
if let Some(m) = ext_max {
let local_m = ext_bound_to_local(m);
- let b = if use_calendar { calendar_bucket_key(local_m, interval_str) } else { local_m.div_euclid(interval_ms) * interval_ms };
+ let b = if use_calendar {
+ calendar_bucket_key(local_m, interval_str)
+ } else {
+ local_m.div_euclid(interval_ms) * interval_ms
+ };
let key = ext_bound_to_utc(b);
range_min = Some(range_min.map_or(key, |k| k.min(key)));
range_max = Some(range_max.map_or(key, |k| k.max(key)));
@@ -3783,7 +4402,11 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
// hard_bounds further clamps the fill range (honored alongside data filter)
if let Some(m) = hard_min {
let local_m = to_local(m);
- let b = if use_calendar { calendar_bucket_key(local_m, interval_str) } else { local_m.div_euclid(interval_ms) * interval_ms };
+ let b = if use_calendar {
+ calendar_bucket_key(local_m, interval_str)
+ } else {
+ local_m.div_euclid(interval_ms) * interval_ms
+ };
let key = to_utc(b);
range_min = Some(range_min.map_or(key, |k| k.max(key)));
}
@@ -3791,7 +4414,11 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
// hard_bounds is inclusive on both ends: the bucket whose start
// key equals `max` is still emitted.
let local_m = to_local(m);
- let b = if use_calendar { calendar_bucket_key(local_m, interval_str) } else { local_m.div_euclid(interval_ms) * interval_ms };
+ let b = if use_calendar {
+ calendar_bucket_key(local_m, interval_str)
+ } else {
+ local_m.div_euclid(interval_ms) * interval_ms
+ };
let key = to_utc(b);
range_max = Some(range_max.map_or(key, |k| k.min(key)));
}
@@ -3838,7 +4465,7 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
// gap-fill iteration). Fall back to a fixed-offset step so the
// loop advances.
if next <= k {
- k = k + interval_ms;
+ k += interval_ms;
} else {
k = next;
}
@@ -3851,7 +4478,12 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
// min_doc_count filter (applied after gap-fill)
if min_doc_count > 0 {
- bucket_keys.retain(|k| buckets.get(k).map(|v| v.len() as u64 >= min_doc_count).unwrap_or(false));
+ bucket_keys.retain(|k| {
+ buckets
+ .get(k)
+ .map(|v| v.len() as u64 >= min_doc_count)
+ .unwrap_or(false)
+ });
}
let empty_indices: Vec<usize> = Vec::new();
@@ -3878,7 +4510,8 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
// synthetic UTC timestamp that embeds the local offset.
let off_ms = off.local_minus_utc() as i64 * 1000;
let shifted_ms = key + off_ms;
- let dt_shifted = chrono::DateTime::<chrono::Utc>::from_timestamp_millis(shifted_ms).unwrap_or_default();
+ let dt_shifted = chrono::DateTime::<chrono::Utc>::from_timestamp_millis(shifted_ms)
+ .unwrap_or_default();
render_date_format(Some(fmt), shifted_ms, dt_shifted)
} else {
dt_local.format("%Y-%m-%dT%H:%M:%S%.3f%:z").to_string()
@@ -3893,8 +4526,7 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
.iter()
.map(|&key| {
let doc_indices = buckets.get(&key).unwrap_or(&empty_indices);
- let bucket_docs: Vec<Value> =
- doc_indices.iter().map(|&i| docs[i].clone()).collect();
+ let bucket_docs: Vec<Value> = doc_indices.iter().map(|&i| docs[i].clone()).collect();
let doc_count = sum_doc_count(&bucket_docs);
let dt = render_key(key);
@@ -3923,7 +4555,10 @@ fn run_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value],
if keyed {
let mut map = serde_json::Map::new();
for b in result_buckets {
- let k = b.get("key_as_string").and_then(Value::as_str).map(str::to_string)
+ let k = b
+ .get("key_as_string")
+ .and_then(Value::as_str)
+ .map(str::to_string)
.or_else(|| b.get("key").and_then(|v| v.as_i64()).map(|i| i.to_string()))
.unwrap_or_default();
map.insert(k, b);
@@ -3969,13 +4604,13 @@ fn run_avg<'d>(params: &Value, docs: &'d [Value], cache: &mut FieldCache<'d>) ->
None => return json!({"value": Value::Null, "__xy_count__": 0, "__xy_sum__": 0.0}),
};
let missing = get_missing_value(params);
- let (sum, count) = cache.get_numeric(field).iter().fold(
- (0.0f64, 0usize),
- |(s, n), v| match v.or(missing) {
+ let (sum, count) = cache
+ .get_numeric(field)
+ .iter()
+ .fold((0.0f64, 0usize), |(s, n), v| match v.or(missing) {
Some(x) => (s + x, n + 1),
None => (s, n),
- },
- );
+ });
if count == 0 {
return json!({ "value": Value::Null, "__xy_count__": 0, "__xy_sum__": 0.0 });
}
@@ -4017,7 +4652,11 @@ fn run_min<'d>(params: &Value, docs: &'d [Value], cache: &mut FieldCache<'d>) ->
None => return json!({"value": Value::Null, "__xy_agg__": "min"}),
};
let missing = get_missing_value(params);
- let min = cache.get_numeric(field).iter().filter_map(|v| v.or(missing)).fold(f64::INFINITY, f64::min);
+ let min = cache
+ .get_numeric(field)
+ .iter()
+ .filter_map(|v| v.or(missing))
+ .fold(f64::INFINITY, f64::min);
if min.is_infinite() {
return json!({ "value": Value::Null, "__xy_agg__": "min" });
}
@@ -4036,7 +4675,11 @@ fn run_max<'d>(params: &Value, docs: &'d [Value], cache: &mut FieldCache<'d>) ->
None => return json!({"value": Value::Null, "__xy_agg__": "max"}),
};
let missing = get_missing_value(params);
- let max = cache.get_numeric(field).iter().filter_map(|v| v.or(missing)).fold(f64::NEG_INFINITY, f64::max);
+ let max = cache
+ .get_numeric(field)
+ .iter()
+ .filter_map(|v| v.or(missing))
+ .fold(f64::NEG_INFINITY, f64::max);
if max.is_infinite() {
return json!({ "value": Value::Null, "__xy_agg__": "max" });
}
@@ -4052,11 +4695,13 @@ fn run_max<'d>(params: &Value, docs: &'d [Value], cache: &mut FieldCache<'d>) ->
fn field_is_date_shape(docs: &[Value], field: &str) -> bool {
docs.iter().any(|d| match get_nested_field(d, field) {
- Value::String(s) => parse_date_ms(&Value::String(s.clone())).is_some()
- && s.parse::<f64>().is_err(),
+ Value::String(s) => {
+ parse_date_ms(&Value::String(s.clone())).is_some() && s.parse::<f64>().is_err()
+ }
Value::Array(arr) => arr.iter().any(|e| match e {
- Value::String(s) => parse_date_ms(&Value::String(s.clone())).is_some()
- && s.parse::<f64>().is_err(),
+ Value::String(s) => {
+ parse_date_ms(&Value::String(s.clone())).is_some() && s.parse::<f64>().is_err()
+ }
_ => false,
}),
_ => false,
@@ -4078,10 +4723,14 @@ fn run_stats<'d>(params: &Value, _docs: &'d [Value], cache: &mut FieldCache<'d>)
};
let missing = get_missing_value(params);
- let (count, sum, min, max) = cache.get_numeric(field).iter().filter_map(|v| v.or(missing)).fold(
- (0usize, 0.0f64, f64::INFINITY, f64::NEG_INFINITY),
- |(cnt, s, mn, mx), x| (cnt + 1, s + x, mn.min(x), mx.max(x)),
- );
+ let (count, sum, min, max) = cache
+ .get_numeric(field)
+ .iter()
+ .filter_map(|v| v.or(missing))
+ .fold(
+ (0usize, 0.0f64, f64::INFINITY, f64::NEG_INFINITY),
+ |(cnt, s, mn, mx), x| (cnt + 1, s + x, mn.min(x), mx.max(x)),
+ );
if count == 0 {
return json!({
@@ -4149,7 +4798,12 @@ fn run_cardinality<'d>(params: &Value, docs: &'d [Value], cache: &mut FieldCache
/// Filter aggregation: filter documents using an ES query, then run sub-aggs.
// (signature updated to thread all_docs through)
/// The `params` here is the filter query itself (e.g. `{"term": {"level": "ERROR"}}`).
-fn run_filter(filter_query: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_filter(
+ filter_query: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let filtered_docs: Vec<Value> = docs
.iter()
.filter(|doc| doc_matches_filter(doc, filter_query))
@@ -4188,7 +4842,12 @@ fn run_filter(filter_query: &Value, sub_aggs: Option<&Value>, docs: &[Value], al
/// { "filters": { "filters": [ { "term": { "status": "200" } },
/// { "term": { "status": "404" } } ] } }
/// ```
-fn run_filters(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_filters(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let filters_val = match params.get("filters") {
Some(v) => v,
None => return json!({"buckets": {}}),
@@ -4305,7 +4964,8 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {
if let Some(field_map) = query_body.as_object() {
for (field, expected_vals) in field_map {
if let Some(arr) = expected_vals.as_array() {
- let expected_strs: Vec<String> = arr.iter().map(value_to_string).collect();
+ let expected_strs: Vec<String> =
+ arr.iter().map(value_to_string).collect();
let actual_values = extract_field_values(doc, field);
if !actual_values.iter().any(|v| expected_strs.contains(v)) {
return false;
@@ -4319,7 +4979,9 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {
for (field, expected) in field_map {
let actual_values = extract_field_values(doc, field);
let expected_str = match expected {
- Value::Object(o) => o.get("query").map(value_to_string).unwrap_or_default(),
+ Value::Object(o) => {
+ o.get("query").map(value_to_string).unwrap_or_default()
+ }
other => value_to_string(other),
};
let exp_lower = expected_str.to_lowercase();
@@ -4332,7 +4994,9 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {
exp_tokens.iter().all(|t| v_lower.contains(t))
}
});
- if !matched { return false; }
+ if !matched {
+ return false;
+ }
}
}
}
@@ -4341,7 +5005,9 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {
for (field, expected) in field_map {
let actual_values = extract_field_values(doc, field);
let prefix = match expected {
- Value::Object(o) => o.get("value").map(value_to_string).unwrap_or_default(),
+ Value::Object(o) => {
+ o.get("value").map(value_to_string).unwrap_or_default()
+ }
other => value_to_string(other),
};
if !actual_values.iter().any(|v| v.starts_with(&prefix)) {
@@ -4355,18 +5021,25 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {
for (field, expected) in field_map {
let actual_values = extract_field_values(doc, field);
let pattern = match expected {
- Value::Object(o) => o.get("value").map(value_to_string).unwrap_or_default(),
+ Value::Object(o) => {
+ o.get("value").map(value_to_string).unwrap_or_default()
+ }
other => value_to_string(other),
};
let matched = actual_values.iter().any(|v| {
- if pattern == "*" { true }
- else if let Some(suffix) = pattern.strip_prefix('*') {
+ if pattern == "*" {
+ true
+ } else if let Some(suffix) = pattern.strip_prefix('*') {
v.ends_with(suffix)
} else if let Some(prefix) = pattern.strip_suffix('*') {
v.starts_with(prefix)
- } else { v == &pattern }
+ } else {
+ v == &pattern
+ }
});
- if !matched { return false; }
+ if !matched {
+ return false;
+ }
}
}
}
@@ -4390,16 +5063,24 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {
let num_val: Option<f64> = values.first().and_then(|s| s.parse().ok());
if let Some(n) = num_val {
if let Some(gte) = bounds.get("gte").and_then(Value::as_f64) {
- if n < gte { return false; }
+ if n < gte {
+ return false;
+ }
}
if let Some(gt) = bounds.get("gt").and_then(Value::as_f64) {
- if n <= gt { return false; }
+ if n <= gt {
+ return false;
+ }
}
if let Some(lte) = bounds.get("lte").and_then(Value::as_f64) {
- if n > lte { return false; }
+ if n > lte {
+ return false;
+ }
}
if let Some(lt) = bounds.get("lt").and_then(Value::as_f64) {
- if n >= lt { return false; }
+ if n >= lt {
+ return false;
+ }
}
}
}
@@ -4439,7 +5120,8 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {
Value::Array(a) => a.clone(),
single => vec![single.clone()],
};
- let has_must = bool_body.contains_key("must") || bool_body.contains_key("filter");
+ let has_must =
+ bool_body.contains_key("must") || bool_body.contains_key("filter");
if !has_must && !clause_list.is_empty() {
let any_match = clause_list.iter().any(|c| doc_matches_filter(doc, c));
if !any_match {
@@ -4477,7 +5159,7 @@ pub(crate) fn format_range_val(v: f64) -> String {
if v == 0.0 {
return "0.0".to_string();
}
- if abs >= 1e7 || abs < 1e-3 {
+ if !(1e-3..1e7).contains(&abs) {
// Java: scientific with single integer digit, capital E,
// trailing `.0` if mantissa has no fractional part, no `+` on
// positive exponents.
@@ -4508,7 +5190,12 @@ fn java_scientific(v: f64) -> String {
format!("{mantissa}E{exp}")
}
-fn run_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_range(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": []}),
@@ -4531,7 +5218,8 @@ fn run_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs:
// they had this numeric value. Applies before range comparison.
let missing_val: Option<f64> = params.get("missing").and_then(|v| match v {
Value::Number(n) => n.as_f64(),
- Value::String(s) if is_date_range => parse_date_ms(v).map(|ms| ms as f64)
+ Value::String(s) if is_date_range => parse_date_ms(v)
+ .map(|ms| ms as f64)
.or_else(|| s.parse::<f64>().ok()),
Value::String(s) => s.parse::<f64>().ok(),
_ => None,
@@ -4559,7 +5247,10 @@ fn run_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs:
.iter()
.filter(|doc| {
let num = if is_date_range {
- extract_date_ms_values(doc, field).first().copied().map(|ms| ms as f64)
+ extract_date_ms_values(doc, field)
+ .first()
+ .copied()
+ .map(|ms| ms as f64)
} else {
extract_numeric(doc, field)
};
@@ -4586,12 +5277,16 @@ fn run_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs:
format_range_val(f)
}
};
- let key = range_def.get("key").and_then(Value::as_str).map(String::from).unwrap_or_else(|| match (from, to) {
- (Some(f), Some(t)) => format!("{}-{}", render_bound(f), render_bound(t)),
- (Some(f), None) => format!("{}-*", render_bound(f)),
- (None, Some(t)) => format!("*-{}", render_bound(t)),
- (None, None) => "*-*".to_string(),
- });
+ let key = range_def
+ .get("key")
+ .and_then(Value::as_str)
+ .map(String::from)
+ .unwrap_or_else(|| match (from, to) {
+ (Some(f), Some(t)) => format!("{}-{}", render_bound(f), render_bound(t)),
+ (Some(f), None) => format!("{}-*", render_bound(f)),
+ (None, Some(t)) => format!("*-{}", render_bound(t)),
+ (None, None) => "*-*".to_string(),
+ });
let mut bucket = json!({
"key": key,
@@ -4632,11 +5327,18 @@ fn run_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs:
// `keyed: true` (or `keyed: false` overriding the default) — render
// buckets as a map keyed by bucket key instead of an array, removing
// the redundant `key` field per ES semantics.
- let keyed = params.get("keyed").and_then(Value::as_bool).unwrap_or(false);
+ let keyed = params
+ .get("keyed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
if keyed {
let mut keyed_map = serde_json::Map::new();
for mut b in buckets {
- let k = b.get("key").and_then(Value::as_str).unwrap_or("").to_string();
+ let k = b
+ .get("key")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
if let Some(obj) = b.as_object_mut() {
obj.remove("key");
}
@@ -4676,7 +5378,12 @@ fn run_missing(params: &Value, docs: &[Value]) -> Value {
// ── Histogram aggregation ─────────────────────────────────────────────────────
-fn run_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_histogram(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": []}),
@@ -4688,7 +5395,10 @@ fn run_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
};
let offset = params.get("offset").and_then(Value::as_f64).unwrap_or(0.0);
- let min_doc_count = params.get("min_doc_count").and_then(|v| v.as_u64()).unwrap_or(0);
+ let min_doc_count = params
+ .get("min_doc_count")
+ .and_then(|v| v.as_u64())
+ .unwrap_or(0);
let extended_bounds = params.get("extended_bounds").and_then(Value::as_object);
let hard_bounds = params.get("hard_bounds").and_then(Value::as_object);
@@ -4709,7 +5419,9 @@ fn run_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
let mut nums = extract_numeric_values(doc, field);
let ranges = if nums.is_empty() {
extract_numeric_ranges(doc, field)
- } else { Vec::new() };
+ } else {
+ Vec::new()
+ };
if nums.is_empty() && ranges.is_empty() {
if let Some(m) = missing_num {
nums.push(m);
@@ -4731,21 +5443,24 @@ fn run_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
// when neither is configured, ES limits to the data's finite span,
// which here reduces to each finite endpoint.
for (from, to) in ranges {
- let clip_lo = hard_min.or(extended_min).unwrap_or({
- if from == f64::MIN { from } else { from }
- });
- let clip_hi = hard_max.or(extended_max).unwrap_or({
- if to == f64::MAX { to } else { to }
- });
+ let clip_lo = hard_min.or(extended_min).unwrap_or(from);
+ let clip_hi = hard_max.or(extended_max).unwrap_or(to);
let lo = from.max(clip_lo);
let hi = to.min(clip_hi);
- if !(hi > lo) { continue; }
+ // Keep the negated `>` verbatim so NaN endpoints skip the bucket
+ // (rewriting to `hi <= lo` would flip the NaN edge case).
+ #[allow(clippy::neg_cmp_op_on_partial_ord)]
+ if !(hi > lo) {
+ continue;
+ }
let k_lo = ((lo - offset) / interval).floor() as i64;
let k_hi = ((hi - offset) / interval).floor() as i64;
for k in k_lo..=k_hi {
let bucket_lo = k as f64 * interval + offset;
// Skip buckets that only touch at a single exclusive edge.
- if to <= bucket_lo { continue; }
+ if to <= bucket_lo {
+ continue;
+ }
if !seen.contains(&k) {
seen.push(k);
buckets.entry(k).or_default().push(i);
@@ -4801,12 +5516,18 @@ fn run_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
if min_doc_count > 0 {
bucket_keys.retain(|key| {
- buckets.get(key).map(|v| v.len() as u64 >= min_doc_count).unwrap_or(false)
+ buckets
+ .get(key)
+ .map(|v| v.len() as u64 >= min_doc_count)
+ .unwrap_or(false)
});
}
let format_pattern: Option<&str> = params.get("format").and_then(Value::as_str);
- let keyed = params.get("keyed").and_then(Value::as_bool).unwrap_or(false);
+ let keyed = params
+ .get("keyed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
let empty_indices: Vec<usize> = Vec::new();
let rendered: Vec<(String, Value)> = bucket_keys
@@ -4814,12 +5535,10 @@ fn run_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
.map(|&key| {
let actual_key = key as f64 * interval + offset;
let doc_indices = buckets.get(&key).unwrap_or(&empty_indices);
- let bucket_docs: Vec<Value> =
- doc_indices.iter().map(|&i| docs[i].clone()).collect();
+ let bucket_docs: Vec<Value> = doc_indices.iter().map(|&i| docs[i].clone()).collect();
let doc_count = sum_doc_count(&bucket_docs);
- let key_as_string = format_pattern
- .map(|p| format_number_pattern(actual_key, p));
+ let key_as_string = format_pattern.map(|p| format_number_pattern(actual_key, p));
let key_json = if actual_key.fract() == 0.0 {
json!(actual_key as i64)
} else {
@@ -4830,7 +5549,9 @@ fn run_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
"doc_count": doc_count
});
if let Some(ref kas) = key_as_string {
- bucket.as_object_mut().unwrap()
+ bucket
+ .as_object_mut()
+ .unwrap()
.insert("key_as_string".to_string(), Value::String(kas.clone()));
}
@@ -4862,19 +5583,28 @@ fn run_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
}
}
-fn run_variable_width_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_variable_width_histogram(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": []}),
};
let num_buckets = params.get("buckets").and_then(Value::as_u64).unwrap_or(10) as usize;
- let mut values: Vec<(f64, usize)> = docs.iter().enumerate()
+ let mut values: Vec<(f64, usize)> = docs
+ .iter()
+ .enumerate()
.filter_map(|(i, doc)| extract_numeric(doc, field).map(|n| (n, i)))
.collect();
values.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal));
- if values.is_empty() { return json!({"buckets": []}); }
+ if values.is_empty() {
+ return json!({"buckets": []});
+ }
// ES variable_width_histogram: adaptive-merge clustering. Start with
// each distinct value as its own bucket, then iteratively merge the
@@ -4914,25 +5644,34 @@ fn run_variable_width_histogram(params: &Value, sub_aggs: Option<&Value>, docs:
left.5 += right.5;
}
let partitions: Vec<(usize, usize)> = buckets.iter().map(|b| (b.0, b.1)).collect();
- let result_buckets: Vec<Value> = partitions.iter().map(|&(s, e)| {
- let chunk = &values[s..e];
- let min_val = chunk.first().map(|v| v.0).unwrap_or(0.0);
- let max_val = chunk.last().map(|v| v.0).unwrap_or(0.0);
- let sum: f64 = chunk.iter().map(|(v, _)| v).sum();
- let key = if chunk.is_empty() { 0.0 } else { sum / chunk.len() as f64 };
- let bucket_docs: Vec<Value> = chunk.iter().map(|(_, i)| docs[*i].clone()).collect();
- let mut bucket = json!({
- "min": min_val, "key": key, "max": max_val,
- "doc_count": chunk.len()
- });
- if let Some(sub) = sub_aggs {
- let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
- if let (Some(bo), Value::Object(so)) = (bucket.as_object_mut(), sub_result) {
- for (k, v) in so { bo.insert(k, v); }
+ let result_buckets: Vec<Value> = partitions
+ .iter()
+ .map(|&(s, e)| {
+ let chunk = &values[s..e];
+ let min_val = chunk.first().map(|v| v.0).unwrap_or(0.0);
+ let max_val = chunk.last().map(|v| v.0).unwrap_or(0.0);
+ let sum: f64 = chunk.iter().map(|(v, _)| v).sum();
+ let key = if chunk.is_empty() {
+ 0.0
+ } else {
+ sum / chunk.len() as f64
+ };
+ let bucket_docs: Vec<Value> = chunk.iter().map(|(_, i)| docs[*i].clone()).collect();
+ let mut bucket = json!({
+ "min": min_val, "key": key, "max": max_val,
+ "doc_count": chunk.len()
+ });
+ if let Some(sub) = sub_aggs {
+ let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
+ if let (Some(bo), Value::Object(so)) = (bucket.as_object_mut(), sub_result) {
+ for (k, v) in so {
+ bo.insert(k, v);
+ }
+ }
}
- }
- bucket
- }).collect();
+ bucket
+ })
+ .collect();
json!({"buckets": result_buckets})
}
@@ -4983,11 +5722,13 @@ fn run_percentiles(params: &Value, docs: &[Value]) -> Value {
let sub_bucket_count: u64 = {
let target = 2u64 * 10u64.pow(hdr_digits);
let mut p = 1u64;
- while p < target { p <<= 1; }
+ while p < target {
+ p <<= 1;
+ }
p
};
let half_sub_bucket = sub_bucket_count / 2; // subBucketHalfCount, e.g. 1024.
- // subBucketHalfCountMagnitude = log2(subBucketCount) - 1.
+ // subBucketHalfCountMagnitude = log2(subBucketCount) - 1.
let sub_bucket_half_count_magnitude = sub_bucket_count.trailing_zeros().saturating_sub(1);
// DoubleHistogram auto-range conversion ratio. ES initialises a
// `DoubleHistogram` with `setAutoResize(true)`; after recording all
@@ -4998,7 +5739,11 @@ fn run_percentiles(params: &Value, docs: &[Value]) -> Value {
// [subBucketHalfCount, subBucketCount) where it is tracked exactly.
let conv_ratio: f64 = if use_hdr && !nums.is_empty() && nums[0] > 0.0 && nums[0].is_finite() {
let l = nums[0].log2().floor().exp2(); // 2^floor(log2(min))
- if l > 0.0 { (half_sub_bucket as f64) / l } else { 1.0 }
+ if l > 0.0 {
+ (half_sub_bucket as f64) / l
+ } else {
+ 1.0
+ }
} else {
1.0
};
@@ -5025,7 +5770,9 @@ fn run_percentiles(params: &Value, docs: &[Value]) -> Value {
(highest_equiv as f64) / conv_ratio
};
let compute = |pct: f64| -> Option<f64> {
- if nums.is_empty() { return None; }
+ if nums.is_empty() {
+ return None;
+ }
if use_hdr {
// Java HdrHistogram.getValueAtPercentile:
// countAtPercentile = (long)((pct / 100 * N) + 0.5) clamped to >=1
@@ -5059,7 +5806,9 @@ fn run_percentiles(params: &Value, docs: &[Value]) -> Value {
.map(|&pct| {
let key = format!("{:.1}", pct);
let val = match compute(pct) {
- Some(v) => serde_json::Number::from_f64(v).map(Value::Number).unwrap_or(Value::Null),
+ Some(v) => serde_json::Number::from_f64(v)
+ .map(Value::Number)
+ .unwrap_or(Value::Null),
None => Value::Null,
};
(key, val)
@@ -5071,10 +5820,14 @@ fn run_percentiles(params: &Value, docs: &[Value]) -> Value {
.iter()
.map(|&pct| {
let val = match compute(pct) {
- Some(v) => serde_json::Number::from_f64(v).map(Value::Number).unwrap_or(Value::Null),
+ Some(v) => serde_json::Number::from_f64(v)
+ .map(Value::Number)
+ .unwrap_or(Value::Null),
None => Value::Null,
};
- let key_num = serde_json::Number::from_f64(pct).map(Value::Number).unwrap_or(Value::Null);
+ let key_num = serde_json::Number::from_f64(pct)
+ .map(Value::Number)
+ .unwrap_or(Value::Null);
json!({ "key": key_num, "value": val })
})
.collect();
@@ -5084,15 +5837,17 @@ fn run_percentiles(params: &Value, docs: &[Value]) -> Value {
// ── Composite aggregation ─────────────────────────────────────────────────────
-fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_composite(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let sources = match params.get("sources").and_then(Value::as_array) {
Some(s) => s,
None => return json!({"buckets": []}),
};
- let size = params
- .get("size")
- .and_then(Value::as_u64)
- .unwrap_or(10) as usize;
+ let size = params.get("size").and_then(Value::as_u64).unwrap_or(10) as usize;
// Each source is { "name": { "type": { "field": "..." } } }
// Build a list of (source_name, source_type, field, type_params).
@@ -5108,7 +5863,12 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
.get("field")
.and_then(Value::as_str)?
.to_string();
- Some((src_name.clone(), src_type.clone(), field, type_params.clone()))
+ Some((
+ src_name.clone(),
+ src_type.clone(),
+ field,
+ type_params.clone(),
+ ))
})
.collect();
@@ -5124,13 +5884,21 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
// Compute the key for each source.
let mut key_parts: Vec<Vec<String>> = Vec::new();
for (_, src_type, field, src_params) in &source_defs {
- let missing_bucket = src_params.get("missing_bucket").and_then(Value::as_bool)
+ let missing_bucket = src_params
+ .get("missing_bucket")
+ .and_then(Value::as_bool)
.unwrap_or(false);
let part_values = match src_type.as_str() {
"terms" => extract_field_values(doc, field),
"histogram" => {
- let interval = src_params.get("interval").and_then(Value::as_f64).unwrap_or(1.0);
- let offset = src_params.get("offset").and_then(Value::as_f64).unwrap_or(0.0);
+ let interval = src_params
+ .get("interval")
+ .and_then(Value::as_f64)
+ .unwrap_or(1.0);
+ let offset = src_params
+ .get("offset")
+ .and_then(Value::as_f64)
+ .unwrap_or(0.0);
let nums = extract_numeric_values(doc, field);
let mut keys: Vec<String> = Vec::with_capacity(nums.len());
for num in nums {
@@ -5144,7 +5912,9 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
keys
}
"date_histogram" => {
- let interval_str = src_params.get("calendar_interval").and_then(Value::as_str)
+ let interval_str = src_params
+ .get("calendar_interval")
+ .and_then(Value::as_str)
.or_else(|| src_params.get("fixed_interval").and_then(Value::as_str))
.or_else(|| src_params.get("interval").and_then(Value::as_str));
let interval_ms = interval_str.and_then(interval_to_ms).unwrap_or(86_400_000);
@@ -5166,16 +5936,24 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
None => 0,
};
// time_zone: resolve to a fixed offset at a representative instant.
- let tz_str = src_params.get("time_zone").and_then(Value::as_str).unwrap_or("");
+ let tz_str = src_params
+ .get("time_zone")
+ .and_then(Value::as_str)
+ .unwrap_or("");
let tz_offset: Option<chrono::FixedOffset> = if tz_str.is_empty() {
None
} else if let Some(off) = parse_time_zone_offset(tz_str) {
Some(off)
} else {
- let sample_ms = extract_date_ms_values(doc, field).into_iter().next().unwrap_or(0);
+ let sample_ms = extract_date_ms_values(doc, field)
+ .into_iter()
+ .next()
+ .unwrap_or(0);
fixed_offset_for_tz_at(tz_str, sample_ms)
};
- let tz_shift_ms: i64 = tz_offset.map(|o| o.local_minus_utc() as i64 * 1000).unwrap_or(0);
+ let tz_shift_ms: i64 = tz_offset
+ .map(|o| o.local_minus_utc() as i64 * 1000)
+ .unwrap_or(0);
let raws = extract_date_ms_values(doc, field);
let mut keys: Vec<String> = Vec::with_capacity(raws.len());
for ts_ms in raws {
@@ -5262,7 +6040,12 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
_ => vec![],
};
if part_values.is_empty() {
- if missing_bucket || params.get("missing_bucket").and_then(Value::as_bool).unwrap_or(false) {
+ if missing_bucket
+ || params
+ .get("missing_bucket")
+ .and_then(Value::as_bool)
+ .unwrap_or(false)
+ {
key_parts.push(vec!["__null__".to_string()]);
} else {
key_parts.push(vec![]);
@@ -5330,7 +6113,11 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
/// numeric sorting; returns `(0, 0, 0)` on parse error.
fn geotile_parse(s: &str) -> (i64, i64, i64) {
let mut parts = s.splitn(3, '/').map(|p| p.parse::<i64>().unwrap_or(0));
- (parts.next().unwrap_or(0), parts.next().unwrap_or(0), parts.next().unwrap_or(0))
+ (
+ parts.next().unwrap_or(0),
+ parts.next().unwrap_or(0),
+ parts.next().unwrap_or(0),
+ )
}
let mut sorted_keys: Vec<Vec<String>> = bucket_map.keys().cloned().collect();
sorted_keys.sort_by(|a, b| {
@@ -5393,10 +6180,8 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
return (sec * 1000).to_string();
}
}
- if fmt == Some("epoch_millis") {
- if s.parse::<i64>().is_ok() {
- return s.clone();
- }
+ if fmt == Some("epoch_millis") && s.parse::<i64>().is_ok() {
+ return s.clone();
}
// Honor the source's `time_zone` when
// parsing the cursor string: a cursor
@@ -5410,15 +6195,22 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
.unwrap_or("");
if !tz_str.is_empty() {
if let Some(off) = parse_time_zone_offset(tz_str) {
- if let Ok(d) = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d") {
+ if let Ok(d) =
+ chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d")
+ {
let local = d.and_hms_opt(0, 0, 0).unwrap();
- let utc = local - chrono::Duration::seconds(off.local_minus_utc() as i64);
+ let utc = local
+ - chrono::Duration::seconds(
+ off.local_minus_utc() as i64
+ );
let ms = utc.and_utc().timestamp_millis();
return ms.to_string();
}
}
}
- parse_date_ms(v).map(|ms| ms.to_string()).unwrap_or_else(|| s.clone())
+ parse_date_ms(v)
+ .map(|ms| ms.to_string())
+ .unwrap_or_else(|| s.clone())
} else {
s.clone()
}
@@ -5486,12 +6278,11 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
.iter()
.map(|key| {
let doc_indices = &bucket_map[key];
- let bucket_docs: Vec<Value> =
- doc_indices.iter().map(|&i| docs[i].clone()).collect();
+ let bucket_docs: Vec<Value> = doc_indices.iter().map(|&i| docs[i].clone()).collect();
let doc_count = sum_doc_count(&bucket_docs);
let mut key_obj = serde_json::Map::new();
- for (i, (src_name, src_type, _, _)) in source_defs.iter().enumerate() {
+ for (i, (src_name, _src_type, _, _)) in source_defs.iter().enumerate() {
let val = key.get(i).cloned().unwrap_or_default();
let (_name_slot, src_type, _field, src_params) = &source_defs[i];
if val == "__null__" {
@@ -5506,7 +6297,10 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
if fmt.is_none() {
key_obj.insert(src_name.clone(), json!(ts));
} else {
- let tz_str = src_params.get("time_zone").and_then(Value::as_str).unwrap_or("");
+ let tz_str = src_params
+ .get("time_zone")
+ .and_then(Value::as_str)
+ .unwrap_or("");
let tz_offset: Option<chrono::FixedOffset> = if tz_str.is_empty() {
None
} else if let Some(off) = parse_time_zone_offset(tz_str) {
@@ -5515,21 +6309,26 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
fixed_offset_for_tz_at(tz_str, ts)
};
let rendered = if let Some(off) = tz_offset {
- let dt_utc = chrono::DateTime::from_timestamp_millis(ts).unwrap_or_default();
+ let dt_utc =
+ chrono::DateTime::from_timestamp_millis(ts).unwrap_or_default();
let dt_local = dt_utc.with_timezone(&off);
match fmt {
Some("epoch_millis") => ts.to_string(),
Some("epoch_second") => (ts / 1000).to_string(),
- Some("iso8601") | Some("strict_date_time") | Some("date_time") =>
- dt_local.format("%Y-%m-%dT%H:%M:%S%.3f%:z").to_string(),
- Some(_) => {
- let strf = java_to_strftime(fmt.unwrap());
+ Some("iso8601")
+ | Some("strict_date_time")
+ | Some("date_time") => {
+ dt_local.format("%Y-%m-%dT%H:%M:%S%.3f%:z").to_string()
+ }
+ Some(f) => {
+ let strf = java_to_strftime(f);
dt_local.format(&strf).to_string()
}
None => unreachable!(),
}
} else {
- let dt_val = chrono::DateTime::from_timestamp_millis(ts).unwrap_or_default();
+ let dt_val =
+ chrono::DateTime::from_timestamp_millis(ts).unwrap_or_default();
render_date_format(fmt, ts, dt_val)
};
key_obj.insert(src_name.clone(), Value::String(rendered));
@@ -5541,7 +6340,12 @@ fn run_composite(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_d
if let Ok(n) = val.parse::<i64>() {
key_obj.insert(src_name.clone(), json!(n));
} else if let Ok(f) = val.parse::<f64>() {
- key_obj.insert(src_name.clone(), serde_json::Number::from_f64(f).map(Value::Number).unwrap_or(Value::String(val)));
+ key_obj.insert(
+ src_name.clone(),
+ serde_json::Number::from_f64(f)
+ .map(Value::Number)
+ .unwrap_or(Value::String(val)),
+ );
} else {
key_obj.insert(src_name.clone(), Value::String(val));
}
@@ -5616,8 +6420,12 @@ fn murmur3_32_partition_hash(data: &[u8]) -> u32 {
let tail_start = nblocks * 4;
let tail_len = data.len() - tail_start;
let mut k1: u32 = 0;
- if tail_len >= 3 { k1 ^= (data[tail_start + 2] as u32) << 16; }
- if tail_len >= 2 { k1 ^= (data[tail_start + 1] as u32) << 8; }
+ if tail_len >= 3 {
+ k1 ^= (data[tail_start + 2] as u32) << 16;
+ }
+ if tail_len >= 2 {
+ k1 ^= (data[tail_start + 1] as u32) << 8;
+ }
if tail_len >= 1 {
k1 ^= data[tail_start] as u32;
k1 = k1.wrapping_mul(c1);
@@ -5646,14 +6454,13 @@ fn run_significant_terms(params: &Value, result_docs: &[Value], all_docs: &[Valu
// Optional `background_filter`: restricts the background-corpus comparison
// to docs matching the filter (typically a `terms` query). Apply the
// filter to all_docs to compute bg_counts and bg_total.
- let bg_docs_owned: Option<Vec<Value>> = params
- .get("background_filter")
- .map(|filter| {
- all_docs.iter()
- .filter(|d| matches_simple_filter(d, filter))
- .cloned()
- .collect()
- });
+ let bg_docs_owned: Option<Vec<Value>> = params.get("background_filter").map(|filter| {
+ all_docs
+ .iter()
+ .filter(|d| matches_simple_filter(d, filter))
+ .cloned()
+ .collect()
+ });
let bg_docs: &[Value] = bg_docs_owned.as_deref().unwrap_or(all_docs);
let all_total = bg_docs.len();
@@ -5677,7 +6484,9 @@ fn run_significant_terms(params: &Value, result_docs: &[Value], all_docs: &[Valu
for v in raw {
if v.contains(char::is_whitespace) {
for tok in v.split_whitespace() {
- let t = tok.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase();
+ let t = tok
+ .trim_matches(|c: char| !c.is_alphanumeric())
+ .to_lowercase();
if !t.is_empty() {
out.insert(t);
}
@@ -5714,14 +6523,18 @@ fn run_significant_terms(params: &Value, result_docs: &[Value], all_docs: &[Valu
// ES `include` / `exclude`: when given as an array of values, treat
// as an allow / deny set. (Regex form is intentionally rejected
// upstream for IP fields per ES semantics.)
- let include_set: Option<HashSet<String>> = params
- .get("include")
- .and_then(Value::as_array)
- .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect());
- let exclude_set: Option<HashSet<String>> = params
- .get("exclude")
- .and_then(Value::as_array)
- .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect());
+ let include_set: Option<HashSet<String>> =
+ params.get("include").and_then(Value::as_array).map(|arr| {
+ arr.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect()
+ });
+ let exclude_set: Option<HashSet<String>> =
+ params.get("exclude").and_then(Value::as_array).map(|arr| {
+ arr.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect()
+ });
// ES `min_doc_count` (default 3) — exclude terms whose foreground
// count is below the threshold. The IP test sets min_doc_count:1
@@ -5743,10 +6556,14 @@ fn run_significant_terms(params: &Value, result_docs: &[Value], all_docs: &[Valu
return None;
}
if let Some(ref inc) = include_set {
- if !inc.contains(&term) { return None; }
+ if !inc.contains(&term) {
+ return None;
+ }
}
if let Some(ref exc) = exclude_set {
- if exc.contains(&term) { return None; }
+ if exc.contains(&term) {
+ return None;
+ }
}
let result_freq = rc as f64 / result_total_f;
let bg_count = *bg_counts.get(&term).unwrap_or(&0);
@@ -5815,7 +6632,9 @@ fn matches_simple_filter(doc: &Value, filter: &Value) -> bool {
Some(o) => o,
None => return false,
};
- if obj.contains_key("match_all") { return true; }
+ if obj.contains_key("match_all") {
+ return true;
+ }
// `match` on a single field — treat as keyword-equivalent term match
// (stringify both sides). For text fields this is a rough
// approximation but sufficient for background_filter use cases where
@@ -5849,13 +6668,19 @@ fn matches_simple_filter(doc: &Value, filter: &Value) -> bool {
}
}
if let Some(b) = obj.get("bool").and_then(Value::as_object) {
- let must_ok = b.get("must").and_then(Value::as_array)
+ let must_ok = b
+ .get("must")
+ .and_then(Value::as_array)
.map(|arr| arr.iter().all(|f| matches_simple_filter(doc, f)))
.unwrap_or(true);
- let filter_ok = b.get("filter").and_then(Value::as_array)
+ let filter_ok = b
+ .get("filter")
+ .and_then(Value::as_array)
.map(|arr| arr.iter().all(|f| matches_simple_filter(doc, f)))
.unwrap_or(true);
- let must_not_ok = b.get("must_not").and_then(Value::as_array)
+ let must_not_ok = b
+ .get("must_not")
+ .and_then(Value::as_array)
.map(|arr| arr.iter().all(|f| !matches_simple_filter(doc, f)))
.unwrap_or(true);
return must_ok && filter_ok && must_not_ok;
@@ -5864,7 +6689,9 @@ fn matches_simple_filter(doc: &Value, filter: &Value) -> bool {
}
fn values_match(dv: &Value, target: &Value) -> bool {
- if dv == target { return true; }
+ if dv == target {
+ return true;
+ }
match (dv, target) {
(Value::Array(arr), _) => arr.iter().any(|e| values_match(e, target)),
(Value::String(s), Value::String(t)) => s == t,
@@ -5888,48 +6715,53 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
None => return json!({"buckets": []}),
};
let size = params.get("size").and_then(Value::as_u64).unwrap_or(10) as usize;
- let min_doc_count = params.get("min_doc_count").and_then(Value::as_u64).unwrap_or(3);
+ let min_doc_count = params
+ .get("min_doc_count")
+ .and_then(Value::as_u64)
+ .unwrap_or(3);
// background_filter: restrict the background corpus to docs matching
// the filter. sig_text's bg frequencies come from this subset when
// provided, so `bg_freq` can collapse to 0 for terms absent from the
// filtered bg (e.g. "bad" when ugly=true only keeps "good" docs).
- let bg_docs_owned: Option<Vec<Value>> = params
- .get("background_filter")
- .map(|filter| {
- all_docs
- .iter()
- .filter(|d| matches_simple_filter(d, filter))
- .cloned()
- .collect()
- });
+ let bg_docs_owned: Option<Vec<Value>> = params.get("background_filter").map(|filter| {
+ all_docs
+ .iter()
+ .filter(|d| matches_simple_filter(d, filter))
+ .cloned()
+ .collect()
+ });
let all_docs: &[Value] = bg_docs_owned.as_deref().unwrap_or(all_docs);
// Optional include/exclude — ES accepts a Lucene-style regex pattern
// (anchored full-term match, not substring) or an explicit array of
// terms. Honor both.
- let (include_re, include_set): (Option<regex::Regex>, Option<HashSet<String>>) = match params.get("include") {
- Some(Value::String(s)) => (
- regex::Regex::new(&format!("^(?:{})$", s)).ok(),
- None,
- ),
- Some(Value::Array(a)) => (
- None,
- Some(a.iter().filter_map(|v| v.as_str().map(String::from)).collect()),
- ),
- _ => (None, None),
- };
- let (exclude_re, exclude_set): (Option<regex::Regex>, Option<HashSet<String>>) = match params.get("exclude") {
- Some(Value::String(s)) => (
- regex::Regex::new(&format!("^(?:{})$", s)).ok(),
- None,
- ),
- Some(Value::Array(a)) => (
- None,
- Some(a.iter().filter_map(|v| v.as_str().map(String::from)).collect()),
- ),
- _ => (None, None),
- };
+ let (include_re, include_set): (Option<regex::Regex>, Option<HashSet<String>>) =
+ match params.get("include") {
+ Some(Value::String(s)) => (regex::Regex::new(&format!("^(?:{})$", s)).ok(), None),
+ Some(Value::Array(a)) => (
+ None,
+ Some(
+ a.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect(),
+ ),
+ ),
+ _ => (None, None),
+ };
+ let (exclude_re, exclude_set): (Option<regex::Regex>, Option<HashSet<String>>) =
+ match params.get("exclude") {
+ Some(Value::String(s)) => (regex::Regex::new(&format!("^(?:{})$", s)).ok(), None),
+ Some(Value::Array(a)) => (
+ None,
+ Some(
+ a.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect(),
+ ),
+ ),
+ _ => (None, None),
+ };
let result_total = result_docs.len();
let all_total = all_docs.len().max(1);
@@ -5937,14 +6769,21 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
return json!({ "buckets": [], "doc_count": 0 });
}
- let filter_dup = params.get("filter_duplicate_text").and_then(Value::as_bool).unwrap_or(false);
+ let filter_dup = params
+ .get("filter_duplicate_text")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
// ES `source_fields` override: when the target field isn't stored, read
// text from these source fields instead. Defaults to the target field.
let source_fields: Vec<String> = params
.get("source_fields")
.and_then(Value::as_array)
- .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect())
+ .map(|arr| {
+ arr.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect()
+ })
.unwrap_or_else(|| vec![field.to_string()]);
let tokenize_doc = |doc: &Value| -> HashSet<String> {
@@ -5952,9 +6791,13 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
for src_field in &source_fields {
for raw in extract_field_values(doc, src_field) {
for tok in raw.split(|c: char| !c.is_alphanumeric()) {
- if tok.len() < 2 { continue; }
+ if tok.len() < 2 {
+ continue;
+ }
let lower = tok.to_lowercase();
- if SIG_TEXT_STOPWORDS.contains(&lower.as_str()) { continue; }
+ if SIG_TEXT_STOPWORDS.contains(&lower.as_str()) {
+ continue;
+ }
out.insert(lower);
}
}
@@ -5971,7 +6814,9 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
for src_field in &source_fields {
for raw in extract_field_values(doc, src_field) {
for tok in raw.split(|c: char| !c.is_alphanumeric()) {
- if tok.len() < 2 { continue; }
+ if tok.len() < 2 {
+ continue;
+ }
seq.push(tok.to_lowercase());
}
}
@@ -5984,10 +6829,13 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
// ES's `SignificantTextAggregator` logic for deduping boilerplate.
const SIG_SHINGLE: usize = 6;
let dup_token_sets: Vec<HashSet<String>> = if filter_dup {
- let per_doc_tokens: Vec<Vec<String>> = result_docs.iter().map(&tokenize_doc_ordered).collect();
+ let per_doc_tokens: Vec<Vec<String>> =
+ result_docs.iter().map(&tokenize_doc_ordered).collect();
let mut shingle_docs: HashMap<String, HashSet<usize>> = HashMap::new();
for (i, toks) in per_doc_tokens.iter().enumerate() {
- if toks.len() < SIG_SHINGLE { continue; }
+ if toks.len() < SIG_SHINGLE {
+ continue;
+ }
for w in toks.windows(SIG_SHINGLE) {
shingle_docs.entry(w.join(" ")).or_default().insert(i);
}
@@ -5996,7 +6844,9 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
.iter()
.map(|toks| {
let mut dups: HashSet<String> = HashSet::new();
- if toks.len() < SIG_SHINGLE { return dups; }
+ if toks.len() < SIG_SHINGLE {
+ return dups;
+ }
for w in toks.windows(SIG_SHINGLE) {
let n = shingle_docs.get(&w.join(" ")).map(|s| s.len()).unwrap_or(0);
if n > 1 {
@@ -6018,13 +6868,17 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
for (i, doc) in result_docs.iter().enumerate() {
let dups = &dup_token_sets[i];
for t in tokenize_doc(doc) {
- if dups.contains(&t) { continue; }
+ if dups.contains(&t) {
+ continue;
+ }
*result_counts.entry(t).or_insert(0) += 1;
}
}
let mut bg_counts: HashMap<String, u64> = HashMap::new();
for doc in all_docs {
- for t in tokenize_doc(doc) { *bg_counts.entry(t).or_insert(0) += 1; }
+ for t in tokenize_doc(doc) {
+ *bg_counts.entry(t).or_insert(0) += 1;
+ }
}
let result_total_f = result_total as f64;
@@ -6039,18 +6893,28 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
let mut scored: Vec<(String, u64, u64, f64)> = result_counts
.into_iter()
.filter_map(|(term, rc)| {
- if rc < min_doc_count { return None; }
+ if rc < min_doc_count {
+ return None;
+ }
if let Some(ref re) = include_re {
- if !re.is_match(&term) { return None; }
+ if !re.is_match(&term) {
+ return None;
+ }
}
if let Some(ref set) = include_set {
- if !set.contains(&term) { return None; }
+ if !set.contains(&term) {
+ return None;
+ }
}
if let Some(ref re) = exclude_re {
- if re.is_match(&term) { return None; }
+ if re.is_match(&term) {
+ return None;
+ }
}
if let Some(ref set) = exclude_set {
- if set.contains(&term) { return None; }
+ if set.contains(&term) {
+ return None;
+ }
}
let bg = *bg_counts.get(&term).unwrap_or(&0);
let result_freq = rc as f64 / result_total_f;
@@ -6070,9 +6934,15 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
let n_21 = (bg as i64 - rc as i64).max(0) as f64;
let n_22 = n_2_dot - n_21;
// Require fg concentration strictly greater than bg.
- if result_freq <= bg_freq { return None; }
+ if result_freq <= bg_freq {
+ return None;
+ }
let ln_safe = |num: f64, den: f64| -> f64 {
- if num <= 0.0 || den <= 0.0 { 0.0 } else { (num / den).ln() }
+ if num <= 0.0 || den <= 0.0 {
+ 0.0
+ } else {
+ (num / den).ln()
+ }
};
let mi = (n_11 / n) * ln_safe(n * n_11, n_1_dot * n_dot_1)
+ (n_12 / n) * ln_safe(n * n_12, n_1_dot * n_dot_2)
@@ -6081,7 +6951,9 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
// ES's default significance threshold on MI is around
// 0.05 (i.e. the term must explain ≥ 5% of the
// classification signal). Weaker terms are dropped.
- if mi < 0.05 { return None; }
+ if mi < 0.05 {
+ return None;
+ }
mi
} else if use_gnd {
// Google Normalized Distance: smaller is more significant.
@@ -6091,14 +6963,20 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
let f_x = result_total_f;
let f_y = bg as f64;
let n = all_total_f;
- if f_xy == 0.0 || f_x == 0.0 || f_y == 0.0 { return None; }
+ if f_xy == 0.0 || f_x == 0.0 || f_y == 0.0 {
+ return None;
+ }
let max_xy = f_x.ln().max(f_y.ln());
let min_xy = f_x.ln().min(f_y.ln());
let denom = n.ln() - min_xy;
- if denom == 0.0 { return None; }
+ if denom == 0.0 {
+ return None;
+ }
let gnd = (max_xy - f_xy.ln()) / denom;
// Drop strongly-unrelated pairs (gnd >= 1 means disjoint).
- if gnd >= 1.0 { return None; }
+ if gnd >= 1.0 {
+ return None;
+ }
-gnd
} else if use_chi {
// chi_square: standard 2x2 contingency test.
@@ -6111,10 +6989,14 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
let n_12 = n_1_dot - n_11;
let n_21 = (bg as i64 - rc as i64).max(0) as f64;
let n_22 = n_2_dot - n_21;
- if result_freq <= bg_freq { return None; }
+ if result_freq <= bg_freq {
+ return None;
+ }
let num = n * (n_11 * n_22 - n_12 * n_21).powi(2);
let den = n_1_dot * n_2_dot * n_dot_1 * n_dot_2;
- if den == 0.0 { return None; }
+ if den == 0.0 {
+ return None;
+ }
num / den
} else {
// JLH-like score: term completely absent from the background
@@ -6137,23 +7019,26 @@ fn run_significant_text(params: &Value, result_docs: &[Value], all_docs: &[Value
// SignificantTermsAggregator, which iterates terms in reverse
// BytesRef order when scores collide).
scored.sort_by(|a, b| {
- b.3.partial_cmp(&a.3).unwrap_or(std::cmp::Ordering::Equal)
+ b.3.partial_cmp(&a.3)
+ .unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| b.0.cmp(&a.0))
});
scored.truncate(size);
let buckets: Vec<Value> = scored
.into_iter()
- .map(|(key, doc_count, bg_count, score)| json!({
- "key": key, "doc_count": doc_count, "bg_count": bg_count, "score": score,
- }))
+ .map(|(key, doc_count, bg_count, score)| {
+ json!({
+ "key": key, "doc_count": doc_count, "bg_count": bg_count, "score": score,
+ })
+ })
.collect();
json!({ "doc_count": result_total, "bg_count": all_total, "buckets": buckets })
}
const SIG_TEXT_STOPWORDS: &[&str] = &[
- "a","an","and","are","as","at","be","by","for","from","has","he","in","is","it",
- "its","of","on","that","the","to","was","were","will","with",
+ "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "has", "he", "in", "is", "it",
+ "its", "of", "on", "that", "the", "to", "was", "were", "will", "with",
];
// ── Top hits sub-aggregation ──────────────────────────────────────────────────
@@ -6172,7 +7057,10 @@ fn extract_match_terms_from_query_value(q: &Value) -> Option<Vec<String>> {
if let Some(mo) = val.as_object() {
for (_, v) in mo {
let s = if let Some(o) = v.as_object() {
- o.get("query").or_else(|| o.get("value")).and_then(Value::as_str).map(String::from)
+ o.get("query")
+ .or_else(|| o.get("value"))
+ .and_then(Value::as_str)
+ .map(String::from)
} else {
v.as_str().map(String::from)
};
@@ -6192,7 +7080,9 @@ fn extract_match_terms_from_query_value(q: &Value) -> Option<Vec<String>> {
} else {
v.as_str().map(String::from)
};
- if let Some(s) = s { out.push(s.to_lowercase()); }
+ if let Some(s) = s {
+ out.push(s.to_lowercase());
+ }
}
}
}
@@ -6214,42 +7104,56 @@ fn extract_match_terms_from_query_value(q: &Value) -> Option<Vec<String>> {
}
}
collect(obj, &mut out);
- if out.is_empty() { None } else { Some(out) }
+ if out.is_empty() {
+ None
+ } else {
+ Some(out)
+ }
}
/// Build a highlight fragment for a text by wrapping case-insensitive
/// occurrences of each term in `pre`..`post`. Returns None when no term
/// is found. Preserves original text case — only the tag sites change.
-fn highlight_text_with_terms(text: &str, terms: &[String], pre: &str, post: &str) -> Option<String> {
+fn highlight_text_with_terms(
+ text: &str,
+ terms: &[String],
+ pre: &str,
+ post: &str,
+) -> Option<String> {
let text_lower = text.to_lowercase();
// Find all match ranges (byte offsets) then build the output.
let mut ranges: Vec<(usize, usize)> = Vec::new();
for term in terms {
- if term.is_empty() { continue; }
+ if term.is_empty() {
+ continue;
+ }
let term_lc = term.to_lowercase();
let mut start = 0usize;
while let Some(pos) = text_lower[start..].find(&term_lc) {
let abs = start + pos;
let end = abs + term_lc.len();
// Ensure token boundaries — avoid matching 'the' inside 'therefore'.
- let before_ok = abs == 0
- || !text_lower.as_bytes()[abs - 1].is_ascii_alphanumeric();
- let after_ok = end == text_lower.len()
- || !text_lower.as_bytes()[end].is_ascii_alphanumeric();
+ let before_ok = abs == 0 || !text_lower.as_bytes()[abs - 1].is_ascii_alphanumeric();
+ let after_ok =
+ end == text_lower.len() || !text_lower.as_bytes()[end].is_ascii_alphanumeric();
if before_ok && after_ok {
ranges.push((abs, end));
}
start = end.max(start + 1);
}
}
- if ranges.is_empty() { return None; }
+ if ranges.is_empty() {
+ return None;
+ }
// Sort + merge overlapping ranges.
ranges.sort_by_key(|r| r.0);
let mut merged: Vec<(usize, usize)> = Vec::new();
for r in ranges {
if let Some(last) = merged.last_mut() {
if r.0 <= last.1 {
- if r.1 > last.1 { last.1 = r.1; }
+ if r.1 > last.1 {
+ last.1 = r.1;
+ }
continue;
}
}
@@ -6274,7 +7178,9 @@ fn painless_value_to_json(v: crate::painless::PainlessValue) -> Value {
match v {
P::Null => Value::Null,
P::Bool(b) => Value::Bool(b),
- P::Number(n) => serde_json::Number::from_f64(n).map(Value::Number).unwrap_or(Value::Null),
+ P::Number(n) => serde_json::Number::from_f64(n)
+ .map(Value::Number)
+ .unwrap_or(Value::Null),
P::String(s) => Value::String(s),
P::Array(a) => Value::Array(a.into_iter().map(painless_value_to_json).collect()),
P::Object(o) => Value::Object(o),
@@ -6299,14 +7205,16 @@ pub(crate) fn run_top_hits_with_total(
// Optional _source filtering. `false` / `[]` disables source; an array
// of strings keeps only listed top-level fields.
- let source_fields: Option<Vec<String>> = params
- .get("_source")
- .and_then(|v| match v {
- Value::Array(arr) => Some(arr.iter().filter_map(|e| e.as_str().map(String::from)).collect()),
- Value::String(s) => Some(vec![s.clone()]),
- Value::Bool(false) => Some(vec![]), // empty = return nothing
- _ => None,
- });
+ let source_fields: Option<Vec<String>> = params.get("_source").and_then(|v| match v {
+ Value::Array(arr) => Some(
+ arr.iter()
+ .filter_map(|e| e.as_str().map(String::from))
+ .collect(),
+ ),
+ Value::String(s) => Some(vec![s.clone()]),
+ Value::Bool(false) => Some(vec![]), // empty = return nothing
+ _ => None,
+ });
// Optional sort spec. ES accepts any of:
// [{"field": "desc"}] — array of shorthand
@@ -6328,7 +7236,8 @@ pub(crate) fn run_top_hits_with_total(
sorted_docs.sort_by(|a, b| {
let sa = a.get("_score").and_then(Value::as_f64).unwrap_or(0.0);
let sb = b.get("_score").and_then(Value::as_f64).unwrap_or(0.0);
- sb.partial_cmp(&sa).unwrap_or(std::cmp::Ordering::Equal)
+ sb.partial_cmp(&sa)
+ .unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| {
let qa = a.get("_seq_no").and_then(Value::as_i64).unwrap_or(i64::MAX);
let qb = b.get("_seq_no").and_then(Value::as_i64).unwrap_or(i64::MAX);
@@ -6351,8 +7260,11 @@ pub(crate) fn run_top_hits_with_total(
for (sort_field, sort_opts) in &sort_obj {
let desc = match sort_opts {
Value::String(s) => s == "desc",
- Value::Object(o) => o.get("order")
- .and_then(Value::as_str).map(|v| v == "desc").unwrap_or(false),
+ Value::Object(o) => o
+ .get("order")
+ .and_then(Value::as_str)
+ .map(|v| v == "desc")
+ .unwrap_or(false),
_ => false,
};
let sf = sort_field.clone();
@@ -6387,7 +7299,11 @@ pub(crate) fn run_top_hits_with_total(
let mut vals = extract_field_values(doc, &sf);
if !vals.is_empty() {
vals.sort();
- return if desc { vals.pop() } else { Some(vals.remove(0)) };
+ return if desc {
+ vals.pop()
+ } else {
+ Some(vals.remove(0))
+ };
}
}
extract_field_values(doc, &sf).into_iter().next()
@@ -6430,7 +7346,11 @@ pub(crate) fn run_top_hits_with_total(
match (av, bv) {
(Some(x), Some(y)) => {
let cmp = x.partial_cmp(&y).unwrap_or(std::cmp::Ordering::Equal);
- if desc { cmp.reverse() } else { cmp }
+ if desc {
+ cmp.reverse()
+ } else {
+ cmp
+ }
}
(Some(_), None) => std::cmp::Ordering::Less,
(None, Some(_)) => std::cmp::Ordering::Greater,
@@ -6540,7 +7460,7 @@ pub(crate) fn run_top_hits_with_total(
// ES emits this for fields that triggered ignore_malformed
// / ignore_above per-doc.
if let Some(ig) = doc.get("_ignored").cloned() {
- if !ig.is_null() && ig.as_array().map_or(true, |a| !a.is_empty()) {
+ if !ig.is_null() && ig.as_array().is_none_or(|a| !a.is_empty()) {
hit_obj.insert("_ignored".to_string(), ig);
}
}
@@ -6753,7 +7673,7 @@ pub(crate) fn run_top_hits_with_total(
// Top-level highlight_query overrides outer query terms.
let top_hl_query_terms: Option<Vec<String>> = hl_cfg
.get("highlight_query")
- .and_then(|q| extract_match_terms_from_query_value(q));
+ .and_then(extract_match_terms_from_query_value);
if let Some(fields) = hl_cfg.get("fields").and_then(Value::as_object) {
let mut hl_out = serde_json::Map::new();
for (fname, fopts) in fields {
@@ -6809,8 +7729,16 @@ pub(crate) fn run_top_hits_with_total(
// ── Sampler aggregation ───────────────────────────────────────────────────────
/// Takes a random sample of documents and runs sub-aggregations on the sample.
-pub(crate) fn run_sampler(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
- let shard_size = params.get("shard_size").and_then(Value::as_u64).unwrap_or(200) as usize;
+pub(crate) fn run_sampler(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
+ let shard_size = params
+ .get("shard_size")
+ .and_then(Value::as_u64)
+ .unwrap_or(200) as usize;
// ES sampler feeds the top shard_size docs by _score — take the score-descending slice.
let mut ordered: Vec<Value> = docs.to_vec();
@@ -6840,9 +7768,20 @@ pub(crate) fn run_sampler(params: &Value, sub_aggs: Option<&Value>, docs: &[Valu
/// `diversified_sampler` — take a sample while capping the number of docs
/// that share the same `field` value (default `max_docs_per_value` = 1).
/// Diversity acts as a pre-filter, shrinking the working set before sub-aggs run.
-fn run_diversified_sampler(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
- let shard_size = params.get("shard_size").and_then(Value::as_u64).unwrap_or(200) as usize;
- let max_per = params.get("max_docs_per_value").and_then(Value::as_u64).unwrap_or(1) as usize;
+fn run_diversified_sampler(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
+ let shard_size = params
+ .get("shard_size")
+ .and_then(Value::as_u64)
+ .unwrap_or(200) as usize;
+ let max_per = params
+ .get("max_docs_per_value")
+ .and_then(Value::as_u64)
+ .unwrap_or(1) as usize;
let field = params.get("field").and_then(Value::as_str).unwrap_or("");
// ES semantics: diversified_sampler feeds the sampler the docs in
@@ -6858,16 +7797,26 @@ fn run_diversified_sampler(params: &Value, sub_aggs: Option<&Value>, docs: &[Val
let mut counts: HashMap<String, usize> = HashMap::new();
let mut sample: Vec<Value> = Vec::new();
for doc in &ordered {
- if sample.len() >= shard_size { break; }
+ if sample.len() >= shard_size {
+ break;
+ }
let values = if field.is_empty() {
vec!["__no_field__".to_string()]
} else {
let vs = extract_field_values(doc, field);
- if vs.is_empty() { vec!["__null__".to_string()] } else { vs }
+ if vs.is_empty() {
+ vec!["__null__".to_string()]
+ } else {
+ vs
+ }
};
// A doc is admitted if ANY of its field values has spare capacity.
- let can_admit = values.iter().any(|v| counts.get(v).copied().unwrap_or(0) < max_per);
- if !can_admit { continue; }
+ let can_admit = values
+ .iter()
+ .any(|v| counts.get(v).copied().unwrap_or(0) < max_per);
+ if !can_admit {
+ continue;
+ }
for v in &values {
*counts.entry(v.clone()).or_insert(0) += 1;
}
@@ -6881,7 +7830,9 @@ fn run_diversified_sampler(params: &Value, sub_aggs: Option<&Value>, docs: &[Val
if let Some(sub) = sub_aggs {
let sub_result = run_aggs_with_all(sub, &sample, all_docs);
if let Value::Object(sub_obj) = sub_result {
- for (k, v) in sub_obj { result.insert(k, v); }
+ for (k, v) in sub_obj {
+ result.insert(k, v);
+ }
}
}
Value::Object(result)
@@ -6908,7 +7859,10 @@ fn run_time_series(
all_docs: &[Value],
) -> Value {
let size = params.get("size").and_then(Value::as_u64).unwrap_or(10000) as usize;
- let keyed = params.get("keyed").and_then(Value::as_bool).unwrap_or(false);
+ let keyed = params
+ .get("keyed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
// Infer dimension fields from doc metadata. xerj stores the
// routing path segments in `__xy_routing_path__` when the index
@@ -6944,10 +7898,10 @@ fn run_time_series(
// ahead of `key` (the actual `time_series_dimension`).
if let Some(first) = docs.first().and_then(Value::as_object) {
const METRIC_NAMES: &[&str] = &[
- "val", "value", "count", "sum", "num", "n", "size",
- "gauge", "counter", "metric",
+ "val", "value", "count", "sum", "num", "n", "size", "gauge", "counter", "metric",
];
- let candidate_keys: Vec<&String> = first.keys()
+ let candidate_keys: Vec<&String> = first
+ .keys()
.filter(|k| {
!k.starts_with('_')
&& !k.starts_with('@')
@@ -6994,8 +7948,14 @@ fn run_time_series(
// match ES. (The search-level `hits.total` is computed outside this
// function and is not affected by this dedup — see notes.)
use std::collections::BTreeMap;
- let mut groups: BTreeMap<String, (Vec<Value>, serde_json::Map<String, Value>, HashMap<i64, usize>)> =
- BTreeMap::new();
+ let mut groups: BTreeMap<
+ String,
+ (
+ Vec<Value>,
+ serde_json::Map<String, Value>,
+ HashMap<i64, usize>,
+ ),
+ > = BTreeMap::new();
for doc in docs {
let mut key_obj = serde_json::Map::new();
let mut key_parts: Vec<String> = Vec::new();
@@ -7015,7 +7975,7 @@ fn run_time_series(
let entry = groups
.entry(key_str)
.or_insert_with(|| (Vec::new(), key_obj, HashMap::new()));
- match doc.get("@timestamp").and_then(|v| parse_date_ms(v)) {
+ match doc.get("@timestamp").and_then(parse_date_ms) {
Some(ts) => {
if let Some(&idx) = entry.2.get(&ts) {
entry.0[idx] = doc.clone(); // last wins
@@ -7070,15 +8030,16 @@ fn run_time_series(
if let Some(sub) = sub_aggs {
let sub_result = run_aggs_with_all(sub, bucket_docs, all_docs);
if let Value::Object(sub_obj) = sub_result {
- for (k, v) in sub_obj { bucket.insert(k, v); }
+ for (k, v) in sub_obj {
+ bucket.insert(k, v);
+ }
}
}
Value::Object(bucket)
};
- let is_selected = |k: &str| -> bool {
- selected.as_ref().map(|s| s.contains(k)).unwrap_or(true)
- };
+ let is_selected =
+ |k: &str| -> bool { selected.as_ref().map(|s| s.contains(k)).unwrap_or(true) };
let mut bucket_list: Vec<Value> = Vec::new();
for (k, (bucket_docs, key_obj, _seen)) in groups.iter() {
@@ -7106,7 +8067,12 @@ fn run_time_series(
/// Creates a matrix of which filter combinations co-occur in the document set.
/// Returns one bucket per filter and one bucket per pair of filters that co-occur.
-fn run_adjacency_matrix(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_adjacency_matrix(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let filters_map = match params.get("filters").and_then(Value::as_object) {
Some(m) => m,
None => return json!({ "buckets": [] }),
@@ -7121,7 +8087,10 @@ fn run_adjacency_matrix(params: &Value, sub_aggs: Option<&Value>, docs: &[Value]
// For each doc, determine which filters it matches.
let mut filter_matches: Vec<Vec<bool>> = Vec::with_capacity(docs.len());
for doc in docs {
- let matches: Vec<bool> = filters.iter().map(|(_, q)| doc_matches_filter(doc, q)).collect();
+ let matches: Vec<bool> = filters
+ .iter()
+ .map(|(_, q)| doc_matches_filter(doc, q))
+ .collect();
filter_matches.push(matches);
}
@@ -7143,7 +8112,9 @@ fn run_adjacency_matrix(params: &Value, sub_aggs: Option<&Value>, docs: &[Value]
if let Some(sub) = sub_aggs {
let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
if let (Some(obj), Value::Object(sub_obj)) = (bucket.as_object_mut(), sub_result) {
- for (k, v) in sub_obj { obj.insert(k, v); }
+ for (k, v) in sub_obj {
+ obj.insert(k, v);
+ }
}
}
buckets.push(bucket);
@@ -7169,7 +8140,9 @@ fn run_adjacency_matrix(params: &Value, sub_aggs: Option<&Value>, docs: &[Value]
if let Some(sub) = sub_aggs {
let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
if let (Some(obj), Value::Object(sub_obj)) = (bucket.as_object_mut(), sub_result) {
- for (k, v) in sub_obj { obj.insert(k, v); }
+ for (k, v) in sub_obj {
+ obj.insert(k, v);
+ }
}
}
buckets.push(bucket);
@@ -7232,31 +8205,6 @@ fn encode_geohash(lat: f64, lon: f64, precision: usize) -> String {
String::from_utf8(hash).unwrap_or_default()
}
-/// Extract lat/lon from a field value (same formats as GeoDistance).
-fn extract_lat_lon(doc: &Value, field: &str) -> Option<(f64, f64)> {
- let val = get_nested_field(doc, field);
- match val {
- Value::Object(obj) => {
- let lat = obj.get("lat").and_then(|v| v.as_f64())?;
- let lon = obj.get("lon").and_then(|v| v.as_f64())?;
- Some((lat, lon))
- }
- Value::Array(arr) if arr.len() == 2 => {
- let lon = arr[0].as_f64()?;
- let lat = arr[1].as_f64()?;
- Some((lat, lon))
- }
- Value::String(s) => {
- let parts: Vec<&str> = s.splitn(2, ',').collect();
- if parts.len() != 2 { return None; }
- let lat = parts[0].trim().parse::<f64>().ok()?;
- let lon = parts[1].trim().parse::<f64>().ok()?;
- Some((lat, lon))
- }
- _ => None,
- }
-}
-
/// Extract a (lat, lon) pair from an ES geo_point value.
/// Accepts:
/// - {"lat": 1.0, "lon": 2.0}
@@ -7279,7 +8227,11 @@ fn parse_geo_point(v: &Value) -> Option<(f64, f64)> {
Value::String(s) => {
let t = s.trim();
if let Some(rest) = t.strip_prefix("POINT") {
- let rest = rest.trim_start().trim_start_matches('(').trim_end_matches(')').trim();
+ let rest = rest
+ .trim_start()
+ .trim_start_matches('(')
+ .trim_end_matches(')')
+ .trim();
let mut parts = rest.split_whitespace();
let lon: f64 = parts.next()?.parse().ok()?;
let lat: f64 = parts.next()?.parse().ok()?;
@@ -7313,8 +8265,7 @@ fn geotile_xy(lat: f64, lon: f64, zoom: u32) -> (i64, i64) {
let lat_rad = lat.to_radians();
let n = 2f64.powi(zoom as i32);
let x = ((lon + 180.0) / 360.0 * n).floor() as i64;
- let y = ((1.0 - (lat_rad.tan() + 1.0 / lat_rad.cos()).ln() / std::f64::consts::PI) / 2.0
- * n)
+ let y = ((1.0 - (lat_rad.tan() + 1.0 / lat_rad.cos()).ln() / std::f64::consts::PI) / 2.0 * n)
.floor() as i64;
(x, y)
}
@@ -7385,7 +8336,12 @@ fn geo_unit_scale(u: &str) -> f64 {
/// ES `geo_distance` aggregation — bucket each doc by its distance
/// (meters by default) from `origin`, against a list of `ranges`.
-fn run_geo_distance(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_geo_distance(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": []}),
@@ -7405,62 +8361,80 @@ fn run_geo_distance(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], al
let unit = params.get("unit").and_then(Value::as_str).unwrap_or("m");
let scale = geo_unit_scale(unit);
- let buckets: Vec<Value> = ranges.iter().map(|r| {
- let from = r.get("from").and_then(Value::as_f64);
- let to = r.get("to").and_then(Value::as_f64);
- let filtered: Vec<Value> = docs.iter().filter(|doc| {
- let loc = match doc.get(field) {
- Some(v) => v,
- None => return false,
- };
- let pt = match parse_geo_point(loc) {
- Some(p) => p,
- None => return false,
- };
- let dist_m = haversine_distance(origin, pt);
- let dist = dist_m / scale;
- match (from, to) {
- (Some(f), Some(t)) => dist >= f && dist < t,
- (Some(f), None) => dist >= f,
- (None, Some(t)) => dist < t,
- (None, None) => true,
- }
- }).cloned().collect();
- let key = r.get("key").and_then(Value::as_str).map(String::from).unwrap_or_else(|| match (from, to) {
- (Some(f), Some(t)) => format!("{}-{}", format_range_val(f), format_range_val(t)),
- (Some(f), None) => format!("{}-*", format_range_val(f)),
- (None, Some(t)) => format!("*-{}", format_range_val(t)),
- (None, None) => "*-*".to_string(),
- });
- let mut bucket = serde_json::Map::new();
- bucket.insert("key".to_string(), Value::String(key));
- if let Some(f) = from { bucket.insert("from".to_string(), json!(f)); }
- if let Some(t) = to { bucket.insert("to".to_string(), json!(t)); }
- bucket.insert("doc_count".to_string(), json!(filtered.len() as u64));
- if let Some(sub) = sub_aggs {
- let sr = run_aggs_with_all(sub, &filtered, all_docs);
- if let Value::Object(so) = sr {
- for (k, v) in so { bucket.insert(k, v); }
+ let buckets: Vec<Value> = ranges
+ .iter()
+ .map(|r| {
+ let from = r.get("from").and_then(Value::as_f64);
+ let to = r.get("to").and_then(Value::as_f64);
+ let filtered: Vec<Value> = docs
+ .iter()
+ .filter(|doc| {
+ let loc = match doc.get(field) {
+ Some(v) => v,
+ None => return false,
+ };
+ let pt = match parse_geo_point(loc) {
+ Some(p) => p,
+ None => return false,
+ };
+ let dist_m = haversine_distance(origin, pt);
+ let dist = dist_m / scale;
+ match (from, to) {
+ (Some(f), Some(t)) => dist >= f && dist < t,
+ (Some(f), None) => dist >= f,
+ (None, Some(t)) => dist < t,
+ (None, None) => true,
+ }
+ })
+ .cloned()
+ .collect();
+ let key = r
+ .get("key")
+ .and_then(Value::as_str)
+ .map(String::from)
+ .unwrap_or_else(|| match (from, to) {
+ (Some(f), Some(t)) => {
+ format!("{}-{}", format_range_val(f), format_range_val(t))
+ }
+ (Some(f), None) => format!("{}-*", format_range_val(f)),
+ (None, Some(t)) => format!("*-{}", format_range_val(t)),
+ (None, None) => "*-*".to_string(),
+ });
+ let mut bucket = serde_json::Map::new();
+ bucket.insert("key".to_string(), Value::String(key));
+ if let Some(f) = from {
+ bucket.insert("from".to_string(), json!(f));
}
- }
- Value::Object(bucket)
- }).collect();
+ if let Some(t) = to {
+ bucket.insert("to".to_string(), json!(t));
+ }
+ bucket.insert("doc_count".to_string(), json!(filtered.len() as u64));
+ if let Some(sub) = sub_aggs {
+ let sr = run_aggs_with_all(sub, &filtered, all_docs);
+ if let Value::Object(so) = sr {
+ for (k, v) in so {
+ bucket.insert(k, v);
+ }
+ }
+ }
+ Value::Object(bucket)
+ })
+ .collect();
json!({"buckets": buckets})
}
-fn run_geotile_grid(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_geotile_grid(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": []}),
};
- let precision = params
- .get("precision")
- .and_then(Value::as_u64)
- .unwrap_or(7) as u32;
- let size = params
- .get("size")
- .and_then(Value::as_u64)
- .unwrap_or(10_000) as usize;
+ let precision = params.get("precision").and_then(Value::as_u64).unwrap_or(7) as u32;
+ let size = params.get("size").and_then(Value::as_u64).unwrap_or(10_000) as usize;
// Optional bounds filter: {top_left: [lon, lat], bottom_right: [lon, lat]}.
// ES filter semantics: a doc's tile counts if that tile
// intersects the bounds rectangle (not if the doc's point itself
@@ -7489,9 +8463,11 @@ fn run_geotile_grid(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], al
for (lat, lon) in pts {
let (x, y) = geotile_xy(lat, lon, precision);
if let Some((xlo, xhi, ylo, yhi)) = tile_bounds {
- if x < xlo || x > xhi || y < ylo || y > yhi { continue; }
+ if x < xlo || x > xhi || y < ylo || y > yhi {
+ continue;
+ }
}
- let n = 2i64.pow(precision as u32);
+ let n = 2i64.pow(precision);
let max = n - 1;
let key = format!("{}/{}/{}", precision, x.clamp(0, max), y.clamp(0, max));
// Bucket cap: skip new keys past the limit; existing keys still grow.
@@ -7505,8 +8481,9 @@ fn run_geotile_grid(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], al
// entries to the front, then O(N log N) on just the prefix. Beats a full
// O(M log M) sort when M (total buckets) >> N (size); for 10M buckets /
// size=10 this is roughly a 20× win.
- let cmp = |a: &(String, Vec<usize>), b: &(String, Vec<usize>)|
- b.1.len().cmp(&a.1.len()).then_with(|| a.0.cmp(&b.0));
+ let cmp = |a: &(String, Vec<usize>), b: &(String, Vec<usize>)| {
+ b.1.len().cmp(&a.1.len()).then_with(|| a.0.cmp(&b.0))
+ };
let mut sorted: Vec<(String, Vec<usize>)> = bucket_map.into_iter().collect();
let n = size.min(sorted.len());
if n > 0 && n < sorted.len() {
@@ -7524,7 +8501,9 @@ fn run_geotile_grid(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], al
if let Some(sub) = sub_aggs {
let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
if let (Some(obj), Value::Object(sub_obj)) = (bucket.as_object_mut(), sub_result) {
- for (k, v) in sub_obj { obj.insert(k, v); }
+ for (k, v) in sub_obj {
+ obj.insert(k, v);
+ }
}
}
bucket
@@ -7567,10 +8546,7 @@ fn collect_geo_points(doc: &Value, field: &str) -> Vec<(f64, f64)> {
}
match v {
Value::Array(arr) => {
- if arr.len() == 2
- && arr[0].as_f64().is_some()
- && arr[1].as_f64().is_some()
- {
+ if arr.len() == 2 && arr[0].as_f64().is_some() && arr[1].as_f64().is_some() {
if let Some(p) = parse_geo_point(v) {
out.push(p);
}
@@ -7585,19 +8561,18 @@ fn collect_geo_points(doc: &Value, field: &str) -> Vec<(f64, f64)> {
out
}
-fn run_geohash_grid(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_geohash_grid(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": []}),
};
- let precision = params
- .get("precision")
- .and_then(Value::as_u64)
- .unwrap_or(5) as usize;
- let size = params
- .get("size")
- .and_then(Value::as_u64)
- .unwrap_or(10_000) as usize;
+ let precision = params.get("precision").and_then(Value::as_u64).unwrap_or(5) as usize;
+ let size = params.get("size").and_then(Value::as_u64).unwrap_or(10_000) as usize;
// Optional bounds filter.
let bounds = params.get("bounds").and_then(Value::as_object);
@@ -7622,7 +8597,9 @@ fn run_geohash_grid(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], al
let bucket_cap = max_buckets();
for (i, doc) in docs.iter().enumerate() {
for (lat, lon) in collect_geo_points(doc, field) {
- if !in_bounds(lat, lon) { continue; }
+ if !in_bounds(lat, lon) {
+ continue;
+ }
let hash = encode_geohash(lat, lon, precision);
if bucket_map.contains_key(&hash) || bucket_map.len() < bucket_cap {
bucket_map.entry(hash).or_default().push(i);
@@ -7635,8 +8612,9 @@ fn run_geohash_grid(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], al
// value in descending order (matching the numeric-id tie-break
// used by Lucene's geohash aggregator). Partial-sort optimization
// (see geotile path above for the rationale).
- let cmp = |a: &(String, Vec<usize>), b: &(String, Vec<usize>)|
- b.1.len().cmp(&a.1.len()).then_with(|| b.0.cmp(&a.0));
+ let cmp = |a: &(String, Vec<usize>), b: &(String, Vec<usize>)| {
+ b.1.len().cmp(&a.1.len()).then_with(|| b.0.cmp(&a.0))
+ };
let mut sorted: Vec<(String, Vec<usize>)> = bucket_map.into_iter().collect();
let n = size.min(sorted.len());
if n > 0 && n < sorted.len() {
@@ -7654,7 +8632,9 @@ fn run_geohash_grid(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], al
if let Some(sub) = sub_aggs {
let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
if let (Some(obj), Value::Object(sub_obj)) = (bucket.as_object_mut(), sub_result) {
- for (k, v) in sub_obj { obj.insert(k, v); }
+ for (k, v) in sub_obj {
+ obj.insert(k, v);
+ }
}
}
bucket
@@ -7666,15 +8646,17 @@ fn run_geohash_grid(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], al
// ── Multi-terms aggregation ───────────────────────────────────────────────────
-fn run_multi_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_multi_terms(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let terms_defs = match params.get("terms").and_then(Value::as_array) {
Some(t) => t,
None => return json!({"buckets": []}),
};
- let size = params
- .get("size")
- .and_then(Value::as_u64)
- .unwrap_or(10) as usize;
+ let size = params.get("size").and_then(Value::as_u64).unwrap_or(10) as usize;
// Extract field names in order.
let fields: Vec<&str> = terms_defs
@@ -7693,7 +8675,12 @@ fn run_multi_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all
// For multi-terms, take the first value for each field (like ES default).
let key: Vec<String> = fields
.iter()
- .map(|f| extract_field_values(doc, f).into_iter().next().unwrap_or_default())
+ .map(|f| {
+ extract_field_values(doc, f)
+ .into_iter()
+ .next()
+ .unwrap_or_default()
+ })
.collect();
if bucket_map.contains_key(&key) || bucket_map.len() < bucket_cap {
bucket_map.entry(key).or_default().push(i);
@@ -7703,8 +8690,9 @@ fn run_multi_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all
// Sort by doc_count descending. Partial-sort: O(M) select_nth_unstable_by
// + O(N log N) prefix sort, vs O(M log M) for full sort. Wins big for
// wide multi-terms aggs over high-cardinality compound keys.
- let cmp = |a: &(Vec<String>, Vec<usize>), b: &(Vec<String>, Vec<usize>)|
- b.1.len().cmp(&a.1.len()).then_with(|| a.0.cmp(&b.0));
+ let cmp = |a: &(Vec<String>, Vec<usize>), b: &(Vec<String>, Vec<usize>)| {
+ b.1.len().cmp(&a.1.len()).then_with(|| a.0.cmp(&b.0))
+ };
let mut sorted: Vec<(Vec<String>, Vec<usize>)> = bucket_map.into_iter().collect();
let n = size.min(sorted.len());
if n > 0 && n < sorted.len() {
@@ -7728,7 +8716,9 @@ fn run_multi_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all
if let Some(sub) = sub_aggs {
let sub_result = run_aggs_with_all(sub, &bucket_docs, all_docs);
if let (Some(obj), Value::Object(sub_obj)) = (bucket.as_object_mut(), sub_result) {
- for (k, v) in sub_obj { obj.insert(k, v); }
+ for (k, v) in sub_obj {
+ obj.insert(k, v);
+ }
}
}
bucket
@@ -7751,33 +8741,35 @@ fn run_multi_terms(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all
// canonical form here.
const AUTO_DATE_INTERVALS: &[(&str, i64)] = &[
("1ms", 1),
- ("1s", 1_000),
+ ("1s", 1_000),
("10s", 10_000),
("30s", 30_000),
- ("1m", 60_000),
- ("5m", 300_000),
+ ("1m", 60_000),
+ ("5m", 300_000),
("10m", 600_000),
("15m", 900_000),
("30m", 1_800_000),
- ("1h", 3_600_000),
- ("3h", 10_800_000),
+ ("1h", 3_600_000),
+ ("3h", 10_800_000),
("12h", 43_200_000),
- ("1d", 86_400_000),
- ("7d", 604_800_000),
+ ("1d", 86_400_000),
+ ("7d", 604_800_000),
("30d", 2_592_000_000),
("90d", 7_776_000_000),
- ("1y", 31_536_000_000),
+ ("1y", 31_536_000_000),
];
-fn run_auto_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_auto_date_histogram(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"buckets": [], "interval": "1d"}),
};
- let target_buckets = params
- .get("buckets")
- .and_then(Value::as_u64)
- .unwrap_or(10) as usize;
+ let target_buckets = params.get("buckets").and_then(Value::as_u64).unwrap_or(10) as usize;
// Collect all timestamps.
let timestamps: Vec<i64> = docs
@@ -7791,7 +8783,7 @@ fn run_auto_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Val
let min_ts = *timestamps.iter().min().unwrap();
let max_ts = *timestamps.iter().max().unwrap();
- let span_ms = (max_ts - min_ts).max(1);
+ let _span_ms = (max_ts - min_ts).max(1);
// Choose the interval that produces the number of buckets closest to
// target_buckets. Use a calendar-aware bucket count that matches ES's
@@ -7824,17 +8816,15 @@ fn run_auto_date_histogram(params: &Value, sub_aggs: Option<&Value>, docs: &[Val
// pin to (e.g. expected first bucket = 2020-03-01, the data min).
// Compute the required `offset` (ms) relative to the current grid
// and pass it through to date_histogram.
- let offset_ms: i64 = if interval_label == "7d"
- || interval_label == "30d"
- || interval_label == "90d"
- {
- let day_ms = 86_400_000i64;
- let min_day = min_ts.div_euclid(day_ms) * day_ms;
- let grid_floor = min_day.div_euclid(interval_ms) * interval_ms;
- min_day - grid_floor
- } else {
- 0
- };
+ let offset_ms: i64 =
+ if interval_label == "7d" || interval_label == "30d" || interval_label == "90d" {
+ let day_ms = 86_400_000i64;
+ let min_day = min_ts.div_euclid(day_ms) * day_ms;
+ let grid_floor = min_day.div_euclid(interval_ms) * interval_ms;
+ min_day - grid_floor
+ } else {
+ 0
+ };
let offset_str = if offset_ms == 0 {
String::new()
} else {
@@ -7883,7 +8873,12 @@ pub enum FastAggResult {
/// Returns `Unsupported` for bucket aggs that have sub-aggs (which would
/// require re-filtering docs — those still go through the JSON scan path),
/// or for any agg type that cannot be served from a single column.
-pub fn run_agg_fast(agg_type: &str, params: &Value, dv: &DocValues, has_sub_aggs: bool) -> FastAggResult {
+pub fn run_agg_fast(
+ agg_type: &str,
+ params: &Value,
+ dv: &DocValues,
+ has_sub_aggs: bool,
+) -> FastAggResult {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return FastAggResult::Unsupported,
@@ -7896,7 +8891,10 @@ pub fn run_agg_fast(agg_type: &str, params: &Value, dv: &DocValues, has_sub_aggs
Some(c) => c,
None => return FastAggResult::Unsupported,
};
- let (sum, count) = col.iter().flatten().fold((0.0f64, 0u64), |(s, c), &v| (s + v, c + 1));
+ let (sum, count) = col
+ .iter()
+ .flatten()
+ .fold((0.0f64, 0u64), |(s, c), &v| (s + v, c + 1));
let value: Value = if count == 0 {
Value::Null
} else {
@@ -7944,19 +8942,24 @@ pub fn run_agg_fast(agg_type: &str, params: &Value, dv: &DocValues, has_sub_aggs
"value_count" => {
// Count non-null values in the keyword column (or numeric fallback).
- let count = dv.keyword
+ let count = dv
+ .keyword
.get(field)
.map(|col| col.iter().filter(|v| v.is_some()).count())
- .or_else(|| dv.numeric.get(field).map(|col| col.iter().filter(|v| v.is_some()).count()))
+ .or_else(|| {
+ dv.numeric
+ .get(field)
+ .map(|col| col.iter().filter(|v| v.is_some()).count())
+ })
.unwrap_or(0) as u64;
FastAggResult::Value(json!({ "value": count }))
}
"cardinality" => {
// Distinct count from the bounded-delta maintained keyword_set.
- let count = dv
- .with_keyword_field(field, |c| c.keyword_set.get(field).map(|s| s.len()).unwrap_or(0))
- as u64;
+ let count = dv.with_keyword_field(field, |c| {
+ c.keyword_set.get(field).map(|s| s.len()).unwrap_or(0)
+ }) as u64;
FastAggResult::Value(json!({ "value": count }))
}
@@ -7972,16 +8975,24 @@ pub fn run_agg_fast(agg_type: &str, params: &Value, dv: &DocValues, has_sub_aggs
for &v in col.iter().flatten() {
count += 1;
sum += v;
- if v < min { min = v; }
- if v > max { max = v; }
+ if v < min {
+ min = v;
+ }
+ if v > max {
+ max = v;
+ }
}
let avg = if count > 0 { sum / count as f64 } else { 0.0 };
let (min_out, max_out) = if count == 0 {
(Value::Null, Value::Null)
} else {
(
- serde_json::Number::from_f64(min).map(Value::Number).unwrap_or(Value::Null),
- serde_json::Number::from_f64(max).map(Value::Number).unwrap_or(Value::Null),
+ serde_json::Number::from_f64(min)
+ .map(Value::Number)
+ .unwrap_or(Value::Null),
+ serde_json::Number::from_f64(max)
+ .map(Value::Number)
+ .unwrap_or(Value::Null),
)
};
FastAggResult::Value(json!({
@@ -8015,13 +9026,15 @@ pub fn run_agg_fast(agg_type: &str, params: &Value, dv: &DocValues, has_sub_aggs
*counts.entry(opt.clone()).or_insert(0) += 1;
}
- let order_by_count_asc = params.get("order")
+ let order_by_count_asc = params
+ .get("order")
.and_then(Value::as_object)
.and_then(|o| o.get("_count"))
.and_then(Value::as_str)
.map(|s| s == "asc")
.unwrap_or(false);
- let order_by_key = params.get("order")
+ let order_by_key = params
+ .get("order")
.and_then(Value::as_object)
.and_then(|o| o.get("_key"))
.and_then(Value::as_str)
@@ -8092,10 +9105,10 @@ pub fn run_agg_fast_filtered(
// Returns owned Strings to avoid lifetime complexity with the closure.
let iter_keyword = |col: &[Option<String>]| -> Vec<String> {
match doc_indices {
- None => col.iter().filter_map(|v| v.as_ref().map(|s| s.clone())).collect(),
+ None => col.iter().filter_map(|v| v.clone()).collect(),
Some(idxs) => idxs
.iter()
- .filter_map(|&i| col.get(i).and_then(|v| v.as_ref().map(|s| s.clone())))
+ .filter_map(|&i| col.get(i).and_then(|v| v.clone()))
.collect(),
}
};
@@ -8168,7 +9181,8 @@ pub fn run_agg_fast_filtered(
"cardinality" => {
// For filtered sets, recount distinct values from the filtered slice.
let count = if let Some(col) = dv.keyword.get(field) {
- let distinct: std::collections::HashSet<String> = iter_keyword(col).into_iter().collect();
+ let distinct: std::collections::HashSet<String> =
+ iter_keyword(col).into_iter().collect();
distinct.len()
} else {
0
@@ -8189,16 +9203,24 @@ pub fn run_agg_fast_filtered(
for v in vals {
count += 1;
sum += v;
- if v < min { min = v; }
- if v > max { max = v; }
+ if v < min {
+ min = v;
+ }
+ if v > max {
+ max = v;
+ }
}
let avg = if count > 0 { sum / count as f64 } else { 0.0 };
let (min_out, max_out) = if count == 0 {
(Value::Null, Value::Null)
} else {
(
- serde_json::Number::from_f64(min).map(Value::Number).unwrap_or(Value::Null),
- serde_json::Number::from_f64(max).map(Value::Number).unwrap_or(Value::Null),
+ serde_json::Number::from_f64(min)
+ .map(Value::Number)
+ .unwrap_or(Value::Null),
+ serde_json::Number::from_f64(max)
+ .map(Value::Number)
+ .unwrap_or(Value::Null),
)
};
FastAggResult::Value(json!({
@@ -8229,13 +9251,15 @@ pub fn run_agg_fast_filtered(
*counts.entry(v.to_string()).or_insert(0) += 1;
}
- let order_by_count_asc = params.get("order")
+ let order_by_count_asc = params
+ .get("order")
.and_then(Value::as_object)
.and_then(|o| o.get("_count"))
.and_then(Value::as_str)
.map(|s| s == "asc")
.unwrap_or(false);
- let order_by_key = params.get("order")
+ let order_by_key = params
+ .get("order")
.and_then(Value::as_object)
.and_then(|o| o.get("_key"))
.and_then(Value::as_str)
@@ -8338,10 +9362,7 @@ fn run_extended_stats<'d>(params: &Value, _docs: &'d [Value], cache: &mut FieldC
};
let std_deviation = variance_pop.max(0.0).sqrt();
let std_deviation_samp = variance_samp.max(0.0).sqrt();
- let sigma = params
- .get("sigma")
- .and_then(Value::as_f64)
- .unwrap_or(2.0);
+ let sigma = params.get("sigma").and_then(Value::as_f64).unwrap_or(2.0);
json!({
"count": count,
@@ -8466,7 +9487,11 @@ fn run_geo_centroid(params: &Value, docs: &[Value]) -> Value {
}
/// `median_absolute_deviation` — median of absolute deviations from the median.
-fn run_median_absolute_deviation<'d>(params: &Value, _docs: &'d [Value], cache: &mut FieldCache<'d>) -> Value {
+fn run_median_absolute_deviation<'d>(
+ params: &Value,
+ _docs: &'d [Value],
+ cache: &mut FieldCache<'d>,
+) -> Value {
let field = match params.get("field").and_then(Value::as_str) {
Some(f) => f,
None => return json!({"value": Value::Null}),
@@ -8484,7 +9509,7 @@ fn run_median_absolute_deviation<'d>(params: &Value, _docs: &'d [Value], cache:
values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let n = values.len();
- let median = if n % 2 == 0 {
+ let median = if n.is_multiple_of(2) {
(values[n / 2 - 1] + values[n / 2]) / 2.0
} else {
values[n / 2]
@@ -8493,7 +9518,7 @@ fn run_median_absolute_deviation<'d>(params: &Value, _docs: &'d [Value], cache:
let mut deviations: Vec<f64> = values.iter().map(|x| (x - median).abs()).collect();
deviations.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let m = deviations.len();
- let mad = if m % 2 == 0 {
+ let mad = if m.is_multiple_of(2) {
(deviations[m / 2 - 1] + deviations[m / 2]) / 2.0
} else {
deviations[m / 2]
@@ -8551,8 +9576,12 @@ fn run_nested(path: &str, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[
if matches!(item, Value::Object(_)) {
let mut wrapped = wrap_synth_for_path(path, item);
if let Some(obj) = wrapped.as_object_mut() {
- if let Some(id) = parent_id.clone() { obj.insert("_id".into(), id); }
- if let Some(ix) = parent_index.clone() { obj.insert("_index".into(), ix); }
+ if let Some(id) = parent_id.clone() {
+ obj.insert("_id".into(), id);
+ }
+ if let Some(ix) = parent_index.clone() {
+ obj.insert("_index".into(), ix);
+ }
obj.insert(
"_nested".into(),
json!({"field": path, "offset": offset as u64}),
@@ -8591,23 +9620,6 @@ fn wrap_synth_for_path(path: &str, item: &Value) -> Value {
cur
}
-fn run_nested_passthrough(sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
- let doc_count = docs.len() as u64;
- let mut result = serde_json::Map::new();
- result.insert("doc_count".to_string(), json!(doc_count));
-
- if let Some(sub) = sub_aggs {
- let sub_result = run_aggs_with_all(sub, docs, all_docs);
- if let Value::Object(sub_obj) = sub_result {
- for (k, v) in sub_obj {
- result.insert(k, v);
- }
- }
- }
-
- Value::Object(result)
-}
-
fn run_reverse_nested(sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
// Deduplicate by parent _id so each parent doc is counted once. Docs
// without _id fall back to identity (treated as already-parent).
@@ -8625,7 +9637,11 @@ fn run_reverse_nested(sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Valu
let parent = doc
.get("_id")
.and_then(Value::as_str)
- .and_then(|id| all_docs.iter().find(|d| d.get("_id").and_then(Value::as_str) == Some(id)))
+ .and_then(|id| {
+ all_docs
+ .iter()
+ .find(|d| d.get("_id").and_then(Value::as_str) == Some(id))
+ })
.cloned()
.unwrap_or_else(|| doc.clone());
unique.push(parent);
@@ -8666,13 +9682,24 @@ fn run_global(sub_aggs: Option<&Value>, all_docs: &[Value]) -> Value {
// ── date_range aggregation ────────────────────────────────────────────────────
-fn run_date_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
- let field = params.get("field").and_then(Value::as_str).unwrap_or("@timestamp");
+fn run_date_range(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
+ let field = params
+ .get("field")
+ .and_then(Value::as_str)
+ .unwrap_or("@timestamp");
let ranges = match params.get("ranges").and_then(Value::as_array) {
Some(r) => r,
None => return json!({"buckets": []}),
};
- let keyed = params.get("keyed").and_then(Value::as_bool).unwrap_or(false);
+ let keyed = params
+ .get("keyed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
let user_fmt = params.get("format").and_then(Value::as_str);
let missing_ms: Option<i64> = params.get("missing").and_then(parse_date_ms);
let tz_param = params.get("time_zone").and_then(Value::as_str);
@@ -8680,11 +9707,23 @@ fn run_date_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_
// Detect fractional-second precision from the longest input. If any
// from/to uses 9-digit (nanos) precision, render with `SSSSSSSSS`; else
// default to ES's 3-digit (millis).
- let nanos_digits: usize = ranges.iter().flat_map(|r| {
- let from_s = r.get("from").and_then(Value::as_str).map(detect_fractional_digits).unwrap_or(0);
- let to_s = r.get("to").and_then(Value::as_str).map(detect_fractional_digits).unwrap_or(0);
- [from_s, to_s]
- }).max().unwrap_or(3);
+ let nanos_digits: usize = ranges
+ .iter()
+ .flat_map(|r| {
+ let from_s = r
+ .get("from")
+ .and_then(Value::as_str)
+ .map(detect_fractional_digits)
+ .unwrap_or(0);
+ let to_s = r
+ .get("to")
+ .and_then(Value::as_str)
+ .map(detect_fractional_digits)
+ .unwrap_or(0);
+ [from_s, to_s]
+ })
+ .max()
+ .unwrap_or(3);
let nanos_digits = if nanos_digits >= 9 { 9 } else { 3 };
let render = |ms: i64| -> String {
@@ -8704,11 +9743,13 @@ fn run_date_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_
let offset = tz_param.and_then(|tz| fixed_offset_for_tz_at(tz, ms));
let pat = java_to_strftime(fmt);
let out = if let Some(off) = offset {
- let dt_utc = chrono::DateTime::<chrono::Utc>::from_timestamp_millis(ms).unwrap_or_default();
+ let dt_utc =
+ chrono::DateTime::<chrono::Utc>::from_timestamp_millis(ms).unwrap_or_default();
let dt_local = dt_utc.with_timezone(&off);
dt_local.format(&pat).to_string()
} else {
- let dt_utc = chrono::DateTime::<chrono::Utc>::from_timestamp_millis(ms).unwrap_or_default();
+ let dt_utc =
+ chrono::DateTime::<chrono::Utc>::from_timestamp_millis(ms).unwrap_or_default();
dt_utc.format(&pat).to_string()
};
// Java's `ZZZZZ` collapses to `Z` for UTC; chrono's `%:z`
@@ -8746,7 +9787,9 @@ fn run_date_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_
_ => None,
};
}
- if let Some(ms) = parse_date_ms(val) { return Some(ms); }
+ if let Some(ms) = parse_date_ms(val) {
+ return Some(ms);
+ }
let s = val.as_str()?;
let fmt = user_fmt?;
let pat = java_to_strftime(fmt);
@@ -8754,7 +9797,9 @@ fn run_date_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_
return Some(dt.and_utc().timestamp_millis());
}
if let Ok(d) = chrono::NaiveDate::parse_from_str(s, &pat) {
- return d.and_hms_opt(0, 0, 0).map(|dt| dt.and_utc().timestamp_millis());
+ return d
+ .and_hms_opt(0, 0, 0)
+ .map(|dt| dt.and_utc().timestamp_millis());
}
None
};
@@ -8765,8 +9810,8 @@ fn run_date_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_
// `from`/`to` can be ISO-8601 strings or raw numbers. Both cases
// convert through parse_date_ms so bucket comparisons work on
// millis.
- let from_ms = range_def.get("from").and_then(|v| parse_with_fmt(v));
- let to_ms = range_def.get("to").and_then(|v| parse_with_fmt(v));
+ let from_ms = range_def.get("from").and_then(&parse_with_fmt);
+ let to_ms = range_def.get("to").and_then(&parse_with_fmt);
let from_str_in = range_def.get("from").and_then(Value::as_str);
let to_str_in = range_def.get("to").and_then(Value::as_str);
@@ -8784,24 +9829,31 @@ fn run_date_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_
_ => "*".to_string(),
};
let default_key = format!("{}-{}", from_key, to_key);
- let key = range_def.get("key").and_then(Value::as_str).map(String::from)
+ let key = range_def
+ .get("key")
+ .and_then(Value::as_str)
+ .map(String::from)
.unwrap_or(default_key);
- let filtered: Vec<Value> = docs.iter().filter(|doc| {
- let v = doc.get(field).or_else(|| {
- doc.get("_source").and_then(|s| s.get(field))
- });
- // Doc values honor the field-declared format too: a doc
- // stored as `{date: 1000}` under `format: epoch_second`
- // means 1000 seconds = 1 000 000 ms.
- let doc_ms = match v.and_then(|vv| parse_with_fmt(vv)).or(missing_ms) {
- Some(m) => m,
- None => return false,
- };
- let passes_from = from_ms.map(|f| doc_ms >= f).unwrap_or(true);
- let passes_to = to_ms.map(|t| doc_ms < t).unwrap_or(true);
- passes_from && passes_to
- }).cloned().collect();
+ let filtered: Vec<Value> = docs
+ .iter()
+ .filter(|doc| {
+ let v = doc
+ .get(field)
+ .or_else(|| doc.get("_source").and_then(|s| s.get(field)));
+ // Doc values honor the field-declared format too: a doc
+ // stored as `{date: 1000}` under `format: epoch_second`
+ // means 1000 seconds = 1 000 000 ms.
+ let doc_ms = match v.and_then(&parse_with_fmt).or(missing_ms) {
+ Some(m) => m,
+ None => return false,
+ };
+ let passes_from = from_ms.map(|f| doc_ms >= f).unwrap_or(true);
+ let passes_to = to_ms.map(|t| doc_ms < t).unwrap_or(true);
+ passes_from && passes_to
+ })
+ .cloned()
+ .collect();
let mut bucket = serde_json::Map::new();
if !keyed {
@@ -8819,7 +9871,9 @@ fn run_date_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_
if let Some(sub) = sub_aggs {
let sr = run_aggs_with_all(sub, &filtered, all_docs);
if let Value::Object(so) = sr {
- for (k, v) in so { bucket.insert(k, v); }
+ for (k, v) in so {
+ bucket.insert(k, v);
+ }
}
}
if keyed {
@@ -8859,10 +9913,15 @@ fn run_percentile_ranks(params: &Value, docs: &[Value]) -> Value {
// Default `keyed` for percentile_ranks is true (matches percentiles).
let keyed = params.get("keyed").and_then(Value::as_bool).unwrap_or(true);
- let mut nums: Vec<f64> = docs.iter().filter_map(|doc| {
- let v = doc.get(field).or_else(|| doc.get("_source").and_then(|s| s.get(field)))?;
- v.as_f64()
- }).collect();
+ let mut nums: Vec<f64> = docs
+ .iter()
+ .filter_map(|doc| {
+ let v = doc
+ .get(field)
+ .or_else(|| doc.get("_source").and_then(|s| s.get(field)))?;
+ v.as_f64()
+ })
+ .collect();
nums.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let total = nums.len() as f64;
@@ -8923,7 +9982,8 @@ fn run_percentile_ranks(params: &Value, docs: &[Value]) -> Value {
fn run_scripted_metric(params: &Value, docs: &[Value]) -> Value {
let src_of = |key: &str| -> Option<String> {
params.get(key).and_then(|v| {
- v.as_str().map(String::from)
+ v.as_str()
+ .map(String::from)
.or_else(|| v.get("source").and_then(Value::as_str).map(String::from))
})
};
@@ -8936,7 +9996,7 @@ fn run_scripted_metric(params: &Value, docs: &[Value]) -> Value {
// Pre-parse the scripts once. Each returns the parsed AST, or an
// error we surface as a null metric.
let parse = |s: &Option<String>| -> Option<Vec<PlStmt>> {
- s.as_deref().map(|code| parse_script(code)).transpose().ok().flatten()
+ s.as_deref().map(parse_script).transpose().ok().flatten()
};
let init_ast = parse(&init_src);
let map_ast = parse(&map_src);
@@ -8949,13 +10009,17 @@ fn run_scripted_metric(params: &Value, docs: &[Value]) -> Value {
let mut state_obj = Value::Object(serde_json::Map::new());
if let Some(ast) = &init_ast {
let mut ctx = ScriptCtx::new(state_obj.clone(), script_params.clone(), None);
- if exec_stmts(ast, &mut ctx).is_err() { return json!({"value": Value::Null}); }
+ if exec_stmts(ast, &mut ctx).is_err() {
+ return json!({"value": Value::Null});
+ }
state_obj = ctx.state;
}
if let Some(ast) = &map_ast {
for doc in docs {
let mut ctx = ScriptCtx::new(state_obj.clone(), script_params.clone(), Some(doc));
- if exec_stmts(ast, &mut ctx).is_err() { return json!({"value": Value::Null}); }
+ if exec_stmts(ast, &mut ctx).is_err() {
+ return json!({"value": Value::Null});
+ }
state_obj = ctx.state;
}
}
@@ -9036,7 +10100,13 @@ struct ScriptCtx<'a> {
}
impl<'a> ScriptCtx<'a> {
fn new(state: Value, params: Value, doc: Option<&'a Value>) -> Self {
- Self { state, params, doc, states: None, locals: std::collections::HashMap::new() }
+ Self {
+ state,
+ params,
+ doc,
+ states: None,
+ locals: std::collections::HashMap::new(),
+ }
}
}
@@ -9047,21 +10117,32 @@ fn lex_script(src: &str) -> Result<Vec<PlTok>, String> {
let mut out = Vec::new();
while i < bytes.len() {
let c = bytes[i] as char;
- if c.is_whitespace() { i += 1; continue; }
+ if c.is_whitespace() {
+ i += 1;
+ continue;
+ }
// Line comments //..., block comments /* ... */
- if c == '/' && i + 1 < bytes.len() && bytes[i+1] as char == '/' {
- while i < bytes.len() && bytes[i] as char != '\n' { i += 1; }
+ if c == '/' && i + 1 < bytes.len() && bytes[i + 1] as char == '/' {
+ while i < bytes.len() && bytes[i] as char != '\n' {
+ i += 1;
+ }
continue;
}
- if c == '/' && i + 1 < bytes.len() && bytes[i+1] as char == '*' {
+ if c == '/' && i + 1 < bytes.len() && bytes[i + 1] as char == '*' {
i += 2;
- while i + 1 < bytes.len() && !(bytes[i] as char == '*' && bytes[i+1] as char == '/') { i += 1; }
+ while i + 1 < bytes.len() && !(bytes[i] as char == '*' && bytes[i + 1] as char == '/') {
+ i += 1;
+ }
i += 2.min(bytes.len() - i);
continue;
}
- if c.is_ascii_digit() || (c == '.' && i + 1 < bytes.len() && (bytes[i+1] as char).is_ascii_digit()) {
+ if c.is_ascii_digit()
+ || (c == '.' && i + 1 < bytes.len() && (bytes[i + 1] as char).is_ascii_digit())
+ {
let start = i;
- while i < bytes.len() && ((bytes[i] as char).is_ascii_digit() || bytes[i] as char == '.') {
+ while i < bytes.len()
+ && ((bytes[i] as char).is_ascii_digit() || bytes[i] as char == '.')
+ {
i += 1;
}
let s = &src[start..i];
@@ -9073,15 +10154,21 @@ fn lex_script(src: &str) -> Result<Vec<PlTok>, String> {
let quote = c;
i += 1;
let start = i;
- while i < bytes.len() && bytes[i] as char != quote { i += 1; }
+ while i < bytes.len() && bytes[i] as char != quote {
+ i += 1;
+ }
let s = src[start..i].to_string();
- if i < bytes.len() { i += 1; }
+ if i < bytes.len() {
+ i += 1;
+ }
out.push(PlTok::StrLit(s));
continue;
}
if c.is_ascii_alphabetic() || c == '_' {
let start = i;
- while i < bytes.len() && ((bytes[i] as char).is_ascii_alphanumeric() || bytes[i] as char == '_') {
+ while i < bytes.len()
+ && ((bytes[i] as char).is_ascii_alphanumeric() || bytes[i] as char == '_')
+ {
i += 1;
}
out.push(PlTok::Ident(src[start..i].to_string()));
@@ -9089,11 +10176,18 @@ fn lex_script(src: &str) -> Result<Vec<PlTok>, String> {
}
// Two-char operators
if i + 1 < bytes.len() {
- let two = &src[i..i+2];
+ let two = &src[i..i + 2];
let op: Option<&'static str> = match two {
- ">=" => Some(">="), "<=" => Some("<="), "==" => Some("=="), "!=" => Some("!="),
- "&&" => Some("&&"), "||" => Some("||"), "+=" => Some("+="), "-=" => Some("-="),
- "*=" => Some("*="), "/=" => Some("/="),
+ ">=" => Some(">="),
+ "<=" => Some("<="),
+ "==" => Some("=="),
+ "!=" => Some("!="),
+ "&&" => Some("&&"),
+ "||" => Some("||"),
+ "+=" => Some("+="),
+ "-=" => Some("-="),
+ "*=" => Some("*="),
+ "/=" => Some("/="),
_ => None,
};
if let Some(o) = op {
@@ -9103,26 +10197,86 @@ fn lex_script(src: &str) -> Result<Vec<PlTok>, String> {
}
}
match c {
- '+' => { out.push(PlTok::Op("+")); i += 1; }
- '-' => { out.push(PlTok::Op("-")); i += 1; }
- '*' => { out.push(PlTok::Op("*")); i += 1; }
- '/' => { out.push(PlTok::Op("/")); i += 1; }
- '%' => { out.push(PlTok::Op("%")); i += 1; }
- '<' => { out.push(PlTok::Op("<")); i += 1; }
- '>' => { out.push(PlTok::Op(">")); i += 1; }
- '=' => { out.push(PlTok::Op("=")); i += 1; }
- '!' => { out.push(PlTok::Op("!")); i += 1; }
- '(' => { out.push(PlTok::LParen); i += 1; }
- ')' => { out.push(PlTok::RParen); i += 1; }
- '{' => { out.push(PlTok::LBrace); i += 1; }
- '}' => { out.push(PlTok::RBrace); i += 1; }
- '[' => { out.push(PlTok::Op("[")); i += 1; }
- ']' => { out.push(PlTok::Op("]")); i += 1; }
- ';' => { out.push(PlTok::Semi); i += 1; }
- ',' => { out.push(PlTok::Comma); i += 1; }
- '.' => { out.push(PlTok::Dot); i += 1; }
- '?' => { out.push(PlTok::Question); i += 1; }
- ':' => { out.push(PlTok::Colon); i += 1; }
+ '+' => {
+ out.push(PlTok::Op("+"));
+ i += 1;
+ }
+ '-' => {
+ out.push(PlTok::Op("-"));
+ i += 1;
+ }
+ '*' => {
+ out.push(PlTok::Op("*"));
+ i += 1;
+ }
+ '/' => {
+ out.push(PlTok::Op("/"));
+ i += 1;
+ }
+ '%' => {
+ out.push(PlTok::Op("%"));
+ i += 1;
+ }
+ '<' => {
+ out.push(PlTok::Op("<"));
+ i += 1;
+ }
+ '>' => {
+ out.push(PlTok::Op(">"));
+ i += 1;
+ }
+ '=' => {
+ out.push(PlTok::Op("="));
+ i += 1;
+ }
+ '!' => {
+ out.push(PlTok::Op("!"));
+ i += 1;
+ }
+ '(' => {
+ out.push(PlTok::LParen);
+ i += 1;
+ }
+ ')' => {
+ out.push(PlTok::RParen);
+ i += 1;
+ }
+ '{' => {
+ out.push(PlTok::LBrace);
+ i += 1;
+ }
+ '}' => {
+ out.push(PlTok::RBrace);
+ i += 1;
+ }
+ '[' => {
+ out.push(PlTok::Op("["));
+ i += 1;
+ }
+ ']' => {
+ out.push(PlTok::Op("]"));
+ i += 1;
+ }
+ ';' => {
+ out.push(PlTok::Semi);
+ i += 1;
+ }
+ ',' => {
+ out.push(PlTok::Comma);
+ i += 1;
+ }
+ '.' => {
+ out.push(PlTok::Dot);
+ i += 1;
+ }
+ '?' => {
+ out.push(PlTok::Question);
+ i += 1;
+ }
+ ':' => {
+ out.push(PlTok::Colon);
+ i += 1;
+ }
_ => return Err(format!("unexpected char `{}`", c)),
}
}
@@ -9134,18 +10288,32 @@ struct PlParser {
pos: usize,
}
impl PlParser {
- fn peek(&self) -> Option<&PlTok> { self.toks.get(self.pos) }
- fn bump(&mut self) -> Option<PlTok> { let t = self.toks.get(self.pos).cloned(); self.pos += 1; t }
+ fn peek(&self) -> Option<&PlTok> {
+ self.toks.get(self.pos)
+ }
+ fn bump(&mut self) -> Option<PlTok> {
+ let t = self.toks.get(self.pos).cloned();
+ self.pos += 1;
+ t
+ }
fn eat_semi(&mut self) {
- while matches!(self.peek(), Some(PlTok::Semi)) { self.pos += 1; }
+ while matches!(self.peek(), Some(PlTok::Semi)) {
+ self.pos += 1;
+ }
}
fn parse_stmts(&mut self, terminator_brace: bool) -> Result<Vec<PlStmt>, String> {
let mut out = Vec::new();
while let Some(t) = self.peek() {
- if terminator_brace && matches!(t, PlTok::RBrace) { break; }
+ if terminator_brace && matches!(t, PlTok::RBrace) {
+ break;
+ }
self.eat_semi();
- if self.peek().is_none() { break; }
- if terminator_brace && matches!(self.peek(), Some(PlTok::RBrace)) { break; }
+ if self.peek().is_none() {
+ break;
+ }
+ if terminator_brace && matches!(self.peek(), Some(PlTok::RBrace)) {
+ break;
+ }
out.push(self.parse_stmt()?);
self.eat_semi();
}
@@ -9170,9 +10338,23 @@ impl PlParser {
// map to ordinary JSON values.
if matches!(
kw.as_str(),
- "long" | "int" | "double" | "float" | "def" | "var" | "boolean" | "String"
- | "Set" | "Map" | "List" | "HashSet" | "TreeSet" | "HashMap" | "TreeMap"
- | "ArrayList" | "Collection"
+ "long"
+ | "int"
+ | "double"
+ | "float"
+ | "def"
+ | "var"
+ | "boolean"
+ | "String"
+ | "Set"
+ | "Map"
+ | "List"
+ | "HashSet"
+ | "TreeSet"
+ | "HashMap"
+ | "TreeMap"
+ | "ArrayList"
+ | "Collection"
) {
// Lookahead: if the next token after the type keyword
// is not an identifier (i.e. this is a bare expression
@@ -9194,15 +10376,21 @@ impl PlParser {
// statement as a no-op expression so parsing proceeds.
if kw == "throw" {
self.bump(); // `throw`
- // Consume `new`, type name, and `(args)`.
+ // Consume `new`, type name, and `(args)`.
if let Some(PlTok::Ident(n)) = self.peek().cloned() {
- if n == "new" { self.bump(); }
+ if n == "new" {
+ self.bump();
+ }
}
// Type name (optionally `Foo.Bar` path).
- if let Some(PlTok::Ident(_)) = self.peek() { self.bump(); }
+ if let Some(PlTok::Ident(_)) = self.peek() {
+ self.bump();
+ }
while matches!(self.peek(), Some(PlTok::Dot)) {
self.bump();
- if let Some(PlTok::Ident(_)) = self.peek() { self.bump(); }
+ if let Some(PlTok::Ident(_)) = self.peek() {
+ self.bump();
+ }
}
if matches!(self.peek(), Some(PlTok::LParen)) {
self.bump();
@@ -9245,7 +10433,10 @@ impl PlParser {
// Accept either `for (t in ...)` or `for (def t : ...)`.
// Skip an optional type keyword before the loop variable.
if let Some(PlTok::Ident(kw)) = self.peek().cloned() {
- if matches!(kw.as_str(), "def" | "var" | "long" | "int" | "double" | "float") {
+ if matches!(
+ kw.as_str(),
+ "def" | "var" | "long" | "int" | "double" | "float"
+ ) {
self.bump();
}
}
@@ -9273,13 +10464,19 @@ impl PlParser {
}
fn parse_if(&mut self) -> Result<PlStmt, String> {
self.bump();
- if !matches!(self.bump(), Some(PlTok::LParen)) { return Err("expected `(` after if".into()); }
+ if !matches!(self.bump(), Some(PlTok::LParen)) {
+ return Err("expected `(` after if".into());
+ }
let cond = self.parse_expr()?;
- if !matches!(self.bump(), Some(PlTok::RParen)) { return Err("expected `)` after if-cond".into()); }
+ if !matches!(self.bump(), Some(PlTok::RParen)) {
+ return Err("expected `)` after if-cond".into());
+ }
let then_body = if matches!(self.peek(), Some(PlTok::LBrace)) {
self.bump();
let b = self.parse_stmts(true)?;
- if !matches!(self.bump(), Some(PlTok::RBrace)) { return Err("expected `}` after if-then".into()); }
+ if !matches!(self.bump(), Some(PlTok::RBrace)) {
+ return Err("expected `}` after if-then".into());
+ }
b
} else {
vec![self.parse_stmt()?]
@@ -9289,31 +10486,42 @@ impl PlParser {
if matches!(self.peek(), Some(PlTok::LBrace)) {
self.bump();
let b = self.parse_stmts(true)?;
- if !matches!(self.bump(), Some(PlTok::RBrace)) { return Err("expected `}` after if-else".into()); }
+ if !matches!(self.bump(), Some(PlTok::RBrace)) {
+ return Err("expected `}` after if-else".into());
+ }
Some(b)
} else {
Some(vec![self.parse_stmt()?])
}
- } else { None };
+ } else {
+ None
+ };
Ok(PlStmt::If(cond, then_body, else_body))
}
- fn parse_expr(&mut self) -> Result<PlExpr, String> { self.parse_ternary() }
+ fn parse_expr(&mut self) -> Result<PlExpr, String> {
+ self.parse_ternary()
+ }
fn parse_ternary(&mut self) -> Result<PlExpr, String> {
let cond = self.parse_binary(1)?;
if matches!(self.peek(), Some(PlTok::Question)) {
self.bump();
let t = self.parse_ternary()?;
- if !matches!(self.bump(), Some(PlTok::Colon)) { return Err("expected `:` in ternary".into()); }
+ if !matches!(self.bump(), Some(PlTok::Colon)) {
+ return Err("expected `:` in ternary".into());
+ }
let f = self.parse_ternary()?;
Ok(PlExpr::Ternary(Box::new(cond), Box::new(t), Box::new(f)))
- } else { Ok(cond) }
+ } else {
+ Ok(cond)
+ }
}
fn parse_binary(&mut self, min_prec: u8) -> Result<PlExpr, String> {
let mut lhs = self.parse_unary()?;
- loop {
- let Some(op) = self.peek_binop() else { break };
+ while let Some(op) = self.peek_binop() {
let prec = binop_prec(op);
- if prec < min_prec { break; }
+ if prec < min_prec {
+ break;
+ }
self.bump();
let rhs = self.parse_binary(prec + 1)?;
lhs = PlExpr::Binary(Box::new(lhs), op, Box::new(rhs));
@@ -9323,11 +10531,13 @@ impl PlParser {
fn peek_binop(&self) -> Option<&'static str> {
if let Some(PlTok::Op(o)) = self.peek() {
match *o {
- "||" | "&&" | "==" | "!=" | "<" | ">" | "<=" | ">=" |
- "+" | "-" | "*" | "/" | "%" => Some(*o),
+ "||" | "&&" | "==" | "!=" | "<" | ">" | "<=" | ">=" | "+" | "-" | "*" | "/"
+ | "%" => Some(*o),
_ => None,
}
- } else { None }
+ } else {
+ None
+ }
}
fn parse_unary(&mut self) -> Result<PlExpr, String> {
if let Some(PlTok::Op("-")) = self.peek() {
@@ -9416,7 +10626,9 @@ impl PlParser {
// Optional dotted type path (`Foo.Bar`).
while matches!(self.peek(), Some(PlTok::Dot)) {
self.bump();
- if let Some(PlTok::Ident(_)) = self.peek() { self.bump(); }
+ if let Some(PlTok::Ident(_)) = self.peek() {
+ self.bump();
+ }
}
// Optional generic args `<…>` — tokenised
// as `<` then idents; skip until `>`.
@@ -9456,7 +10668,9 @@ impl PlParser {
}
Some(PlTok::LParen) => {
let e = self.parse_expr()?;
- if !matches!(self.bump(), Some(PlTok::RParen)) { return Err("expected `)`".into()); }
+ if !matches!(self.bump(), Some(PlTok::RParen)) {
+ return Err("expected `)`".into());
+ }
Ok(e)
}
Some(PlTok::Op("[")) => {
@@ -9480,7 +10694,8 @@ impl PlParser {
}
fn binop_prec(op: &str) -> u8 {
match op {
- "||" => 1, "&&" => 2,
+ "||" => 1,
+ "&&" => 2,
"==" | "!=" => 3,
"<" | ">" | "<=" | ">=" => 4,
"+" | "-" => 5,
@@ -9512,7 +10727,10 @@ fn exec_stmt(stmt: &PlStmt, ctx: &mut ScriptCtx) -> Result<Option<Value>, String
ctx.locals.insert(name.clone(), v);
Ok(None)
}
- PlStmt::Expr(e) => { eval_expr(e, ctx)?; Ok(None) }
+ PlStmt::Expr(e) => {
+ eval_expr(e, ctx)?;
+ Ok(None)
+ }
PlStmt::Assign(lhs, rhs) => {
let v = eval_expr(rhs, ctx)?;
assign_to(lhs, v, ctx)?;
@@ -9576,13 +10794,16 @@ fn eval_expr(e: &PlExpr, ctx: &mut ScriptCtx) -> Result<Value, String> {
PlExpr::Num(n) => Ok(json!(n)),
PlExpr::Str(s) => Ok(Value::String(s.clone())),
PlExpr::Ident(n) => {
- if let Some(v) = ctx.locals.get(n) { return Ok(v.clone()); }
+ if let Some(v) = ctx.locals.get(n) {
+ return Ok(v.clone());
+ }
match n.as_str() {
"state" => Ok(ctx.state.clone()),
"params" => Ok(ctx.params.clone()),
"states" => Ok(ctx.states.clone().map(Value::Array).unwrap_or(Value::Null)),
"doc" => Ok(ctx.doc.cloned().unwrap_or(Value::Null)),
- "_score" => Ok(ctx.doc
+ "_score" => Ok(ctx
+ .doc
.and_then(|d| d.get("_score").cloned())
.unwrap_or(json!(1.0))),
_ => Ok(Value::Null),
@@ -9590,7 +10811,9 @@ fn eval_expr(e: &PlExpr, ctx: &mut ScriptCtx) -> Result<Value, String> {
}
PlExpr::ArrayLit(items) => {
let mut out = Vec::with_capacity(items.len());
- for it in items { out.push(eval_expr(it, ctx)?); }
+ for it in items {
+ out.push(eval_expr(it, ctx)?);
+ }
Ok(Value::Array(out))
}
PlExpr::Member(obj_expr, name) => {
@@ -9602,7 +10825,9 @@ fn eval_expr(e: &PlExpr, ctx: &mut ScriptCtx) -> Result<Value, String> {
// below collapses it. We surface the array form here.
// Supports dotted paths (`doc['k8s.pod.uid']` →
// walk `k8s.pod.uid` or look up literal-dotted key).
- let Some(doc) = ctx.doc else { return Ok(Value::Null) };
+ let Some(doc) = ctx.doc else {
+ return Ok(Value::Null);
+ };
let src = doc.get("_source").unwrap_or(doc);
let raw = walk_dotted_or_literal(src, name);
let as_array = match raw {
@@ -9622,12 +10847,16 @@ fn eval_expr(e: &PlExpr, ctx: &mut ScriptCtx) -> Result<Value, String> {
}
// `.size()` — property form for arrays.
if name == "size" || name == "length" {
- if let Value::Array(a) = base { return Ok(json!(a.len() as u64)); }
+ if let Value::Array(a) = base {
+ return Ok(json!(a.len() as u64));
+ }
return Ok(json!(0));
}
// Generic property lookup on objects.
if let Value::Object(ref obj) = base {
- if let Some(v) = obj.get(name) { return Ok(v.clone()); }
+ if let Some(v) = obj.get(name) {
+ return Ok(v.clone());
+ }
}
Ok(Value::Null)
}
@@ -9667,7 +10896,9 @@ fn eval_expr(e: &PlExpr, ctx: &mut ScriptCtx) -> Result<Value, String> {
return Ok(Value::Bool(true));
}
if name == "size" && args.is_empty() {
- if let Value::Array(a) = recv_val { return Ok(json!(a.len() as u64)); }
+ if let Value::Array(a) = recv_val {
+ return Ok(json!(a.len() as u64));
+ }
return Ok(json!(0));
}
if name == "contains" && args.len() == 1 {
@@ -9705,8 +10936,11 @@ fn eval_expr(e: &PlExpr, ctx: &mut ScriptCtx) -> Result<Value, String> {
let a = value_as_f64(&lv).unwrap_or(0.0);
let b = value_as_f64(&rv).unwrap_or(0.0);
let res = match *op {
- "<" => a < b, "<=" => a <= b,
- ">" => a > b, ">=" => a >= b, _ => false,
+ "<" => a < b,
+ "<=" => a <= b,
+ ">" => a > b,
+ ">=" => a >= b,
+ _ => false,
};
Ok(Value::Bool(res))
}
@@ -9718,17 +10952,23 @@ fn eval_expr(e: &PlExpr, ctx: &mut ScriptCtx) -> Result<Value, String> {
let lhs_str = matches!(lv, Value::String(_));
let rhs_str = matches!(rv, Value::String(_));
if lhs_str || rhs_str {
- let fmt = |v: &Value| -> String { match v {
- Value::String(s) => s.clone(),
- Value::Number(n) => {
- if n.is_i64() { n.as_i64().unwrap().to_string() }
- else if n.is_u64() { n.as_u64().unwrap().to_string() }
- else { n.to_string() }
+ let fmt = |v: &Value| -> String {
+ match v {
+ Value::String(s) => s.clone(),
+ Value::Number(n) => {
+ if n.is_i64() {
+ n.as_i64().unwrap().to_string()
+ } else if n.is_u64() {
+ n.as_u64().unwrap().to_string()
+ } else {
+ n.to_string()
+ }
+ }
+ Value::Bool(b) => b.to_string(),
+ Value::Null => "null".to_string(),
+ other => other.to_string(),
}
- Value::Bool(b) => b.to_string(),
- Value::Null => "null".to_string(),
- other => other.to_string(),
- }};
+ };
Ok(Value::String(format!("{}{}", fmt(&lv), fmt(&rv))))
} else {
num_binop(&lv, op, &rv)
@@ -9751,7 +10991,11 @@ fn eval_expr(e: &PlExpr, ctx: &mut ScriptCtx) -> Result<Value, String> {
}
PlExpr::Ternary(c, t, f) => {
let cv = eval_expr(c, ctx)?;
- if is_truthy(&cv) { eval_expr(t, ctx) } else { eval_expr(f, ctx) }
+ if is_truthy(&cv) {
+ eval_expr(t, ctx)
+ } else {
+ eval_expr(f, ctx)
+ }
}
}
}
@@ -9762,13 +11006,19 @@ fn eval_expr(e: &PlExpr, ctx: &mut ScriptCtx) -> Result<Value, String> {
/// accepts both shapes interchangeably.
fn walk_dotted_or_literal(v: &Value, key: &str) -> Value {
if let Value::Object(obj) = v {
- if let Some(direct) = obj.get(key) { return direct.clone(); }
- if !key.contains('.') { return Value::Null; }
+ if let Some(direct) = obj.get(key) {
+ return direct.clone();
+ }
+ if !key.contains('.') {
+ return Value::Null;
+ }
let mut cur: &Value = v;
for seg in key.split('.') {
match cur {
Value::Object(o) => {
- let Some(next) = o.get(seg) else { return Value::Null };
+ let Some(next) = o.get(seg) else {
+ return Value::Null;
+ };
cur = next;
}
_ => return Value::Null,
@@ -9791,14 +11041,14 @@ fn json_value_eq(a: &Value, b: &Value) -> bool {
match (a, b) {
(Value::String(x), Value::String(y)) => x == y,
(Value::String(x), other) | (other, Value::String(x)) => {
- if let Some(nx) = x.parse::<f64>().ok() {
- if let Some(ny) = value_as_f64(other) { return (nx - ny).abs() < 1e-12; }
+ if let Ok(nx) = x.parse::<f64>() {
+ if let Some(ny) = value_as_f64(other) {
+ return (nx - ny).abs() < 1e-12;
+ }
}
other.as_str().map(|s| s == x).unwrap_or(false)
}
- (Value::Number(_), Value::Number(_)) => {
- value_as_f64(a) == value_as_f64(b)
- }
+ (Value::Number(_), Value::Number(_)) => value_as_f64(a) == value_as_f64(b),
_ => a == b,
}
}
@@ -9806,9 +11056,23 @@ fn num_binop(a: &Value, op: &str, b: &Value) -> Result<Value, String> {
let x = value_as_f64(a).unwrap_or(0.0);
let y = value_as_f64(b).unwrap_or(0.0);
let r = match op {
- "+" => x + y, "-" => x - y, "*" => x * y,
- "/" => if y != 0.0 { x / y } else { 0.0 },
- "%" => if y != 0.0 { x % y } else { 0.0 },
+ "+" => x + y,
+ "-" => x - y,
+ "*" => x * y,
+ "/" => {
+ if y != 0.0 {
+ x / y
+ } else {
+ 0.0
+ }
+ }
+ "%" => {
+ if y != 0.0 {
+ x % y
+ } else {
+ 0.0
+ }
+ }
_ => return Err(format!("bad num op `{}`", op)),
};
// Preserve integer shape when the result is representable as an int
@@ -9822,7 +11086,9 @@ fn num_binop(a: &Value, op: &str, b: &Value) -> Result<Value, String> {
}
fn value_looks_int(v: &Value) -> bool {
match v {
- Value::Number(n) => n.is_i64() || n.is_u64() || n.as_f64().map(|f| f.fract() == 0.0).unwrap_or(false),
+ Value::Number(n) => {
+ n.is_i64() || n.is_u64() || n.as_f64().map(|f| f.fract() == 0.0).unwrap_or(false)
+ }
_ => false,
}
}
@@ -9842,7 +11108,10 @@ fn assign_to(target: &PlExpr, value: Value, ctx: &mut ScriptCtx) -> Result<(), S
ctx.locals.insert(root, value);
Ok(())
} else {
- let mut holder = ctx.locals.remove(&root).unwrap_or(Value::Object(serde_json::Map::new()));
+ let mut holder = ctx
+ .locals
+ .remove(&root)
+ .unwrap_or(Value::Object(serde_json::Map::new()));
write_chain(&mut holder, &segments, value)?;
ctx.locals.insert(root, holder);
Ok(())
@@ -9874,7 +11143,9 @@ fn write_chain(holder: &mut Value, segs: &[String], value: Value) -> Result<(),
if segs.len() == 1 {
obj.insert(first.clone(), value);
} else {
- let entry = obj.entry(first.clone()).or_insert_with(|| Value::Object(serde_json::Map::new()));
+ let entry = obj
+ .entry(first.clone())
+ .or_insert_with(|| Value::Object(serde_json::Map::new()));
write_chain(entry, &segs[1..], value)?;
}
Ok(())
@@ -9890,7 +11161,12 @@ fn format_ipv6_u128(u: u128) -> String {
std::net::Ipv6Addr::from(bytes).to_string()
}
-fn run_ip_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_docs: &[Value]) -> Value {
+fn run_ip_range(
+ params: &Value,
+ sub_aggs: Option<&Value>,
+ docs: &[Value],
+ all_docs: &[Value],
+) -> Value {
let field = params.get("field").and_then(Value::as_str).unwrap_or("ip");
let ranges = match params.get("ranges").and_then(Value::as_array) {
Some(r) => r,
@@ -9920,11 +11196,17 @@ fn run_ip_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_do
// uses v6-mapped form, so shift v4 prefixes by +96 and require the
// candidate addr to be v4-mapped (upper 80 bits = 0, next 16 = 0xFFFF).
let is_v4_cidr = !net.contains(':');
- let Some(net_u) = ip_as_u128(net) else { return false };
- let Some(addr_u) = ip_as_u128(addr_str) else { return false };
+ let Some(net_u) = ip_as_u128(net) else {
+ return false;
+ };
+ let Some(addr_u) = ip_as_u128(addr_str) else {
+ return false;
+ };
if is_v4_cidr {
// Candidate must be v4-mapped for a v4 CIDR to apply.
- if (addr_u >> 32) != 0xFFFF_u128 { return false; }
+ if (addr_u >> 32) != 0xFFFF_u128 {
+ return false;
+ }
let effective = prefix + 96;
let shift = 128u32.saturating_sub(effective);
let mask = if shift >= 128 { 0 } else { (!0u128) << shift };
@@ -9948,29 +11230,38 @@ fn run_ip_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_do
} else {
format!("{}-{}", from_str.unwrap_or("*"), to_str.unwrap_or("*"))
};
- let key = range_def.get("key").and_then(Value::as_str).map(String::from)
+ let key = range_def
+ .get("key")
+ .and_then(Value::as_str)
+ .map(String::from)
.unwrap_or(default_key);
let from_u = from_str.and_then(ip_as_u128);
let to_u = to_str.and_then(ip_as_u128);
- let filtered: Vec<Value> = docs.iter().filter(|doc| {
- let val = doc.get(field).or_else(|| doc.get("_source").and_then(|s| s.get(field)));
- let ip_str = match val.and_then(Value::as_str) {
- Some(s) => s,
- None => return false,
- };
- if let Some(m) = mask {
- cidr_matches(ip_str, m)
- } else {
- let ip_u = match ip_as_u128(ip_str) {
- Some(v) => v,
+ let filtered: Vec<Value> = docs
+ .iter()
+ .filter(|doc| {
+ let val = doc
+ .get(field)
+ .or_else(|| doc.get("_source").and_then(|s| s.get(field)));
+ let ip_str = match val.and_then(Value::as_str) {
+ Some(s) => s,
None => return false,
};
- let passes_from = from_u.map(|f| ip_u >= f).unwrap_or(true);
- let passes_to = to_u.map(|t| ip_u < t).unwrap_or(true);
- passes_from && passes_to
- }
- }).cloned().collect();
+ if let Some(m) = mask {
+ cidr_matches(ip_str, m)
+ } else {
+ let ip_u = match ip_as_u128(ip_str) {
+ Some(v) => v,
+ None => return false,
+ };
+ let passes_from = from_u.map(|f| ip_u >= f).unwrap_or(true);
+ let passes_to = to_u.map(|t| ip_u < t).unwrap_or(true);
+ passes_from && passes_to
+ }
+ })
+ .cloned()
+ .collect();
let mut bucket = serde_json::Map::new();
bucket.insert("key".to_string(), json!(key));
@@ -9990,7 +11281,7 @@ fn run_ip_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_do
let shift = 128u32.saturating_sub(effective_prefix);
let mask_bits = if shift >= 128 { 0 } else { (!0u128) << shift };
let start = net_u & mask_bits;
- let end = start.checked_add(1u128 << shift.min(127)).unwrap_or(u128::MAX);
+ let end = start.saturating_add(1u128 << shift.min(127));
let fmt = |u: u128| -> String {
if is_v4 {
// Render as IPv4 (strip the v6-mapped prefix).
@@ -10017,7 +11308,9 @@ fn run_ip_range(params: &Value, sub_aggs: Option<&Value>, docs: &[Value], all_do
if let Some(sub) = sub_aggs {
let sr = run_aggs_with_all(sub, &filtered, all_docs);
if let Value::Object(so) = sr {
- for (k, v) in so { bucket.insert(k, v); }
+ for (k, v) in so {
+ bucket.insert(k, v);
+ }
}
}
buckets.push(Value::Object(bucket));
diff --git a/engine/crates/xerj-engine/src/audit.rs b/engine/crates/xerj-engine/src/audit.rs
index d5060e566..de97849d9 100644
--- a/engine/crates/xerj-engine/src/audit.rs
+++ b/engine/crates/xerj-engine/src/audit.rs
@@ -69,14 +69,7 @@ impl AuditLog {
/// Append an entry. Computes the hash over (prev_hash || canonical
/// JSON of the entry without its `hash` field).
- pub fn append(
- &self,
- op: &str,
- subject: &str,
- resource: &str,
- outcome: &str,
- note: &str,
- ) {
+ pub fn append(&self, op: &str, subject: &str, resource: &str, outcome: &str, note: &str) {
let seq = self.next_seq.fetch_add(1, Ordering::Relaxed);
let at_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
@@ -84,7 +77,9 @@ impl AuditLog {
.unwrap_or(0);
let prev_hash = {
let buf = self.buf.read();
- buf.back().map(|e| e.hash.clone()).unwrap_or_else(|| "0".repeat(64))
+ buf.back()
+ .map(|e| e.hash.clone())
+ .unwrap_or_else(|| "0".repeat(64))
};
let mut entry = AuditEntry {
seq,
@@ -165,8 +160,8 @@ mod tests {
fn tampering_detected() {
let log = AuditLog::new(8);
log.append("search", "alice", "x", "ok", "");
- log.append("delete", "bob", "x", "ok", "");
- log.append("admin", "root", "y", "ok", "");
+ log.append("delete", "bob", "x", "ok", "");
+ log.append("admin", "root", "y", "ok", "");
// Tamper with entry 2's `subject`.
{
let mut buf = log.buf.write();
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index d9e146512..f09a2d9d5 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -91,11 +91,7 @@ struct ParsedAction {
/// All `index` and `create` operations are batched per-index and submitted
/// via `index_batch_turbo()` for 10–20x higher throughput than one-at-a-time.
/// `update` and `delete` operations are still executed individually.
-pub async fn process_bulk(
- engine: &Engine,
- default_index: Option<&str>,
- body: &str,
-) -> BulkResult {
+pub async fn process_bulk(engine: &Engine, default_index: Option<&str>, body: &str) -> BulkResult {
process_bulk_with_opts(engine, default_index, body, BulkOpts::default()).await
}
@@ -159,14 +155,24 @@ pub async fn process_bulk_with_opts(
fn action_line_is_delete(line: &str) -> bool {
let b = line.as_bytes();
let mut p = 0usize;
- while p < b.len() && (b[p] == b' ' || b[p] == b'\t') { p += 1; }
- if p >= b.len() || b[p] != b'{' { return false; }
+ while p < b.len() && (b[p] == b' ' || b[p] == b'\t') {
+ p += 1;
+ }
+ if p >= b.len() || b[p] != b'{' {
+ return false;
+ }
p += 1;
- while p < b.len() && (b[p] == b' ' || b[p] == b'\t') { p += 1; }
- if p >= b.len() || b[p] != b'"' { return false; }
+ while p < b.len() && (b[p] == b' ' || b[p] == b'\t') {
+ p += 1;
+ }
+ if p >= b.len() || b[p] != b'"' {
+ return false;
+ }
p += 1;
let start = p;
- while p < b.len() && b[p] != b'"' { p += 1; }
+ while p < b.len() && b[p] != b'"' {
+ p += 1;
+ }
p < b.len() && &b[start..p] == b"delete"
}
let mut pairs: Vec<(usize, &str, Option<&str>, usize)> = Vec::new();
@@ -191,167 +197,239 @@ pub async fn process_bulk_with_opts(
enum ParseOutcome {
Ok(ParsedAction),
Err(BulkItemResult, usize),
- Skip(usize /* extra item_index to pop, because we optimistically consumed a line */),
}
let t_parse = std::time::Instant::now();
// Dedicated ingest pool: keeps bulk-parse bursts off the global rayon
// pool that the search path fans out on (see `crate::ingest_pool`).
- let parse_results: Vec<ParseOutcome> = crate::ingest_pool().install(|| pairs
- .par_iter()
- .map(|(_line_idx, action_line, doc_line, item_index)| {
- // M5.12 — FAST-PATH manual parse of the action line.
- //
- // Pre-M5.12 the action line hit `serde_json::from_str::<Value>`
- // which allocates a Map<String,Value> for every one of 5000
- // action lines per batch = ~5 us × 5000 = 25 ms of pure
- // allocation per bulk batch.
- //
- // We handle the three common shapes inline and fall back to
- // full serde_json parse only on a cache miss:
- // {"index":{}}
- // {"index":{"_id":"..."}}
- // {"index":{"_index":"..."}}
- // {"index":{"_index":"...","_id":"..."}}
- // (same for "create", "update", "delete")
- let bytes = action_line.as_bytes();
- // Any require_alias mention forces full parse (bool extraction is
- // not handled in the fast path); uncommon enough to not matter.
- let has_require_alias = bytes.windows(14).any(|w| w == b"require_alias\"");
- let has_pipeline = bytes.windows(9).any(|w| w == b"pipeline\"");
- let has_if_seq_no = bytes.windows(11).any(|w| w == b"\"if_seq_no\"");
- let has_routing = bytes.windows(9).any(|w| w == b"\"routing\"");
- let has_source = bytes.windows(9).any(|w| w == b"\"_source\"");
- let has_dynamic_templates = bytes.windows(19).any(|w| w == b"\"dynamic_templates\"");
- let mut require_alias_per_item: Option<bool> = None;
- let mut pipeline_per_item: Option<String> = None;
- let mut if_seq_no_per_item: Option<u64> = None;
- let mut if_primary_term_per_item: Option<u64> = None;
- let mut routing_per_item: Option<String> = None;
- let mut source_req_per_item: Option<Value> = None;
- let mut dynamic_templates_per_item: Option<std::collections::BTreeMap<String, String>> = None;
- let (action_type, target_index, doc_id) = 'fast: {
- if has_require_alias || has_pipeline || has_if_seq_no || has_routing || has_source || has_dynamic_templates {
- break 'fast (None, None, None);
- }
- // Parse state: skip whitespace, expect `{`, "action": "index" etc.
- let mut p = 0usize;
- while p < bytes.len() && (bytes[p] == b' ' || bytes[p] == b'\t') { p += 1; }
- if p >= bytes.len() || bytes[p] != b'{' { break 'fast (None, None, None); }
- p += 1;
- while p < bytes.len() && (bytes[p] == b' ' || bytes[p] == b'\t') { p += 1; }
- if p >= bytes.len() || bytes[p] != b'"' { break 'fast (None, None, None); }
- p += 1;
- // Find closing quote of action name.
- let key_start = p;
- while p < bytes.len() && bytes[p] != b'"' { p += 1; }
- if p >= bytes.len() { break 'fast (None, None, None); }
- let key = &bytes[key_start..p];
- let action_ty: &'static str = match key {
- b"index" => "index",
- b"create" => "create",
- b"update" => "update",
- b"delete" => "delete",
- _ => break 'fast (None, None, None),
- };
- p += 1;
- while p < bytes.len() && (bytes[p] == b' ' || bytes[p] == b':') { p += 1; }
- if p >= bytes.len() || bytes[p] != b'{' { break 'fast (None, None, None); }
- // Extract optional _index and _id inside the inner object.
- // Both are bare-quoted strings with no embedded quotes in
- // the canonical bulk format produced by ES clients. The
- // inner object itself may contain nested objects (e.g.
- // `dynamic_templates:{...}`), so we track brace depth —
- // a naive "first `}`" scan would truncate the inner slice
- // at the first nested `}` and hide fields like `op_type`.
- let inner_end = {
- let mut j = p + 1;
- let mut depth: i32 = 1;
- let mut in_str = false;
- while j < bytes.len() {
- let c = bytes[j];
- if in_str {
- if c == b'\\' { j += 2; continue; }
- if c == b'"' { in_str = false; }
- } else {
- match c {
- b'"' => in_str = true,
- b'{' => depth += 1,
- b'}' => {
- depth -= 1;
- if depth == 0 { break; }
+ let parse_results: Vec<ParseOutcome> = crate::ingest_pool().install(|| {
+ pairs
+ .par_iter()
+ .map(|(_line_idx, action_line, doc_line, item_index)| {
+ // M5.12 — FAST-PATH manual parse of the action line.
+ //
+ // Pre-M5.12 the action line hit `serde_json::from_str::<Value>`
+ // which allocates a Map<String,Value> for every one of 5000
+ // action lines per batch = ~5 us × 5000 = 25 ms of pure
+ // allocation per bulk batch.
+ //
+ // We handle the three common shapes inline and fall back to
+ // full serde_json parse only on a cache miss:
+ // {"index":{}}
+ // {"index":{"_id":"..."}}
+ // {"index":{"_index":"..."}}
+ // {"index":{"_index":"...","_id":"..."}}
+ // (same for "create", "update", "delete")
+ let bytes = action_line.as_bytes();
+ // Any require_alias mention forces full parse (bool extraction is
+ // not handled in the fast path); uncommon enough to not matter.
+ let has_require_alias = bytes.windows(14).any(|w| w == b"require_alias\"");
+ let has_pipeline = bytes.windows(9).any(|w| w == b"pipeline\"");
+ let has_if_seq_no = bytes.windows(11).any(|w| w == b"\"if_seq_no\"");
+ let has_routing = bytes.windows(9).any(|w| w == b"\"routing\"");
+ let has_source = bytes.windows(9).any(|w| w == b"\"_source\"");
+ let has_dynamic_templates =
+ bytes.windows(19).any(|w| w == b"\"dynamic_templates\"");
+ let mut require_alias_per_item: Option<bool> = None;
+ let mut pipeline_per_item: Option<String> = None;
+ let mut if_seq_no_per_item: Option<u64> = None;
+ let mut if_primary_term_per_item: Option<u64> = None;
+ let mut routing_per_item: Option<String> = None;
+ let mut source_req_per_item: Option<Value> = None;
+ let mut dynamic_templates_per_item: Option<
+ std::collections::BTreeMap<String, String>,
+ > = None;
+ let (action_type, target_index, doc_id) = 'fast: {
+ if has_require_alias
+ || has_pipeline
+ || has_if_seq_no
+ || has_routing
+ || has_source
+ || has_dynamic_templates
+ {
+ break 'fast (None, None, None);
+ }
+ // Parse state: skip whitespace, expect `{`, "action": "index" etc.
+ let mut p = 0usize;
+ while p < bytes.len() && (bytes[p] == b' ' || bytes[p] == b'\t') {
+ p += 1;
+ }
+ if p >= bytes.len() || bytes[p] != b'{' {
+ break 'fast (None, None, None);
+ }
+ p += 1;
+ while p < bytes.len() && (bytes[p] == b' ' || bytes[p] == b'\t') {
+ p += 1;
+ }
+ if p >= bytes.len() || bytes[p] != b'"' {
+ break 'fast (None, None, None);
+ }
+ p += 1;
+ // Find closing quote of action name.
+ let key_start = p;
+ while p < bytes.len() && bytes[p] != b'"' {
+ p += 1;
+ }
+ if p >= bytes.len() {
+ break 'fast (None, None, None);
+ }
+ let key = &bytes[key_start..p];
+ let action_ty: &'static str = match key {
+ b"index" => "index",
+ b"create" => "create",
+ b"update" => "update",
+ b"delete" => "delete",
+ _ => break 'fast (None, None, None),
+ };
+ p += 1;
+ while p < bytes.len() && (bytes[p] == b' ' || bytes[p] == b':') {
+ p += 1;
+ }
+ if p >= bytes.len() || bytes[p] != b'{' {
+ break 'fast (None, None, None);
+ }
+ // Extract optional _index and _id inside the inner object.
+ // Both are bare-quoted strings with no embedded quotes in
+ // the canonical bulk format produced by ES clients. The
+ // inner object itself may contain nested objects (e.g.
+ // `dynamic_templates:{...}`), so we track brace depth —
+ // a naive "first `}`" scan would truncate the inner slice
+ // at the first nested `}` and hide fields like `op_type`.
+ let inner_end = {
+ let mut j = p + 1;
+ let mut depth: i32 = 1;
+ let mut in_str = false;
+ while j < bytes.len() {
+ let c = bytes[j];
+ if in_str {
+ if c == b'\\' {
+ j += 2;
+ continue;
+ }
+ if c == b'"' {
+ in_str = false;
+ }
+ } else {
+ match c {
+ b'"' => in_str = true,
+ b'{' => depth += 1,
+ b'}' => {
+ depth -= 1;
+ if depth == 0 {
+ break;
+ }
+ }
+ _ => {}
}
- _ => {}
}
- }
- j += 1;
- }
- if j >= bytes.len() { break 'fast (None, None, None); }
- j
- };
- let inner = &bytes[p + 1..inner_end];
- let find_field = |needle: &[u8]| -> Option<String> {
- // Look for `"_key":"value"` (quoted) or `"_key":NUM`
- // (numeric, e.g. `_id:1`) inside `inner`. ES bulk
- // accepts both shapes — numeric ids are stringified at
- // the coordinator. An empty quoted value is preserved
- // as `Some(String::new())` so the caller can tell
- // "key absent" from "key present, empty".
- let mut i = 0;
- while i + needle.len() <= inner.len() {
- if &inner[i..i + needle.len()] == needle {
- let mut j = i + needle.len();
- while j < inner.len() && inner[j] != b':' { j += 1; }
j += 1;
- while j < inner.len() && inner[j] == b' ' { j += 1; }
- if j >= inner.len() { return None; }
- if inner[j] == b'"' {
+ }
+ if j >= bytes.len() {
+ break 'fast (None, None, None);
+ }
+ j
+ };
+ let inner = &bytes[p + 1..inner_end];
+ let find_field = |needle: &[u8]| -> Option<String> {
+ // Look for `"_key":"value"` (quoted) or `"_key":NUM`
+ // (numeric, e.g. `_id:1`) inside `inner`. ES bulk
+ // accepts both shapes — numeric ids are stringified at
+ // the coordinator. An empty quoted value is preserved
+ // as `Some(String::new())` so the caller can tell
+ // "key absent" from "key present, empty".
+ let mut i = 0;
+ while i + needle.len() <= inner.len() {
+ if &inner[i..i + needle.len()] == needle {
+ let mut j = i + needle.len();
+ while j < inner.len() && inner[j] != b':' {
+ j += 1;
+ }
j += 1;
+ while j < inner.len() && inner[j] == b' ' {
+ j += 1;
+ }
+ if j >= inner.len() {
+ return None;
+ }
+ if inner[j] == b'"' {
+ j += 1;
+ let v_start = j;
+ while j < inner.len() && inner[j] != b'"' {
+ j += 1;
+ }
+ return Some(
+ std::str::from_utf8(&inner[v_start..j]).ok()?.to_string(),
+ );
+ }
+ // Unquoted numeric id (e.g. `"_id":1`).
let v_start = j;
- while j < inner.len() && inner[j] != b'"' { j += 1; }
+ while j < inner.len()
+ && (inner[j].is_ascii_digit()
+ || inner[j] == b'-'
+ || inner[j] == b'.')
+ {
+ j += 1;
+ }
+ if v_start == j {
+ return None;
+ }
return Some(
std::str::from_utf8(&inner[v_start..j]).ok()?.to_string(),
);
}
- // Unquoted numeric id (e.g. `"_id":1`).
- let v_start = j;
- while j < inner.len() && (inner[j].is_ascii_digit() || inner[j] == b'-' || inner[j] == b'.') {
- j += 1;
- }
- if v_start == j { return None; }
- return Some(
- std::str::from_utf8(&inner[v_start..j]).ok()?.to_string(),
- );
+ i += 1;
}
- i += 1;
- }
- None
- };
- let idx_name = find_field(br#""_index""#);
- let id = find_field(br#""_id""#);
- // ES allows `{"index": {"op_type": "create", ...}}` as a shorthand
- // for a `create` action; promote it so downstream treats it as create.
- let op_type = find_field(br#""op_type""#);
- let final_ty: &'static str = if action_ty == "index" && op_type.as_deref() == Some("create") {
- "create"
- } else {
- action_ty
+ None
+ };
+ let idx_name = find_field(br#""_index""#);
+ let id = find_field(br#""_id""#);
+ // ES allows `{"index": {"op_type": "create", ...}}` as a shorthand
+ // for a `create` action; promote it so downstream treats it as create.
+ let op_type = find_field(br#""op_type""#);
+ let final_ty: &'static str =
+ if action_ty == "index" && op_type.as_deref() == Some("create") {
+ "create"
+ } else {
+ action_ty
+ };
+ (Some(final_ty), idx_name, id)
};
- (Some(final_ty), idx_name, id)
- };
- let (action_type, target_index, doc_id) = if let Some(ty) = action_type {
- (ty, target_index, doc_id)
- } else {
- // Fallback: serde_json for non-canonical action lines.
- // simd_json would need a mutable buffer (action_line is &str
- // borrowed from the request body), forcing a per-action
- // to_vec(). For the small action JSONs typical here (a few
- // hundred bytes), serde_json's borrow-only path beats
- // simd_json + allocation on the wall clock.
- let action_val: Value = match serde_json::from_slice(action_line.as_bytes()) {
- Ok(v) => v,
- Err(e) => {
+ let (action_type, target_index, doc_id) = if let Some(ty) = action_type {
+ (ty, target_index, doc_id)
+ } else {
+ // Fallback: serde_json for non-canonical action lines.
+ // simd_json would need a mutable buffer (action_line is &str
+ // borrowed from the request body), forcing a per-action
+ // to_vec(). For the small action JSONs typical here (a few
+ // hundred bytes), serde_json's borrow-only path beats
+ // simd_json + allocation on the wall clock.
+ let action_val: Value = match serde_json::from_slice(action_line.as_bytes()) {
+ Ok(v) => v,
+ Err(e) => {
+ return ParseOutcome::Err(
+ BulkItemResult {
+ action: "unknown".into(),
+ index: default_index.unwrap_or("_unknown").to_string(),
+ id: String::new(),
+ status: 400,
+ result: None,
+ error: Some(format!("failed to parse action line: {e}")),
+ get_source: None,
+ },
+ *item_index,
+ );
+ }
+ };
+ let (a, meta) = if let Some(obj) = action_val.get("index") {
+ ("index", obj)
+ } else if let Some(obj) = action_val.get("create") {
+ ("create", obj)
+ } else if let Some(obj) = action_val.get("update") {
+ ("update", obj)
+ } else if let Some(obj) = action_val.get("delete") {
+ ("delete", obj)
+ } else {
return ParseOutcome::Err(
BulkItemResult {
action: "unknown".into(),
@@ -359,132 +437,84 @@ pub async fn process_bulk_with_opts(
id: String::new(),
status: 400,
result: None,
- error: Some(format!("failed to parse action line: {e}")),
- get_source: None,
+ error: Some("unknown action type".to_string()),
+ get_source: None,
},
*item_index,
);
- }
- };
- let (a, meta) = if let Some(obj) = action_val.get("index") {
- ("index", obj)
- } else if let Some(obj) = action_val.get("create") {
- ("create", obj)
- } else if let Some(obj) = action_val.get("update") {
- ("update", obj)
- } else if let Some(obj) = action_val.get("delete") {
- ("delete", obj)
- } else {
- return ParseOutcome::Err(
- BulkItemResult {
- action: "unknown".into(),
- index: default_index.unwrap_or("_unknown").to_string(),
- id: String::new(),
- status: 400,
- result: None,
- error: Some("unknown action type".to_string()),
- get_source: None,
- },
- *item_index,
- );
- };
- let op_type_str = meta.get("op_type").and_then(Value::as_str);
- let final_a: &'static str = if a == "index" && op_type_str == Some("create") {
- "create"
- } else {
- a
+ };
+ let op_type_str = meta.get("op_type").and_then(Value::as_str);
+ let final_a: &'static str = if a == "index" && op_type_str == Some("create") {
+ "create"
+ } else {
+ a
+ };
+ require_alias_per_item = meta.get("require_alias").and_then(Value::as_bool);
+ pipeline_per_item = meta
+ .get("pipeline")
+ .and_then(Value::as_str)
+ .map(str::to_owned);
+ if_seq_no_per_item = meta.get("if_seq_no").and_then(Value::as_u64);
+ if_primary_term_per_item = meta.get("if_primary_term").and_then(Value::as_u64);
+ routing_per_item = meta
+ .get("routing")
+ .and_then(Value::as_str)
+ .map(str::to_owned);
+ source_req_per_item = meta.get("_source").cloned();
+ dynamic_templates_per_item = meta
+ .get("dynamic_templates")
+ .and_then(Value::as_object)
+ .map(|o| {
+ o.iter()
+ .filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string())))
+ .collect()
+ });
+ // ES bulk accepts `_id` as either a string or a number;
+ // numeric ids are stringified at the coordinating node.
+ let id_from_meta = meta.get("_id").and_then(|v| match v {
+ Value::String(s) => Some(s.clone()),
+ Value::Number(n) => Some(n.to_string()),
+ Value::Bool(b) => Some(b.to_string()),
+ _ => None,
+ });
+ (
+ final_a,
+ meta.get("_index")
+ .and_then(Value::as_str)
+ .map(str::to_owned),
+ id_from_meta,
+ )
};
- require_alias_per_item = meta.get("require_alias").and_then(Value::as_bool);
- pipeline_per_item = meta.get("pipeline").and_then(Value::as_str).map(str::to_owned);
- if_seq_no_per_item = meta.get("if_seq_no").and_then(Value::as_u64);
- if_primary_term_per_item = meta.get("if_primary_term").and_then(Value::as_u64);
- routing_per_item = meta.get("routing").and_then(Value::as_str).map(str::to_owned);
- source_req_per_item = meta.get("_source").cloned();
- dynamic_templates_per_item = meta.get("dynamic_templates")
- .and_then(Value::as_object)
- .map(|o| o.iter()
- .filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string())))
- .collect());
- // ES bulk accepts `_id` as either a string or a number;
- // numeric ids are stringified at the coordinating node.
- let id_from_meta = meta.get("_id").and_then(|v| match v {
- Value::String(s) => Some(s.clone()),
- Value::Number(n) => Some(n.to_string()),
- Value::Bool(b) => Some(b.to_string()),
- _ => None,
- });
- (
- final_a,
- meta.get("_index").and_then(Value::as_str).map(str::to_owned),
- id_from_meta,
- )
- };
- let target_index = target_index
- .or_else(|| default_index.map(str::to_owned))
- .unwrap_or_else(|| "_unknown".to_string());
+ let target_index = target_index
+ .or_else(|| default_index.map(str::to_owned))
+ .unwrap_or_else(|| "_unknown".to_string());
- // ES rejects an explicit empty `_id` with an
- // illegal_argument_exception per-item error. `_id` missing
- // entirely still auto-generates; only the present-but-empty
- // case is wrong.
- if let Some(s) = doc_id.as_deref() {
- if s.is_empty() {
- return ParseOutcome::Err(
- BulkItemResult {
- action: action_type.to_string(),
- index: target_index,
- id: String::new(),
- status: 400,
- result: None,
- error: Some("if _id is specified it must not be empty".to_string()),
- get_source: None,
- },
- *item_index,
- );
- }
- }
-
- let (doc_body, doc_bytes) = if action_type != "delete" {
- let dl = match doc_line {
- Some(dl) => dl,
- None => {
+ // ES rejects an explicit empty `_id` with an
+ // illegal_argument_exception per-item error. `_id` missing
+ // entirely still auto-generates; only the present-but-empty
+ // case is wrong.
+ if let Some(s) = doc_id.as_deref() {
+ if s.is_empty() {
return ParseOutcome::Err(
BulkItemResult {
action: action_type.to_string(),
index: target_index,
- id: doc_id.unwrap_or_default(),
+ id: String::new(),
status: 400,
result: None,
- error: Some("missing document body".to_string()),
- get_source: None,
+ error: Some("if _id is specified it must not be empty".to_string()),
+ get_source: None,
},
*item_index,
);
}
- };
- // M5.11 — SKIP the doc-body JSON parse for `index`
- // actions. The turbo-raw ingest path accepts the raw
- // NDJSON bytes and defers the parse to drain-for-flush
- // on a background thread, taking ~5 µs/doc off the HTTP
- // worker's critical path. `update` and `create`
- // actions still need the parsed Value because they
- // take different code paths below — but `index` is
- // the dominant one on ingest benchmarks.
- if action_type == "index" {
- let bytes: std::sync::Arc<[u8]> =
- std::sync::Arc::from(dl.as_bytes());
- (None, bytes)
- } else {
- // See action_line fallback above for why serde_json beats
- // simd_json + to_vec() on small bulk doc lines.
- match serde_json::from_slice::<Value>(dl.as_bytes()) {
- Ok(v) => {
- let bytes: std::sync::Arc<[u8]> =
- std::sync::Arc::from(dl.as_bytes());
- (Some(v), bytes)
- }
- Err(e) => {
+ }
+
+ let (doc_body, doc_bytes) = if action_type != "delete" {
+ let dl = match doc_line {
+ Some(dl) => dl,
+ None => {
return ParseOutcome::Err(
BulkItemResult {
action: action_type.to_string(),
@@ -492,76 +522,112 @@ pub async fn process_bulk_with_opts(
id: doc_id.unwrap_or_default(),
status: 400,
result: None,
- error: Some(format!("invalid document JSON: {e}")),
- get_source: None,
+ error: Some("missing document body".to_string()),
+ get_source: None,
},
*item_index,
);
}
+ };
+ // M5.11 — SKIP the doc-body JSON parse for `index`
+ // actions. The turbo-raw ingest path accepts the raw
+ // NDJSON bytes and defers the parse to drain-for-flush
+ // on a background thread, taking ~5 µs/doc off the HTTP
+ // worker's critical path. `update` and `create`
+ // actions still need the parsed Value because they
+ // take different code paths below — but `index` is
+ // the dominant one on ingest benchmarks.
+ if action_type == "index" {
+ let bytes: std::sync::Arc<[u8]> = std::sync::Arc::from(dl.as_bytes());
+ (None, bytes)
+ } else {
+ // See action_line fallback above for why serde_json beats
+ // simd_json + to_vec() on small bulk doc lines.
+ match serde_json::from_slice::<Value>(dl.as_bytes()) {
+ Ok(v) => {
+ let bytes: std::sync::Arc<[u8]> =
+ std::sync::Arc::from(dl.as_bytes());
+ (Some(v), bytes)
+ }
+ Err(e) => {
+ return ParseOutcome::Err(
+ BulkItemResult {
+ action: action_type.to_string(),
+ index: target_index,
+ id: doc_id.unwrap_or_default(),
+ status: 400,
+ result: None,
+ error: Some(format!("invalid document JSON: {e}")),
+ get_source: None,
+ },
+ *item_index,
+ );
+ }
+ }
}
- }
- } else {
- // Delete action — we may have optimistically consumed
- // a line during pair building that actually belongs to
- // the NEXT action. Signal the caller to pop it back
- // into the input stream.
- let skip_back = if doc_line.is_some() { 1usize } else { 0usize };
- let action = ParsedAction {
+ } else {
+ // Delete action — we may have optimistically consumed
+ // a line during pair building that actually belongs to
+ // the NEXT action. Signal the caller to pop it back
+ // into the input stream.
+ let skip_back = if doc_line.is_some() { 1usize } else { 0usize };
+ let action = ParsedAction {
+ action_type: action_type.to_string(),
+ target_index,
+ doc_id,
+ doc_body: None,
+ doc_bytes: std::sync::Arc::from(&[][..]),
+ item_index: *item_index,
+ require_alias: require_alias_per_item,
+ pipeline: pipeline_per_item.clone(),
+ if_seq_no: if_seq_no_per_item,
+ if_primary_term: if_primary_term_per_item,
+ routing: routing_per_item.clone(),
+ source_req: source_req_per_item.clone(),
+ dynamic_templates: dynamic_templates_per_item.clone(),
+ };
+ if skip_back > 0 {
+ // We need to signal that the "body" we consumed
+ // was actually a new action line — but with a
+ // parallel parse this is hard because lines are
+ // not in order. For now, fall back to the
+ // serial path for delete-heavy batches.
+ // In practice nginx log ingest has ZERO deletes,
+ // so this branch is cold.
+ return ParseOutcome::Ok(action);
+ }
+ return ParseOutcome::Ok(action);
+ };
+
+ // For update actions, the request may put `_source` on the
+ // doc body rather than the action meta
+ // (`{_source: true, doc: {...}}`). Honor that too.
+ let source_req_final = source_req_per_item.or_else(|| {
+ if action_type == "update" {
+ doc_body.as_ref().and_then(|b| b.get("_source").cloned())
+ } else {
+ None
+ }
+ });
+
+ ParseOutcome::Ok(ParsedAction {
action_type: action_type.to_string(),
target_index,
doc_id,
- doc_body: None,
- doc_bytes: std::sync::Arc::from(&[][..]),
+ doc_body,
+ doc_bytes,
item_index: *item_index,
require_alias: require_alias_per_item,
- pipeline: pipeline_per_item.clone(),
+ pipeline: pipeline_per_item,
if_seq_no: if_seq_no_per_item,
if_primary_term: if_primary_term_per_item,
- routing: routing_per_item.clone(),
- source_req: source_req_per_item.clone(),
- dynamic_templates: dynamic_templates_per_item.clone(),
- };
- if skip_back > 0 {
- // We need to signal that the "body" we consumed
- // was actually a new action line — but with a
- // parallel parse this is hard because lines are
- // not in order. For now, fall back to the
- // serial path for delete-heavy batches.
- // In practice nginx log ingest has ZERO deletes,
- // so this branch is cold.
- return ParseOutcome::Ok(action);
- }
- return ParseOutcome::Ok(action);
- };
-
- // For update actions, the request may put `_source` on the
- // doc body rather than the action meta
- // (`{_source: true, doc: {...}}`). Honor that too.
- let source_req_final = source_req_per_item.or_else(|| {
- if action_type == "update" {
- doc_body.as_ref().and_then(|b| b.get("_source").cloned())
- } else {
- None
- }
- });
-
- ParseOutcome::Ok(ParsedAction {
- action_type: action_type.to_string(),
- target_index,
- doc_id,
- doc_body,
- doc_bytes,
- item_index: *item_index,
- require_alias: require_alias_per_item,
- pipeline: pipeline_per_item,
- if_seq_no: if_seq_no_per_item,
- if_primary_term: if_primary_term_per_item,
- routing: routing_per_item,
- source_req: source_req_final,
- dynamic_templates: dynamic_templates_per_item,
+ routing: routing_per_item,
+ source_req: source_req_final,
+ dynamic_templates: dynamic_templates_per_item,
+ })
})
- })
- .collect());
+ .collect()
+ });
let parse_ms = t_parse.elapsed().as_millis() as u64;
let t_group = std::time::Instant::now();
@@ -573,7 +639,6 @@ pub async fn process_bulk_with_opts(
items[item_idx] = Some(err_item);
errors = true;
}
- ParseOutcome::Skip(_) => {}
}
}
@@ -592,7 +657,7 @@ pub async fn process_bulk_with_opts(
status: 400,
result: None,
error: Some(format!("pipeline with id [{pid}] does not exist")),
- get_source: None,
+ get_source: None,
});
errors = true;
return false;
@@ -680,10 +745,8 @@ pub async fn process_bulk_with_opts(
// Value straight into the batch.
// M5.11 — index actions carry only `(item_index, doc_id, doc_bytes)`.
// The parsed `Value` is never built for index actions on this path.
- let mut index_batches: HashMap<
- String,
- Vec<(usize, String, std::sync::Arc<[u8]>)>,
- > = HashMap::new();
+ let mut index_batches: HashMap<String, Vec<(usize, String, std::sync::Arc<[u8]>)>> =
+ HashMap::new();
// AUTO-ID plain-index actions (no explicit `_id`). These are
// guaranteed new — they can NEVER overwrite — so every one is a
// "created"/201. That makes them safe to push through the single
@@ -691,10 +754,8 @@ pub async fn process_bulk_with_opts(
// to "created". Explicit-id index actions stay in `index_batches`
// and run the per-doc loop because they may overwrite and so need
// created-vs-updated / 201-vs-200 semantics turbo can't express.
- let mut auto_id_batches: HashMap<
- String,
- Vec<(usize, String, std::sync::Arc<[u8]>)>,
- > = HashMap::new();
+ let mut auto_id_batches: HashMap<String, Vec<(usize, String, std::sync::Arc<[u8]>)>> =
+ HashMap::new();
let mut non_index_actions: Vec<ParsedAction> = Vec::new();
// Precompute which target indices declare any strict-format date
@@ -703,12 +764,21 @@ pub async fn process_bulk_with_opts(
// slow path so the validation loop below can see them.
let mut index_needs_date_validation: HashMap<String, bool> = HashMap::new();
let index_has_strict_date = |target: &str, cache: &mut HashMap<String, bool>| -> bool {
- if let Some(b) = cache.get(target) { return *b; }
- let has = engine.index_mappings.get(target).map(|m| {
- let props = m.get("properties").cloned()
- .or_else(|| m.get("mappings").and_then(|mm| mm.get("properties")).cloned());
- mapping_has_strict_date(&props.unwrap_or(Value::Null))
- }).unwrap_or(false);
+ if let Some(b) = cache.get(target) {
+ return *b;
+ }
+ let has = engine
+ .index_mappings
+ .get(target)
+ .map(|m| {
+ let props = m.get("properties").cloned().or_else(|| {
+ m.get("mappings")
+ .and_then(|mm| mm.get("properties"))
+ .cloned()
+ });
+ mapping_has_strict_date(&props.unwrap_or(Value::Null))
+ })
+ .unwrap_or(false);
cache.insert(target.to_string(), has);
has
};
@@ -719,17 +789,34 @@ pub async fn process_bulk_with_opts(
// so route those actions through the slow path.
let mut index_needs_dynamic_copy: HashMap<String, bool> = HashMap::new();
let index_has_dynamic_copy = |target: &str, cache: &mut HashMap<String, bool>| -> bool {
- if let Some(b) = cache.get(target) { return *b; }
- let has = engine.index_mappings.get(target).map(|m| {
- let tmpls = m.get("mappings")
- .and_then(|mm| mm.get("dynamic_templates"))
- .or_else(|| m.get("dynamic_templates"));
- tmpls.and_then(Value::as_array).map(|arr| arr.iter().any(|v| {
- v.as_object().and_then(|o| o.iter().next()).map(|(_, body)| {
- body.get("mapping").and_then(|mm| mm.get("copy_to")).is_some()
- }).unwrap_or(false)
- })).unwrap_or(false)
- }).unwrap_or(false);
+ if let Some(b) = cache.get(target) {
+ return *b;
+ }
+ let has = engine
+ .index_mappings
+ .get(target)
+ .map(|m| {
+ let tmpls = m
+ .get("mappings")
+ .and_then(|mm| mm.get("dynamic_templates"))
+ .or_else(|| m.get("dynamic_templates"));
+ tmpls
+ .and_then(Value::as_array)
+ .map(|arr| {
+ arr.iter().any(|v| {
+ v.as_object()
+ .and_then(|o| o.iter().next())
+ .map(|(_, body)| {
+ body.get("mapping")
+ .and_then(|mm| mm.get("copy_to"))
+ .is_some()
+ })
+ .unwrap_or(false)
+ })
+ })
+ .unwrap_or(false)
+ })
+ .unwrap_or(false);
cache.insert(target.to_string(), has);
has
};
@@ -749,22 +836,25 @@ pub async fn process_bulk_with_opts(
&& !index_has_strict_date(&action.target_index, &mut index_needs_date_validation)
&& !index_has_dynamic_copy(&action.target_index, &mut index_needs_dynamic_copy);
if is_plain_index {
- if action.doc_id.is_none() {
- // Auto-id: brand-new doc, always "created"/201. Route the
- // whole group through ONE `index_batch_turbo_raw` call.
- let id = uuid::Uuid::new_v4().to_string();
- auto_id_batches
- .entry(action.target_index)
- .or_default()
- .push((action.item_index, id, action.doc_bytes));
- } else {
- // Explicit id: may overwrite → keep the per-doc path so
- // created/updated + 201/200 is resolved correctly.
- let id = action.doc_id.unwrap();
- index_batches
- .entry(action.target_index)
- .or_default()
- .push((action.item_index, id, action.doc_bytes));
+ match action.doc_id {
+ None => {
+ // Auto-id: brand-new doc, always "created"/201. Route the
+ // whole group through ONE `index_batch_turbo_raw` call.
+ let id = uuid::Uuid::new_v4().to_string();
+ auto_id_batches
+ .entry(action.target_index)
+ .or_default()
+ .push((action.item_index, id, action.doc_bytes));
+ }
+ Some(id) => {
+ // Explicit id: may overwrite → keep the per-doc path so
+ // created/updated + 201/200 is resolved correctly.
+ index_batches.entry(action.target_index).or_default().push((
+ action.item_index,
+ id,
+ action.doc_bytes,
+ ));
+ }
}
} else {
non_index_actions.push(action);
@@ -861,7 +951,7 @@ pub async fn process_bulk_with_opts(
status: 500,
result: None,
error: Some(e.to_string()),
- get_source: None,
+ get_source: None,
});
errors = true;
}
@@ -881,7 +971,11 @@ pub async fn process_bulk_with_opts(
bytes_len = doc_bytes.len(),
"bulk: indexing doc via per-doc path"
);
- let id_opt = if doc_id.is_empty() { None } else { Some(doc_id) };
+ let id_opt = if doc_id.is_empty() {
+ None
+ } else {
+ Some(doc_id)
+ };
match idx.index_document(id_opt, source).await {
Ok(resp) => {
// 201 for new doc, 200 for overwrite (ES semantics).
@@ -893,12 +987,14 @@ pub async fn process_bulk_with_opts(
status,
result: Some(resp.result),
error: None,
- get_source: None,
+ get_source: None,
});
}
Err(e) => {
let status = match &e {
- EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. }) => 429,
+ EngineError::Common(xerj_common::XerjError::ResourceExhausted {
+ ..
+ }) => 429,
_ => 500,
};
items[item_idx] = Some(BulkItemResult {
@@ -908,7 +1004,7 @@ pub async fn process_bulk_with_opts(
status,
result: None,
error: Some(e.to_string()),
- get_source: None,
+ get_source: None,
});
errors = true;
}
@@ -953,7 +1049,9 @@ pub async fn process_bulk_with_opts(
// forced a per-doc to_vec(). serde_json takes &[u8]
// immutably and avoids the allocation entirely.
serde_json::from_slice::<Value>(&doc_bytes).ok()
- } else { None }
+ } else {
+ None
+ }
});
// Apply dynamic-template `copy_to` at ingest. When a doc
// field name matches a dynamic template's `match` pattern
@@ -989,21 +1087,34 @@ pub async fn process_bulk_with_opts(
// template instead).
let templates: Vec<(String, Option<String>)> = mapping
.as_ref()
- .and_then(|m| m.get("mappings").and_then(|mm| mm.get("dynamic_templates"))
- .or_else(|| m.get("dynamic_templates")))
+ .and_then(|m| {
+ m.get("mappings")
+ .and_then(|mm| mm.get("dynamic_templates"))
+ .or_else(|| m.get("dynamic_templates"))
+ })
.and_then(Value::as_array)
- .map(|arr| arr.iter().filter_map(|v| v.as_object().and_then(|o| {
- let (name, spec) = o.iter().next()?;
- let pattern = spec.get("match").and_then(Value::as_str).map(String::from);
- Some((name.clone(), pattern))
- })).collect())
+ .map(|arr| {
+ arr.iter()
+ .filter_map(|v| {
+ v.as_object().and_then(|o| {
+ let (name, spec) = o.iter().next()?;
+ let pattern =
+ spec.get("match").and_then(Value::as_str).map(String::from);
+ Some((name.clone(), pattern))
+ })
+ })
+ .collect()
+ })
.unwrap_or_default();
- let declared_names: std::collections::HashSet<String> = templates
- .iter().map(|(n, _)| n.clone()).collect();
+ let declared_names: std::collections::HashSet<String> =
+ templates.iter().map(|(n, _)| n.clone()).collect();
let declared_fields: std::collections::HashSet<String> = mapping
.as_ref()
- .and_then(|m| m.get("mappings").and_then(|mm| mm.get("properties"))
- .or_else(|| m.get("properties")))
+ .and_then(|m| {
+ m.get("mappings")
+ .and_then(|mm| mm.get("properties"))
+ .or_else(|| m.get("properties"))
+ })
.and_then(Value::as_object)
.map(|o| o.keys().cloned().collect())
.unwrap_or_default();
@@ -1024,15 +1135,28 @@ pub async fn process_bulk_with_opts(
};
let mut first_bad: Option<(String, String)> = None;
for (field, tmpl) in dt_map {
- let present_in_doc = doc_body.as_ref()
- .map(|db| db.as_object().map(|o| o.contains_key(field)).unwrap_or(false))
+ let present_in_doc = doc_body
+ .as_ref()
+ .map(|db| {
+ db.as_object()
+ .map(|o| o.contains_key(field))
+ .unwrap_or(false)
+ })
.unwrap_or(true);
- if declared_fields.contains(field) { continue; }
- if !present_in_doc { continue; }
- if declared_names.contains(tmpl) { continue; }
+ if declared_fields.contains(field) {
+ continue;
+ }
+ if !present_in_doc {
+ continue;
+ }
+ if declared_names.contains(tmpl) {
+ continue;
+ }
// Fall back: the field might be implicitly mapped by
// a pattern-matching template (e.g. `match: "my*"`).
- if field_matches_any_template(field) { continue; }
+ if field_matches_any_template(field) {
+ continue;
+ }
first_bad = Some((field.clone(), tmpl.clone()));
break;
}
@@ -1048,36 +1172,68 @@ pub async fn process_bulk_with_opts(
// spec so we can inspect its `type`.
let tmpl_specs: std::collections::HashMap<String, Value> = mapping
.as_ref()
- .and_then(|m| m.get("mappings").and_then(|mm| mm.get("dynamic_templates"))
- .or_else(|| m.get("dynamic_templates")))
+ .and_then(|m| {
+ m.get("mappings")
+ .and_then(|mm| mm.get("dynamic_templates"))
+ .or_else(|| m.get("dynamic_templates"))
+ })
.and_then(Value::as_array)
- .map(|arr| arr.iter().filter_map(|v| v.as_object().and_then(|o| {
- let (name, spec) = o.iter().next()?;
- let mapping_spec = spec.get("mapping").cloned().unwrap_or(Value::Null);
- Some((name.clone(), mapping_spec))
- })).collect())
+ .map(|arr| {
+ arr.iter()
+ .filter_map(|v| {
+ v.as_object().and_then(|o| {
+ let (name, spec) = o.iter().next()?;
+ let mapping_spec =
+ spec.get("mapping").cloned().unwrap_or(Value::Null);
+ Some((name.clone(), mapping_spec))
+ })
+ })
+ .collect()
+ })
.unwrap_or_default();
let mut first_type_clash: Option<(String, String, String)> = None;
for (field, tmpl) in dt_map {
- let Some(spec) = tmpl_specs.get(tmpl) else { continue };
+ let Some(spec) = tmpl_specs.get(tmpl) else {
+ continue;
+ };
let ftype = spec.get("type").and_then(Value::as_str).unwrap_or("");
// Scalar-like types that can't hold an object.
let scalar = matches!(
ftype,
- "keyword" | "text" | "match_only_text" | "long" | "integer" |
- "short" | "byte" | "double" | "float" | "boolean" | "date" |
- "date_nanos" | "ip" | "binary" | "half_float" | "scaled_float" |
- "unsigned_long" | "geo_point"
+ "keyword"
+ | "text"
+ | "match_only_text"
+ | "long"
+ | "integer"
+ | "short"
+ | "byte"
+ | "double"
+ | "float"
+ | "boolean"
+ | "date"
+ | "date_nanos"
+ | "ip"
+ | "binary"
+ | "half_float"
+ | "scaled_float"
+ | "unsigned_long"
+ | "geo_point"
);
- if !scalar { continue; }
+ if !scalar {
+ continue;
+ }
let prefix = format!("{}.", field);
let has_dotted = body.keys().any(|k| k.starts_with(&prefix));
- if !has_dotted { continue; }
+ if !has_dotted {
+ continue;
+ }
// Build a preview of the "object-shaped"
// value ES would render (e.g. `{bar=hello world}`).
let mut parts: Vec<String> = Vec::new();
for (k, v) in body.iter() {
- if !k.starts_with(&prefix) { continue; }
+ if !k.starts_with(&prefix) {
+ continue;
+ }
let sub_key = &k[prefix.len()..];
let rendered = match v {
Value::String(s) => s.clone(),
@@ -1137,11 +1293,11 @@ pub async fn process_bulk_with_opts(
// `create`; updates operate on already-stored docs.
if matches!(action_type.as_str(), "index" | "create") {
if let Some(body) = doc_body.as_ref().and_then(Value::as_object) {
- let mapping_props = engine.index_mappings.get(&target_index)
- .and_then(|m| {
- m.get("properties").or_else(|| m.get("mappings").and_then(|mm| mm.get("properties")))
- .cloned()
- });
+ let mapping_props = engine.index_mappings.get(&target_index).and_then(|m| {
+ m.get("properties")
+ .or_else(|| m.get("mappings").and_then(|mm| mm.get("properties")))
+ .cloned()
+ });
if let Some(props) = mapping_props {
if let Some(bad) = find_bad_date_field(body, &props) {
let reason = format!(
@@ -1192,21 +1348,28 @@ pub async fn process_bulk_with_opts(
// via `items[N].update.get._source`. Source spec comes
// from (priority): action metadata → doc-body `_source`
// → URL-level default (?_source=true).
- let effective_source_req = source_req.clone().or_else(|| default_source_req.clone());
- let get_source: Option<Value> = if action_type == "update" && effective_source_req.is_some() {
- match doc_id_for_get.as_deref() {
- Some(id) => {
- match engine.get_index(&target_index_for_get) {
+ let effective_source_req =
+ source_req.clone().or_else(|| default_source_req.clone());
+ let get_source: Option<Value> =
+ if action_type == "update" && effective_source_req.is_some() {
+ match doc_id_for_get.as_deref() {
+ Some(id) => match engine.get_index(&target_index_for_get) {
Ok(idx) => {
let fetched = idx.get_document(id).await.ok().flatten();
- fetched.map(|src| apply_bulk_source_filter(&src, effective_source_req.as_ref().unwrap()))
+ fetched.map(|src| {
+ apply_bulk_source_filter(
+ &src,
+ effective_source_req.as_ref().unwrap(),
+ )
+ })
}
Err(_) => None,
- }
+ },
+ None => None,
}
- None => None,
- }
- } else { None };
+ } else {
+ None
+ };
items[item_idx] = Some(BulkItemResult {
action: action_type.clone(),
index: target_index,
@@ -1235,17 +1398,14 @@ pub async fn process_bulk_with_opts(
status,
result: None,
error: Some(e.to_string()),
- get_source: None,
+ get_source: None,
});
}
}
}
// Flatten — any slot still None is a parse error that already set its slot.
- let final_items: Vec<BulkItemResult> = items
- .into_iter()
- .filter_map(|opt| opt)
- .collect();
+ let final_items: Vec<BulkItemResult> = items.into_iter().flatten().collect();
let exec_ms = t_exec.elapsed().as_millis() as u64;
let took_ms = started.elapsed().as_millis() as u64;
@@ -1253,7 +1413,12 @@ pub async fn process_bulk_with_opts(
if std::env::var_os("XERJ_PROF").is_some() {
eprintln!(
"XERJ_PROF bulk total_ms={} lines_ms={} parse_ms={} group_ms={} exec_ms={} n_lines={}",
- took_ms, lines_ms, parse_ms, group_ms, exec_ms, lines.len()
+ took_ms,
+ lines_ms,
+ parse_ms,
+ group_ms,
+ exec_ms,
+ lines.len()
);
}
if took_ms >= 50 {
@@ -1267,7 +1432,11 @@ pub async fn process_bulk_with_opts(
"process_bulk timings"
);
}
- BulkResult { took_ms, errors, items: final_items }
+ BulkResult {
+ took_ms,
+ errors,
+ items: final_items,
+ }
}
/// Apply a bulk-action `_source` request to a fetched document.
@@ -1287,15 +1456,18 @@ fn apply_bulk_source_filter(src: &Value, spec: &Value) -> Value {
Value::Bool(false) => return Value::Null,
Value::String(s) => (vec![s.clone()], Vec::new()),
Value::Array(arr) => (
- arr.iter().filter_map(|v| v.as_str().map(String::from)).collect(),
+ arr.iter()
+ .filter_map(|v| v.as_str().map(String::from))
+ .collect(),
Vec::new(),
),
Value::Object(o) => {
let pull = |k: &str| -> Vec<String> {
match o.get(k) {
- Some(Value::Array(a)) => {
- a.iter().filter_map(|x| x.as_str().map(String::from)).collect()
- }
+ Some(Value::Array(a)) => a
+ .iter()
+ .filter_map(|x| x.as_str().map(String::from))
+ .collect(),
Some(Value::String(s)) => vec![s.clone()],
_ => Vec::new(),
}
@@ -1342,8 +1514,12 @@ fn apply_dynamic_template_copy_to(
.get("mappings")
.and_then(|m| m.get("dynamic_templates"))
.or_else(|| mapping.get("dynamic_templates"));
- let Some(templates_arr) = templates_v.and_then(Value::as_array) else { return };
- if templates_arr.is_empty() { return; }
+ let Some(templates_arr) = templates_v.and_then(Value::as_array) else {
+ return;
+ };
+ if templates_arr.is_empty() {
+ return;
+ }
let declared_props: std::collections::HashSet<String> = mapping
.get("mappings")
.and_then(|m| m.get("properties"))
@@ -1358,14 +1534,14 @@ fn apply_dynamic_template_copy_to(
.filter_map(|v| {
let (name, body) = v.as_object().and_then(|o| o.iter().next())?;
let pattern = body.get("match").and_then(Value::as_str).map(String::from);
- let copy_to = body
- .get("mapping")
- .and_then(|m| m.get("copy_to"))
- .and_then(|c| match c {
- Value::String(s) => Some(s.clone()),
- Value::Array(a) => a.first().and_then(Value::as_str).map(String::from),
- _ => None,
- });
+ let copy_to =
+ body.get("mapping")
+ .and_then(|m| m.get("copy_to"))
+ .and_then(|c| match c {
+ Value::String(s) => Some(s.clone()),
+ Value::Array(a) => a.first().and_then(Value::as_str).map(String::from),
+ _ => None,
+ });
Some((name.clone(), pattern, copy_to))
})
.collect();
@@ -1374,7 +1550,9 @@ fn apply_dynamic_template_copy_to(
let mut touched_targets: Vec<String> = Vec::new();
let mut pristine: serde_json::Map<String, Value> = serde_json::Map::new();
for field in field_names {
- if declared_props.contains(&field) { continue; }
+ if declared_props.contains(&field) {
+ continue;
+ }
// Priority 1: per-item `dynamic_templates: {field: name}`.
let forced = per_item.and_then(|m| m.get(&field).cloned());
let mut chosen: Option<String> = None;
@@ -1398,7 +1576,9 @@ fn apply_dynamic_template_copy_to(
}
}
let Some(target_path) = chosen else { continue };
- let Some(val) = body.get(&field).cloned() else { continue };
+ let Some(val) = body.get(&field).cloned() else {
+ continue;
+ };
// Snapshot the target's pristine value before mutation so
// the synthetic-source emit path can restore the explicit-
// only source shape. Matches apply_copy_to's convention.
@@ -1414,7 +1594,9 @@ fn apply_dynamic_template_copy_to(
// value shape here.
match val {
Value::Array(arr) => {
- for v in arr { append_copy_to_path(body, &target_path, v); }
+ for v in arr {
+ append_copy_to_path(body, &target_path, v);
+ }
}
other => append_copy_to_path(body, &target_path, other),
}
@@ -1428,13 +1610,18 @@ fn apply_dynamic_template_copy_to(
pristine.entry(k).or_insert(v);
}
}
- body.insert("__xy_copy_to_pristine__".to_string(), Value::Object(pristine));
+ body.insert(
+ "__xy_copy_to_pristine__".to_string(),
+ Value::Object(pristine),
+ );
}
}
fn path_get<'a>(root: &'a serde_json::Map<String, Value>, path: &str) -> Option<&'a Value> {
let segs: Vec<&str> = path.split('.').collect();
- if segs.is_empty() { return None; }
+ if segs.is_empty() {
+ return None;
+ }
let mut cur: &Value = root.get(segs[0])?;
for seg in &segs[1..] {
cur = cur.as_object()?.get(*seg)?;
@@ -1442,21 +1629,25 @@ fn path_get<'a>(root: &'a serde_json::Map<String, Value>, path: &str) -> Option<
Some(cur)
}
-fn append_copy_to_path(
- target: &mut serde_json::Map<String, Value>,
- path: &str,
- val: Value,
-) {
+fn append_copy_to_path(target: &mut serde_json::Map<String, Value>, path: &str, val: Value) {
let segs: Vec<&str> = path.split('.').collect();
- if segs.is_empty() { return; }
+ if segs.is_empty() {
+ return;
+ }
let last = segs.len() - 1;
let mut cur: &mut serde_json::Map<String, Value> = target;
for (i, seg) in segs.iter().enumerate() {
if i == last {
let entry = cur.entry((*seg).to_string()).or_insert(Value::Null);
match entry {
- Value::Null => { *entry = val; return; }
- Value::Array(arr) => { arr.push(val); return; }
+ Value::Null => {
+ *entry = val;
+ return;
+ }
+ Value::Array(arr) => {
+ arr.push(val);
+ return;
+ }
existing => {
let prev = existing.clone();
*existing = Value::Array(vec![prev, val]);
@@ -1464,7 +1655,9 @@ fn append_copy_to_path(
}
}
}
- let entry = cur.entry((*seg).to_string()).or_insert_with(|| Value::Object(serde_json::Map::new()));
+ let entry = cur
+ .entry((*seg).to_string())
+ .or_insert_with(|| Value::Object(serde_json::Map::new()));
if !entry.is_object() {
let old = entry.take();
let mut m = serde_json::Map::new();
@@ -1480,17 +1673,24 @@ fn append_copy_to_path(
/// `format` set AND does not opt into `ignore_malformed`? When true,
/// ingest must parse-validate date values on this index.
fn mapping_has_strict_date(props: &Value) -> bool {
- let Some(obj) = props.as_object() else { return false };
+ let Some(obj) = props.as_object() else {
+ return false;
+ };
for (_, spec) in obj {
let ftype = spec.get("type").and_then(Value::as_str).unwrap_or("");
if (ftype == "date" || ftype == "date_nanos")
&& spec.get("format").and_then(Value::as_str).is_some()
- && !spec.get("ignore_malformed").and_then(Value::as_bool).unwrap_or(false)
+ && !spec
+ .get("ignore_malformed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false)
{
return true;
}
if let Some(sub) = spec.get("properties") {
- if mapping_has_strict_date(sub) { return true; }
+ if mapping_has_strict_date(sub) {
+ return true;
+ }
}
}
false
@@ -1511,11 +1711,20 @@ fn find_bad_date_field(
for (fname, spec) in props_obj {
let Some(val) = doc.get(fname) else { continue };
let ftype = spec.get("type").and_then(Value::as_str).unwrap_or("");
- if ftype != "date" && ftype != "date_nanos" { continue; }
+ if ftype != "date" && ftype != "date_nanos" {
+ continue;
+ }
let fmt = spec.get("format").and_then(Value::as_str).unwrap_or("");
- if fmt.is_empty() { continue; }
- let ignore_mal = spec.get("ignore_malformed").and_then(Value::as_bool).unwrap_or(false);
- if ignore_mal { continue; }
+ if fmt.is_empty() {
+ continue;
+ }
+ let ignore_mal = spec
+ .get("ignore_malformed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
+ if ignore_mal {
+ continue;
+ }
// Pick the single value to validate — arrays recurse over
// each element, using the first failure we find.
let candidates: Vec<&Value> = match val {
@@ -1585,19 +1794,30 @@ fn simple_glob_matches(pat: &[u8], txt: &[u8]) -> bool {
let (mut star, mut star_t) = (None::<usize>, 0usize);
while t < txt.len() {
if p < pat.len() && (pat[p] == b'?' || pat[p] == txt[t]) {
- p += 1; t += 1;
+ p += 1;
+ t += 1;
} else if p < pat.len() && pat[p] == b'*' {
- star = Some(p); star_t = t; p += 1;
+ star = Some(p);
+ star_t = t;
+ p += 1;
} else if let Some(sp) = star {
- p = sp + 1; star_t += 1; t = star_t;
+ p = sp + 1;
+ star_t += 1;
+ t = star_t;
} else {
return false;
}
}
- while p < pat.len() && pat[p] == b'*' { p += 1; }
+ while p < pat.len() && pat[p] == b'*' {
+ p += 1;
+ }
p == pat.len()
}
+// Per-item bulk execution needs all 8 pieces of action metadata (target,
+// id, body, CAS pair, routing); grouping them into a struct would just
+// reshuffle the same fields, so allow the arg count here.
+#[allow(clippy::too_many_arguments)]
async fn execute_bulk_action(
engine: &Engine,
action_type: &str,
@@ -1631,7 +1851,8 @@ async fn execute_bulk_action(
let idx = engine.get_or_create_index(index)?;
let source = inject_routing(body).unwrap_or(Value::Object(serde_json::Map::new()));
if if_seq_no.is_some() || if_primary_term.is_some() {
- idx.index_document_with_version(doc_id, source, if_seq_no, if_primary_term).await
+ idx.index_document_with_version(doc_id, source, if_seq_no, if_primary_term)
+ .await
} else {
idx.index_document(doc_id, source).await
}
@@ -1660,7 +1881,10 @@ async fn execute_bulk_action(
.and_then(Value::as_bool)
.unwrap_or(false);
- match idx.update_document_with_upsert(&id, partial, upsert, doc_as_upsert).await? {
+ match idx
+ .update_document_with_upsert(&id, partial, upsert, doc_as_upsert)
+ .await?
+ {
Some(resp) => Ok(resp),
None => Err(crate::EngineError::Common(
xerj_common::XerjError::document_not_found(&id, index),
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index cd0259216..54814351d 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -312,10 +312,7 @@ impl Engine {
let index_name = match IndexName::new(&name_str) {
Ok(n) => n,
Err(_) => {
- warn!(
- "Skipping directory '{}': not a valid index name",
- name_str
- );
+ warn!("Skipping directory '{}': not a valid index name", name_str);
continue;
}
};
@@ -368,9 +365,7 @@ impl Engine {
fn spawn_pit_sweeper(&self) {
let pits = Arc::clone(&self.pits);
- let interval = std::time::Duration::from_secs(
- self.config.pit.sweep_interval_secs.max(1),
- );
+ let interval = std::time::Duration::from_secs(self.config.pit.sweep_interval_secs.max(1));
tokio::spawn(async move {
let mut tick = tokio::time::interval(interval);
// Skip the immediate first tick — Engine::new just ran, so
@@ -384,7 +379,9 @@ impl Engine {
.filter(|e| e.value().expires_at <= now)
.map(|e| e.key().clone())
.collect();
- if expired.is_empty() { continue; }
+ if expired.is_empty() {
+ continue;
+ }
for id in &expired {
pits.remove(id);
}
@@ -399,8 +396,7 @@ impl Engine {
/// Create a new index, applying any matching index template.
pub fn create_index(&self, name: &str, schema: Schema) -> Result<()> {
- let index_name = IndexName::new(name)
- .map_err(|e| EngineError::Common(e))?;
+ let index_name = IndexName::new(name).map_err(EngineError::Common)?;
if self.indices.contains_key(name) {
return Err(EngineError::Common(
@@ -421,8 +417,15 @@ impl Engine {
.and_then(Value::as_str)
.unwrap_or("object");
let native_type = es_type_to_field_type(es_type);
- if !effective_schema.fields.iter().any(|f| &f.name == field_name) {
- let fc = xerj_common::types::FieldConfig::new(field_name.clone(), native_type);
+ if !effective_schema
+ .fields
+ .iter()
+ .any(|f| &f.name == field_name)
+ {
+ let fc = xerj_common::types::FieldConfig::new(
+ field_name.clone(),
+ native_type,
+ );
let _ = effective_schema.add_field(fc);
}
}
@@ -446,8 +449,7 @@ impl Engine {
schema: Schema,
settings: serde_json::Value,
) -> Result<()> {
- let index_name = IndexName::new(name)
- .map_err(|e| EngineError::Common(e))?;
+ let index_name = IndexName::new(name).map_err(EngineError::Common)?;
if self.indices.contains_key(name) {
return Err(EngineError::Common(
@@ -479,10 +481,9 @@ impl Engine {
if index_dir.is_dir() {
match serde_json::to_vec_pretty(&mapping) {
Ok(bytes) => {
- if let Err(e) = crate::index::write_file_atomic(
- &index_dir.join("es_mapping.json"),
- &bytes,
- ) {
+ if let Err(e) =
+ crate::index::write_file_atomic(&index_dir.join("es_mapping.json"), &bytes)
+ {
warn!(index = name, error = %e, "failed to persist es_mapping.json");
}
}
@@ -501,7 +502,9 @@ impl Engine {
/// back to schema-derived properties from `schema.json`.
fn load_persisted_es_mapping(&self, name: &str) {
let path = self.data_dir.join(name).join("es_mapping.json");
- let Ok(bytes) = std::fs::read(&path) else { return };
+ let Ok(bytes) = std::fs::read(&path) else {
+ return;
+ };
match serde_json::from_slice::<Value>(&bytes) {
Ok(mapping) => {
self.index_mappings.insert(name.to_string(), mapping);
@@ -517,7 +520,10 @@ impl Engine {
let mut best: Option<(i32, IndexTemplate)> = None;
for entry in self.templates.iter() {
let tmpl = entry.value();
- let matches = tmpl.index_patterns.iter().any(|pat| glob_match(pat, index_name));
+ let matches = tmpl
+ .index_patterns
+ .iter()
+ .any(|pat| glob_match(pat, index_name));
if matches {
let priority = tmpl.priority;
if best.as_ref().map(|(p, _)| priority > *p).unwrap_or(true) {
@@ -532,7 +538,7 @@ impl Engine {
/// Add an alias pointing to an index.
pub fn add_alias(&self, alias: &str, index: &str) {
- let mut entry = self.aliases.entry(alias.to_string()).or_insert_with(Vec::new);
+ let mut entry = self.aliases.entry(alias.to_string()).or_default();
if !entry.contains(&index.to_string()) {
entry.push(index.to_string());
}
@@ -562,11 +568,10 @@ impl Engine {
/// semantics) and clears the `closed_indices` flag so the name is
/// truly gone when another test recreates it.
pub async fn delete_index(&self, name: &str) -> Result<()> {
- let idx = self
- .indices
- .remove(name)
- .map(|(_, v)| v)
- .ok_or_else(|| EngineError::Common(xerj_common::XerjError::index_not_found(name)))?;
+ let idx =
+ self.indices.remove(name).map(|(_, v)| v).ok_or_else(|| {
+ EngineError::Common(xerj_common::XerjError::index_not_found(name))
+ })?;
idx.delete_all_data().await?;
@@ -603,10 +608,13 @@ impl Engine {
// Check if name is an alias — if so, resolve to the first backing index.
if let Some(aliased) = self.aliases.get(name) {
if let Some(real_name) = aliased.first() {
- return self.indices
+ return self
+ .indices
.get(real_name.as_str())
.map(|r| Arc::clone(r.value()))
- .ok_or_else(|| EngineError::Common(xerj_common::XerjError::index_not_found(real_name)));
+ .ok_or_else(|| {
+ EngineError::Common(xerj_common::XerjError::index_not_found(real_name))
+ });
}
}
self.indices
@@ -838,7 +846,11 @@ impl Engine {
ds.backing_indices.push(new_backing.clone());
}
- info!(name, new_backing = new_backing.as_str(), "data stream rolled over");
+ info!(
+ name,
+ new_backing = new_backing.as_str(),
+ "data stream rolled over"
+ );
Ok(new_backing)
}
@@ -887,8 +899,7 @@ impl Engine {
config_json: Value,
) -> std::result::Result<(), xerj_wasm::WasmError> {
let cfg: xerj_wasm::pipeline::PipelineConfig =
- serde_json::from_value(config_json.clone())
- .map_err(xerj_wasm::WasmError::Json)?;
+ serde_json::from_value(config_json.clone()).map_err(xerj_wasm::WasmError::Json)?;
let pipeline = xerj_wasm::pipeline::Pipeline::from_config(name, &cfg)?;
self.pipelines.insert(name.to_string(), config_json);
self.transform_pipelines.insert(name.to_string(), pipeline);
@@ -905,10 +916,8 @@ impl Engine {
&self,
pipeline_name: &str,
mut docs: Vec<Value>,
- ) -> std::result::Result<
- Vec<(xerj_wasm::pipeline::ProcessAction, Value)>,
- xerj_wasm::WasmError,
- > {
+ ) -> std::result::Result<Vec<(xerj_wasm::pipeline::ProcessAction, Value)>, xerj_wasm::WasmError>
+ {
let pipeline = self
.transform_pipelines
.get(pipeline_name)
@@ -1061,7 +1070,7 @@ impl Engine {
info!(snapshot = name, repo = repo_path, "snapshot restored");
Ok(())
}
-} // end impl Engine
+} // end impl Engine
// ── Private helpers ───────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index adcd75b7c..ecb8d32e2 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -44,12 +44,11 @@
use super::*;
use crate::aggs::{
- apply_bucket_pipeline_ops, doc_count_weight, doc_matches_filter, extract_date_ms_values,
- extract_field_values, extract_numeric, get_nested_field, interval_to_ms, is_calendar_interval,
- calendar_bucket_key, next_calendar_bucket, parse_date_ms, render_date_format,
- resolve_sibling_pipelines, run_pipeline_agg, run_sampler, run_top_hits_with_total,
- typed_term_key, render_iso_date, detect_fractional_digits,
- format_range_val,
+ apply_bucket_pipeline_ops, calendar_bucket_key, detect_fractional_digits, doc_count_weight,
+ doc_matches_filter, extract_date_ms_values, extract_field_values, extract_numeric,
+ format_range_val, get_nested_field, interval_to_ms, is_calendar_interval, next_calendar_bucket,
+ parse_date_ms, render_date_format, render_iso_date, resolve_sibling_pipelines,
+ run_pipeline_agg, run_sampler, run_top_hits_with_total, typed_term_key,
};
use serde_json::{json, Map, Value};
use std::collections::HashMap;
@@ -62,7 +61,11 @@ const FAST_AGG_MIN_DOCS: u64 = 10_000;
fn fast_aggs_disabled() -> bool {
static DISABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
- *DISABLED.get_or_init(|| std::env::var("XERJ_DISABLE_FAST_AGGS").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false))
+ *DISABLED.get_or_init(|| {
+ std::env::var("XERJ_DISABLE_FAST_AGGS")
+ .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
+ .unwrap_or(false)
+ })
}
/// One on-disk segment's columns + identity.
@@ -85,7 +88,10 @@ struct SegEntry {
/// version-map lookups) dominated terms/cardinality latency.
enum MemDocs {
Owned(Vec<Value>),
- Shared { ids: Vec<String>, srcs: Vec<std::sync::Arc<Value>> },
+ Shared {
+ ids: Vec<String>,
+ srcs: Vec<std::sync::Arc<Value>>,
+ },
}
impl MemDocs {
@@ -181,8 +187,7 @@ impl<'a> FastCtx<'a> {
o.entry("_index".to_string())
.or_insert_with(|| Value::String(self.idx.name.to_string()));
if let Some(seq) = self.idx.lookup_seq_no(&id) {
- o.entry("_seq_no".to_string())
- .or_insert_with(|| json!(seq));
+ o.entry("_seq_no".to_string()).or_insert_with(|| json!(seq));
}
}
v
@@ -287,9 +292,21 @@ struct SubPlan {
}
const PIPELINE_TYPES: &[&str] = &[
- "avg_bucket", "max_bucket", "min_bucket", "sum_bucket", "derivative", "moving_avg",
- "cumulative_sum", "bucket_selector", "bucket_sort", "bucket_script", "percentiles_bucket",
- "stats_bucket", "serial_diff", "moving_fn", "extended_stats_bucket",
+ "avg_bucket",
+ "max_bucket",
+ "min_bucket",
+ "sum_bucket",
+ "derivative",
+ "moving_avg",
+ "cumulative_sum",
+ "bucket_selector",
+ "bucket_sort",
+ "bucket_script",
+ "percentiles_bucket",
+ "stats_bucket",
+ "serial_diff",
+ "moving_fn",
+ "extended_stats_bucket",
];
fn metric_kind_of(t: &str) -> Option<MetricKind> {
@@ -494,11 +511,16 @@ impl<'a> FastCtx<'a> {
let seg = &self.segs[si];
let stored = self.idx.stored_values_for(&seg.id)?;
let d = stored.get(row as usize)?;
- let id = d.get("_id").and_then(Value::as_str).unwrap_or("").to_string();
+ let id = d
+ .get("_id")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
let mut src = d.get("_source").cloned().unwrap_or_else(|| d.clone());
if let Some(o) = src.as_object_mut() {
if !id.is_empty() {
- o.entry("_id".to_string()).or_insert(Value::String(id.clone()));
+ o.entry("_id".to_string())
+ .or_insert(Value::String(id.clone()));
}
o.entry("_index".to_string())
.or_insert_with(|| Value::String(self.idx.name.to_string()));
@@ -529,7 +551,9 @@ impl<'a> FastCtx<'a> {
top_hits: None,
pipelines: Vec::new(),
};
- let Some(sub) = sub_aggs else { return Some(plan) };
+ let Some(sub) = sub_aggs else {
+ return Some(plan);
+ };
let sub_obj = sub.as_object()?;
for (name, body) in sub_obj {
let (t, params, nested_sub, meta) = split_agg_body(body)?;
@@ -576,12 +600,17 @@ impl<'a> FastCtx<'a> {
}
fn mem_field_numeric_safe(&self, field: &str) -> bool {
- self.mem().iter().all(|d| {
- matches!(get_nested_field(d, field), Value::Number(_) | Value::Null)
- })
+ self.mem()
+ .iter()
+ .all(|d| matches!(get_nested_field(d, field), Value::Number(_) | Value::Null))
}
- fn plan_top_hits(&self, name: &str, params: &Value, meta: Option<&Value>) -> Option<TopHitsSpec> {
+ fn plan_top_hits(
+ &self,
+ name: &str,
+ params: &Value,
+ meta: Option<&Value>,
+ ) -> Option<TopHitsSpec> {
if !params_only(params, &["size", "from", "sort", "_source"]) {
return None;
}
@@ -614,7 +643,10 @@ impl<'a> FastCtx<'a> {
if !o.keys().all(|k| k == "order") {
return None; // mode / nested / missing → bail
}
- o.get("order").and_then(Value::as_str).map(|v| v == "desc").unwrap_or(false)
+ o.get("order")
+ .and_then(Value::as_str)
+ .map(|v| v == "desc")
+ .unwrap_or(false)
}
_ => return None,
};
@@ -632,7 +664,7 @@ impl<'a> FastCtx<'a> {
sort_field: field.clone(),
desc,
k,
- meta: meta.map(Value::clone),
+ meta: meta.cloned(),
})
}
@@ -711,7 +743,10 @@ impl<'a> FastCtx<'a> {
for (mi, spec) in plan.metrics.iter().enumerate() {
let mut v = Self::emit_metric(spec.kind, &accs[mi]);
if let Some(o) = v.as_object_mut() {
- o.insert("__type__".into(), Value::String(metric_type_name(spec.kind).to_string()));
+ o.insert(
+ "__type__".into(),
+ Value::String(metric_type_name(spec.kind).to_string()),
+ );
if let Some(m) = &spec.meta {
o.insert("meta".into(), m.clone());
}
@@ -780,7 +815,7 @@ impl<'a> FastCtx<'a> {
/// `run_aggs_with_all` global bucket: `{doc_count, <subs>}`.
fn exec_global(&self, params: &Value, sub: Option<&Value>) -> Option<Value> {
// `global` takes no params.
- if !params.as_object().map_or(false, |o| o.is_empty()) {
+ if !params.as_object().is_some_and(|o| o.is_empty()) {
return None;
}
let child = FastCtx {
@@ -1015,13 +1050,13 @@ impl<'a> FastCtx<'a> {
// `contains()` entirely, which otherwise dominates this loop.
let dense_num: Vec<bool> = num_cols
.iter()
- .map(|c| c.map_or(false, |n| n.null_bitmap.is_empty()))
+ .map(|c| c.is_some_and(|n| n.null_bitmap.is_empty()))
.collect();
let dense_kw: Vec<bool> = kw_cols
.iter()
- .map(|c| c.map_or(false, |k| k.null_bitmap.is_empty()))
+ .map(|c| c.is_some_and(|k| k.null_bitmap.is_empty()))
.collect();
- let th_dense = th_col.map_or(false, |n| n.null_bitmap.is_empty());
+ let th_dense = th_col.is_some_and(|n| n.null_bitmap.is_empty());
for row in 0..seg.docs {
if let Some(sp) = &top_sp {
@@ -1029,7 +1064,9 @@ impl<'a> FastCtx<'a> {
continue;
}
}
- let Some(slot) = slot_of_row(row) else { continue };
+ let Some(slot) = slot_of_row(row) else {
+ continue;
+ };
for (mi, m) in plan.metrics.iter().enumerate() {
match m.kind {
MetricKind::ValueCount => {
@@ -1090,7 +1127,11 @@ impl<'a> FastCtx<'a> {
.iter()
.map(|(_, _, r)| self.fetch_doc(*r))
.collect::<Option<Vec<_>>>()?;
- Some(run_top_hits_with_total(&spec.params, &docs, Some(bucket_total)))
+ Some(run_top_hits_with_total(
+ &spec.params,
+ &docs,
+ Some(bucket_total),
+ ))
}
// ── terms ────────────────────────────────────────────────────────────
@@ -1128,16 +1169,14 @@ impl<'a> FastCtx<'a> {
_ => return None,
}
// Same `missing` placeholder semantics as `run_cardinality`.
- let missing_placeholder: Option<String> =
- params.get("missing").and_then(|v| match v {
- Value::String(s) => Some(s.clone()),
- Value::Number(n) => Some(n.to_string()),
- Value::Bool(b) => Some(b.to_string()),
- _ => None,
- });
+ let missing_placeholder: Option<String> = params.get("missing").and_then(|v| match v {
+ Value::String(s) => Some(s.clone()),
+ Value::Number(n) => Some(n.to_string()),
+ Value::Bool(b) => Some(b.to_string()),
+ _ => None,
+ });
- let mut distinct: std::collections::HashSet<String> =
- std::collections::HashSet::new();
+ let mut distinct: std::collections::HashSet<String> = std::collections::HashSet::new();
const BOOL_TERMS: [&str; 2] = ["false", "true"];
for seg in &self.segs {
match seg.cols.get(field) {
@@ -1148,7 +1187,7 @@ impl<'a> FastCtx<'a> {
}
}
if let Some(ph) = &missing_placeholder {
- if k.null_bitmap.len() > 0 {
+ if !k.null_bitmap.is_empty() {
distinct.insert(ph.clone());
}
}
@@ -1168,7 +1207,7 @@ impl<'a> FastCtx<'a> {
distinct.insert(BOOL_TERMS[1].to_string());
}
if let Some(ph) = &missing_placeholder {
- if n.null_bitmap.len() > 0 {
+ if !n.null_bitmap.is_empty() {
distinct.insert(ph.clone());
}
}
@@ -1227,10 +1266,7 @@ impl<'a> FastCtx<'a> {
// Identical output shape to `run_cardinality` including the
// internal `__xy_*` fields the cross-index merge consumes.
- let values: Vec<Value> = distinct
- .iter()
- .map(|s| Value::String(s.clone()))
- .collect();
+ let values: Vec<Value> = distinct.iter().map(|s| Value::String(s.clone())).collect();
Some(json!({
"value": distinct.len(),
"__xy_agg__": "cardinality",
@@ -1239,12 +1275,18 @@ impl<'a> FastCtx<'a> {
}
fn exec_terms(&self, params: &Value, sub: Option<&Value>) -> Option<Value> {
- if !params_only(params, &["field", "size", "order", "shard_size", "min_doc_count"]) {
+ if !params_only(
+ params,
+ &["field", "size", "order", "shard_size", "min_doc_count"],
+ ) {
return None;
}
// Brute default & only supported min_doc_count is 1 (0 needs the
// background corpus, >1 is honored below since it's a plain filter).
- let min_doc_count = params.get("min_doc_count").and_then(Value::as_u64).unwrap_or(1);
+ let min_doc_count = params
+ .get("min_doc_count")
+ .and_then(Value::as_u64)
+ .unwrap_or(1);
if params.get("min_doc_count").is_some() && min_doc_count == 0 {
return None;
}
@@ -1265,7 +1307,10 @@ impl<'a> FastCtx<'a> {
if orders.iter().any(|(k, _)| k != "_count" && k != "_key") {
return None;
}
- let size_opt: Option<usize> = params.get("size").and_then(Value::as_u64).map(|v| v as usize);
+ let size_opt: Option<usize> = params
+ .get("size")
+ .and_then(Value::as_u64)
+ .map(|v| v as usize);
let cap: Option<usize> = match size_opt {
Some(0) => None,
Some(n) => Some(n),
@@ -1304,7 +1349,9 @@ impl<'a> FastCtx<'a> {
for si in 0..self.segs.len() {
let seg = &self.segs[si];
- let Some(col) = seg.cols.get(field) else { continue };
+ let Some(col) = seg.cols.get(field) else {
+ continue;
+ };
let tcol = match col {
Column::Keyword(k) => TCol::Kw(k),
Column::Numeric(n) if is_bool => {
@@ -1468,7 +1515,14 @@ impl<'a> FastCtx<'a> {
}
if let Some(spec) = &plan.top_hits {
match extract_numeric(doc, &spec.sort_field) {
- Some(v) => push_top(&mut st.top, spec.k, spec.desc, v, di as u64, DocRef::Mem(di)),
+ Some(v) => push_top(
+ &mut st.top,
+ spec.k,
+ spec.desc,
+ v,
+ di as u64,
+ DocRef::Mem(di),
+ ),
None => missing_sort_seen = true,
}
}
@@ -1489,7 +1543,13 @@ impl<'a> FastCtx<'a> {
.into_iter()
.filter(|(_, st)| st.count >= min_doc_count)
.collect();
- candidates.sort_by(|a, b| cmp_by_orders(&(a.0.clone(), a.1.count), &(b.0.clone(), b.1.count), &orders));
+ candidates.sort_by(|a, b| {
+ cmp_by_orders(
+ &(a.0.clone(), a.1.count),
+ &(b.0.clone(), b.1.count),
+ &orders,
+ )
+ });
if let Some(n) = cap {
candidates.truncate(n);
}
@@ -1566,9 +1626,11 @@ impl<'a> FastCtx<'a> {
let mut accs: Vec<MetricAcc> = vec![MetricAcc::default(); plan.metrics.len()];
for si in 0..self.segs.len() {
let seg = &self.segs[si];
- let Some(Column::Numeric(n)) = seg.cols.get(field) else { continue };
+ let Some(Column::Numeric(n)) = seg.cols.get(field) else {
+ continue;
+ };
if plan.metrics.is_empty() {
- count += n.range_count(lo, hi, true, hi_incl) as u64;
+ count += n.range_count(lo, hi, true, hi_incl);
} else {
let rows = n.range_doc_ids(lo, hi, true, hi_incl);
count += rows.len() as u64;
@@ -1600,7 +1662,9 @@ impl<'a> FastCtx<'a> {
}
}
for doc in self.mem().iter() {
- let Some(v) = extract_numeric(doc, field) else { continue };
+ let Some(v) = extract_numeric(doc, field) else {
+ continue;
+ };
let matches = match (from, to) {
(Some(f), Some(t)) => v >= f && v < t,
(Some(f), None) => v >= f,
@@ -1621,7 +1685,9 @@ impl<'a> FastCtx<'a> {
.and_then(Value::as_str)
.map(String::from)
.unwrap_or_else(|| match (from, to) {
- (Some(f), Some(t)) => format!("{}-{}", format_range_val(f), format_range_val(t)),
+ (Some(f), Some(t)) => {
+ format!("{}-{}", format_range_val(f), format_range_val(t))
+ }
(Some(f), None) => format!("{}-*", format_range_val(f)),
(None, Some(t)) => format!("*-{}", format_range_val(t)),
(None, None) => "*-*".to_string(),
@@ -1640,11 +1706,18 @@ impl<'a> FastCtx<'a> {
}
let buckets = apply_bucket_pipeline_ops(buckets, sub);
- let keyed = params.get("keyed").and_then(Value::as_bool).unwrap_or(false);
+ let keyed = params
+ .get("keyed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
if keyed {
let mut keyed_map = Map::new();
for mut b in buckets {
- let k = b.get("key").and_then(Value::as_str).unwrap_or("").to_string();
+ let k = b
+ .get("key")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
if let Some(obj) = b.as_object_mut() {
obj.remove("key");
}
@@ -1664,7 +1737,10 @@ impl<'a> FastCtx<'a> {
if !params_only(params, &["field", "ranges", "keyed"]) {
return None;
}
- let field = params.get("field").and_then(Value::as_str).unwrap_or("@timestamp");
+ let field = params
+ .get("field")
+ .and_then(Value::as_str)
+ .unwrap_or("@timestamp");
let ranges = params.get("ranges").and_then(Value::as_array)?;
match self.seg_field_kind(field) {
Ok(Some(ColKind::Keyword)) | Ok(None) => {}
@@ -1675,8 +1751,16 @@ impl<'a> FastCtx<'a> {
let nanos_digits: usize = ranges
.iter()
.flat_map(|r| {
- let f = r.get("from").and_then(Value::as_str).map(detect_fractional_digits).unwrap_or(0);
- let t = r.get("to").and_then(Value::as_str).map(detect_fractional_digits).unwrap_or(0);
+ let f = r
+ .get("from")
+ .and_then(Value::as_str)
+ .map(detect_fractional_digits)
+ .unwrap_or(0);
+ let t = r
+ .get("to")
+ .and_then(Value::as_str)
+ .map(detect_fractional_digits)
+ .unwrap_or(0);
[f, t]
})
.max()
@@ -1708,7 +1792,10 @@ impl<'a> FastCtx<'a> {
}
}
- let keyed = params.get("keyed").and_then(Value::as_bool).unwrap_or(false);
+ let keyed = params
+ .get("keyed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
let mut keyed_map = Map::new();
let mut buckets: Vec<Value> = Vec::new();
for range_def in ranges {
@@ -1731,7 +1818,9 @@ impl<'a> FastCtx<'a> {
let mut count: u64 = 0;
for (si, s) in self.segs.iter().enumerate() {
- let Some(Column::Keyword(_)) = s.cols.get(field) else { continue };
+ let Some(Column::Keyword(_)) = s.cols.get(field) else {
+ continue;
+ };
let sp = sorted_per_seg[si].as_ref()?;
let (ms_v, prefix) = (&sp.0, &sp.1);
// Semantics identical to the old per-ord walk: from is
@@ -1751,7 +1840,9 @@ impl<'a> FastCtx<'a> {
}
for doc in self.mem().iter() {
let v = doc.get(field);
- let Some(ms) = v.and_then(parse_date_ms) else { continue };
+ let Some(ms) = v.and_then(parse_date_ms) else {
+ continue;
+ };
let pass_from = from_ms.map(|f| ms >= f).unwrap_or(true);
let pass_to = to_ms.map(|t| ms < t).unwrap_or(true);
if pass_from && pass_to {
@@ -1817,7 +1908,9 @@ impl<'a> FastCtx<'a> {
ord_ms.push(parse_date_ms(&Value::String(t.clone())).unwrap_or(i64::MIN));
}
let arc = std::sync::Arc::new(ord_ms);
- self.idx.fast_date_cache.insert(key, std::sync::Arc::clone(&arc));
+ self.idx
+ .fast_date_cache
+ .insert(key, std::sync::Arc::clone(&arc));
Some(arc)
}
@@ -1926,7 +2019,11 @@ impl<'a> FastCtx<'a> {
_ => return None,
};
let named = matches!(filters_val, Value::Object(_));
- let keyed = if named { keyed_override.unwrap_or(true) } else { false };
+ let keyed = if named {
+ keyed_override.unwrap_or(true)
+ } else {
+ false
+ };
let mut rendered: Vec<(Option<String>, Value)> = Vec::with_capacity(entries.len());
for (name, q) in &entries {
@@ -2261,7 +2358,10 @@ impl<'a> FastCtx<'a> {
// ── sampler / random_sampler ─────────────────────────────────────────
fn exec_sampler(&self, params: &Value, sub: Option<&Value>) -> Option<Value> {
- if !params_only(params, &["shard_size", "probability", "seed", "max_docs_per_value"]) {
+ if !params_only(
+ params,
+ &["shard_size", "probability", "seed", "max_docs_per_value"],
+ ) {
return None;
}
if let Some(s) = sub {
@@ -2269,7 +2369,10 @@ impl<'a> FastCtx<'a> {
return None;
}
}
- let shard_size = params.get("shard_size").and_then(Value::as_u64).unwrap_or(200) as usize;
+ let shard_size = params
+ .get("shard_size")
+ .and_then(Value::as_u64)
+ .unwrap_or(200) as usize;
if shard_size > 10_000 {
return None; // keep the materialisation bounded
}
@@ -2328,7 +2431,9 @@ impl<'a> FastCtx<'a> {
// corpus paid ~300 ms of SipHash there), no global re-sort.
let mut runs: Vec<Vec<(f64, u64)>> = Vec::with_capacity(self.segs.len() + 1);
for seg in &self.segs {
- let Some(Column::Numeric(n)) = seg.cols.get(field) else { continue };
+ let Some(Column::Numeric(n)) = seg.cols.get(field) else {
+ continue;
+ };
let mut run: Vec<(f64, u64)> = Vec::new();
let mut i = 0usize;
while i < n.sorted.len() {
@@ -2400,7 +2505,15 @@ impl<'a> FastCtx<'a> {
fn exec_date_histogram(&self, params: &Value, sub: Option<&Value>) -> Option<Value> {
if !params_only(
params,
- &["field", "calendar_interval", "fixed_interval", "interval", "format", "keyed", "min_doc_count"],
+ &[
+ "field",
+ "calendar_interval",
+ "fixed_interval",
+ "interval",
+ "format",
+ "keyed",
+ "min_doc_count",
+ ],
) {
return None;
}
@@ -2430,7 +2543,11 @@ impl<'a> FastCtx<'a> {
} else {
key + interval_ms
};
- if next <= key { key + interval_ms } else { next }
+ if next <= key {
+ key + interval_ms
+ } else {
+ next
+ }
};
// Parallel pre-warm of the per-segment date-ordinal cache (chrono
@@ -2451,26 +2568,28 @@ impl<'a> FastCtx<'a> {
let mut bucket_keys: Vec<i64> = Vec::new();
let mut counts: Vec<u64> = Vec::new();
let mut accs: Vec<Vec<MetricAcc>> = plan.metrics.iter().map(|_| Vec::new()).collect();
- let mut ensure_bucket =
- |key: i64,
- bucket_ids: &mut HashMap<i64, usize>,
- bucket_keys: &mut Vec<i64>,
- counts: &mut Vec<u64>,
- accs: &mut Vec<Vec<MetricAcc>>| -> usize {
- *bucket_ids.entry(key).or_insert_with(|| {
- bucket_keys.push(key);
- counts.push(0);
- for a in accs.iter_mut() {
- a.push(MetricAcc::default());
- }
- bucket_keys.len() - 1
- })
- };
+ let ensure_bucket = |key: i64,
+ bucket_ids: &mut HashMap<i64, usize>,
+ bucket_keys: &mut Vec<i64>,
+ counts: &mut Vec<u64>,
+ accs: &mut Vec<Vec<MetricAcc>>|
+ -> usize {
+ *bucket_ids.entry(key).or_insert_with(|| {
+ bucket_keys.push(key);
+ counts.push(0);
+ for a in accs.iter_mut() {
+ a.push(MetricAcc::default());
+ }
+ bucket_keys.len() - 1
+ })
+ };
let mut missing_sort_seen = false;
for si in 0..self.segs.len() {
let seg = &self.segs[si];
- let Some(Column::Keyword(k)) = seg.cols.get(field) else { continue };
+ let Some(Column::Keyword(k)) = seg.cols.get(field) else {
+ continue;
+ };
let ord_ms = self.date_ord_index(seg, field)?;
// ord → bucket slot table. Term ordinals are lexicographic and
// ISO-8601 timestamps sort chronologically, so `ord_ms` is
@@ -2555,7 +2674,13 @@ impl<'a> FastCtx<'a> {
continue;
}
seen.push(key);
- let slot = ensure_bucket(key, &mut bucket_ids, &mut bucket_keys, &mut counts, &mut accs);
+ let slot = ensure_bucket(
+ key,
+ &mut bucket_ids,
+ &mut bucket_keys,
+ &mut counts,
+ &mut accs,
+ );
counts[slot] += weight;
for (mi, spec) in plan.metrics.iter().enumerate() {
Self::fold_mem_metric(doc, spec, &mut accs[mi][slot]);
@@ -2563,8 +2688,14 @@ impl<'a> FastCtx<'a> {
}
}
- let min_doc_count = params.get("min_doc_count").and_then(Value::as_u64).unwrap_or(0);
- let keyed = params.get("keyed").and_then(Value::as_bool).unwrap_or(false);
+ let min_doc_count = params
+ .get("min_doc_count")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
+ let keyed = params
+ .get("keyed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
// Bucket-key set: gap-fill when min_doc_count == 0 (brute default).
const MAX_BUCKETS: i64 = 65_536;
@@ -2623,7 +2754,12 @@ impl<'a> FastCtx<'a> {
bucket.insert("key_as_string".to_string(), json!(render_key(key)));
bucket.insert("doc_count".to_string(), json!(count));
let bucket_accs: Vec<MetricAcc> = match slot {
- Some(s) => plan.metrics.iter().enumerate().map(|(mi, _)| accs[mi][s]).collect(),
+ Some(s) => plan
+ .metrics
+ .iter()
+ .enumerate()
+ .map(|(mi, _)| accs[mi][s])
+ .collect(),
None => empty_accs.clone(),
};
self.finish_bucket(&mut bucket, &plan, &bucket_accs, None);
@@ -2687,7 +2823,7 @@ fn push_top(buf: &mut Vec<(f64, u64, DocRef)>, k: usize, desc: bool, v: f64, ran
}
}
-fn sort_top(buf: &mut Vec<(f64, u64, DocRef)>, desc: bool) {
+fn sort_top(buf: &mut [(f64, u64, DocRef)], desc: bool) {
buf.sort_by(|a, b| {
let c = if desc {
b.0.partial_cmp(&a.0)
@@ -2730,7 +2866,11 @@ fn parse_orders(params: &Value) -> Vec<(String, bool)> {
}
/// `cmp_terms_by_orders` for (key, count) with no sub-agg entries.
-fn cmp_by_orders(a: &(String, u64), b: &(String, u64), orders: &[(String, bool)]) -> std::cmp::Ordering {
+fn cmp_by_orders(
+ a: &(String, u64),
+ b: &(String, u64),
+ orders: &[(String, bool)],
+) -> std::cmp::Ordering {
use std::cmp::Ordering;
if orders.is_empty() {
return b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0));
@@ -2772,8 +2912,10 @@ fn composite_cmp(a: &[String], b: &[String]) -> std::cmp::Ordering {
fn sub_tree_needs_background(sub: &Value) -> bool {
match sub {
Value::Object(o) => o.iter().any(|(k, v)| {
- matches!(k.as_str(), "significant_terms" | "significant_text" | "global")
- || (k == "min_doc_count" && v.as_u64() == Some(0))
+ matches!(
+ k.as_str(),
+ "significant_terms" | "significant_text" | "global"
+ ) || (k == "min_doc_count" && v.as_u64() == Some(0))
|| sub_tree_needs_background(v)
}),
Value::Array(a) => a.iter().any(sub_tree_needs_background),
@@ -2785,9 +2927,21 @@ fn sub_tree_needs_background(sub: &Value) -> bool {
enum Pred {
MatchAll,
- TermKw { field: String, value: String },
- TermsKw { field: String, values: Vec<String> },
- RangeNum { field: String, lo: f64, lo_incl: bool, hi: f64, hi_incl: bool },
+ TermKw {
+ field: String,
+ value: String,
+ },
+ TermsKw {
+ field: String,
+ values: Vec<String>,
+ },
+ RangeNum {
+ field: String,
+ lo: f64,
+ lo_incl: bool,
+ hi: f64,
+ hi_incl: bool,
+ },
/// Conjunction of leaf predicates — a `bool` with only `must`/`filter`
/// clauses (produced by `compile_top_pred` for the top-level query filter;
/// `compile_pred` never yields this, so the filter/filters/adjacency
@@ -2876,7 +3030,10 @@ fn compile_pred(filter: &Value) -> Option<Pred> {
},
_ => return None, // numeric/bool string-form subtleties → bail
};
- Some(Pred::TermKw { field: field.clone(), value })
+ Some(Pred::TermKw {
+ field: field.clone(),
+ value,
+ })
}
"terms" => {
let fm = body.as_object()?;
@@ -2892,7 +3049,10 @@ fn compile_pred(filter: &Value) -> Option<Pred> {
_ => return None,
}
}
- Some(Pred::TermsKw { field: field.clone(), values })
+ Some(Pred::TermsKw {
+ field: field.clone(),
+ values,
+ })
}
"range" => {
let fm = body.as_object()?;
@@ -2901,7 +3061,10 @@ fn compile_pred(filter: &Value) -> Option<Pred> {
}
let (field, bounds) = fm.iter().next()?;
let bo = bounds.as_object()?;
- if !bo.keys().all(|k| matches!(k.as_str(), "gte" | "gt" | "lte" | "lt")) {
+ if !bo
+ .keys()
+ .all(|k| matches!(k.as_str(), "gte" | "gt" | "lte" | "lt"))
+ {
return None;
}
let get_num = |k: &str| -> Option<Option<f64>> {
@@ -2925,7 +3088,13 @@ fn compile_pred(filter: &Value) -> Option<Pred> {
(None, Some(v)) => (v, false),
(None, None) => (f64::INFINITY, true),
};
- Some(Pred::RangeNum { field: field.clone(), lo, lo_incl, hi, hi_incl })
+ Some(Pred::RangeNum {
+ field: field.clone(),
+ lo,
+ lo_incl,
+ hi,
+ hi_incl,
+ })
}
_ => None,
}
@@ -2968,7 +3137,13 @@ fn resolve_pred<'a>(
Some(Column::Numeric(_)) => return None,
None => SegPred::Never,
},
- Pred::RangeNum { field, lo, lo_incl, hi, hi_incl } => match cols.get(field) {
+ Pred::RangeNum {
+ field,
+ lo,
+ lo_incl,
+ hi,
+ hi_incl,
+ } => match cols.get(field) {
Some(Column::Numeric(n)) => {
SegPred::Num(n, *lo, *lo_incl, *hi, *hi_incl, n.null_bitmap.is_empty())
}
@@ -3034,7 +3209,7 @@ fn seg_pred_count(sp: &SegPred<'_>, docs: u32) -> u64 {
.iter()
.map(|o| k.per_ord_count.get(*o as usize).copied().unwrap_or(0) as u64)
.sum(),
- SegPred::Num(n, lo, lo_incl, hi, hi_incl, _) => n.range_count(*lo, *hi, *lo_incl, *hi_incl) as u64,
+ SegPred::Num(n, lo, lo_incl, hi, hi_incl, _) => n.range_count(*lo, *hi, *lo_incl, *hi_incl),
// Conjunction has no O(1) form — count matching rows directly. Only
// reached from the top-level filter count (compile_pred, used by the
// filter/filters executors, never yields `And`).
@@ -3072,7 +3247,11 @@ fn merge_weighted_runs(mut runs: Vec<Vec<(f64, u64)>>) -> Vec<(f64, u64)> {
// from finite doc values, so the sign-flip trick keeps ordering exact).
fn key(v: f64) -> u64 {
let b = v.to_bits();
- if b >> 63 == 1 { !b } else { b | (1 << 63) }
+ if b >> 63 == 1 {
+ !b
+ } else {
+ b | (1 << 63)
+ }
}
let total: usize = runs.iter().map(Vec::len).sum();
let mut out: Vec<(f64, u64)> = Vec::with_capacity(total);
@@ -3135,15 +3314,30 @@ fn vwh_cluster(vals: &[(f64, u64)], num_buckets: usize) -> Vec<VwhBucket> {
for &(v, w) in vals {
if merges_needed >= w - 1 {
merges_needed -= w - 1;
- out.push(VwhBucket { min: v, max: v, sum: v * w as f64, count: w });
+ out.push(VwhBucket {
+ min: v,
+ max: v,
+ sum: v * w as f64,
+ count: w,
+ });
} else {
// Partially collapsed run: one merged bucket of
// (merges_needed + 1) instances, then singletons.
let m = merges_needed;
merges_needed = 0;
- out.push(VwhBucket { min: v, max: v, sum: v * (m + 1) as f64, count: m + 1 });
+ out.push(VwhBucket {
+ min: v,
+ max: v,
+ sum: v * (m + 1) as f64,
+ count: m + 1,
+ });
for _ in 0..(w - 1 - m) {
- out.push(VwhBucket { min: v, max: v, sum: v, count: 1 });
+ out.push(VwhBucket {
+ min: v,
+ max: v,
+ sum: v,
+ count: 1,
+ });
}
}
}
@@ -3167,7 +3361,11 @@ fn vwh_cluster(vals: &[(f64, u64)], num_buckets: usize) -> Vec<VwhBucket> {
use std::cmp::Reverse;
use std::collections::BinaryHeap;
#[derive(PartialEq, Eq, PartialOrd, Ord)]
- struct Entry(u64 /*gap bits (total_cmp key)*/, usize /*left idx*/, u64 /*gen sum*/);
+ struct Entry(
+ u64, /*gap bits (total_cmp key)*/
+ usize, /*left idx*/
+ u64, /*gen sum*/
+ );
let gap_key = |g: f64| -> u64 {
// total_cmp-compatible ordering for non-negative floats.
g.to_bits()
@@ -3179,7 +3377,9 @@ fn vwh_cluster(vals: &[(f64, u64)], num_buckets: usize) -> Vec<VwhBucket> {
heap.push(Reverse(Entry(gap_key(g), i, 0)));
}
while remaining > 0 {
- let Some(Reverse(Entry(_, left, gsum))) = heap.pop() else { break };
+ let Some(Reverse(Entry(_, left, gsum))) = heap.pop() else {
+ break;
+ };
if !alive[left] {
continue;
}
@@ -3225,7 +3425,12 @@ fn vwh_cluster(vals: &[(f64, u64)], num_buckets: usize) -> Vec<VwhBucket> {
i += 1;
}
while i < d {
- out.push(VwhBucket { min: mn[i], max: mx[i], sum: sum[i], count: cnt[i] });
+ out.push(VwhBucket {
+ min: mn[i],
+ max: mx[i],
+ sum: sum[i],
+ count: cnt[i],
+ });
i = next[i];
while i < d && !alive[i] {
i += 1;
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 46426dad2..1038ed23b 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -8,6 +8,14 @@
//! - [`FtsMemtable`] — in-memory inverted index for unflushed documents
//! - [`bulk`] — NDJSON bulk operation processing
+// The engine wires aggregation, segment, and memtable pipelines whose closures
+// and columnar accumulators carry deeply-nested generic types (e.g.
+// `Vec<(String, HashMap<String, String>, Arc<Value>)>`). Naming each with a
+// type alias hurts more than it helps at these call sites, so we opt out of
+// `clippy::type_complexity` crate-wide — the standard idiom for data-plane
+// internals.
+#![allow(clippy::type_complexity)]
+
pub mod aggs;
pub mod audit;
pub mod bulk;
@@ -23,7 +31,10 @@ pub mod turbo_ingest;
// ── Re-exports ────────────────────────────────────────────────────────────────
pub use engine::{Engine, HealthStatus, IndexInfo};
-pub use index::{Index, IndexResponse, IndexStats, FieldEncodingInfo, LogFormat, EnrichTable, detect_log_format, resolve_field_alias, resolve_date_math};
+pub use index::{
+ detect_log_format, resolve_date_math, resolve_field_alias, EnrichTable, FieldEncodingInfo,
+ Index, IndexResponse, IndexStats, LogFormat,
+};
pub use memtable::FtsMemtable;
// ── Ingest/flush/merge rayon pool ────────────────────────────────────────────
@@ -181,8 +192,8 @@ pub(crate) fn flush_finalize_gate() -> &'static tokio::sync::Semaphore {
use std::sync::Arc;
use xerj_fts::analyzer::{
- AnalyzerRegistry, LowercaseFilter, StemmerFilter, StopwordsFilter,
- StandardTokenizer, Tokenizer, TokenFilter,
+ AnalyzerRegistry, LowercaseFilter, StandardTokenizer, StemmerFilter, StopwordsFilter,
+ TokenFilter, Tokenizer,
};
/// Return a default [`AnalyzerRegistry`] pre-populated with all built-in analyzers.
diff --git a/engine/crates/xerj-engine/src/painless.rs b/engine/crates/xerj-engine/src/painless.rs
index 441f222d5..f724a14d9 100644
--- a/engine/crates/xerj-engine/src/painless.rs
+++ b/engine/crates/xerj-engine/src/painless.rs
@@ -75,7 +75,9 @@ impl PainlessValue {
Value::Bool(b) => PainlessValue::Bool(*b),
Value::Number(n) => PainlessValue::Number(n.as_f64().unwrap_or(0.0)),
Value::String(s) => PainlessValue::String(s.clone()),
- Value::Array(arr) => PainlessValue::Array(arr.iter().map(PainlessValue::from_json).collect()),
+ Value::Array(arr) => {
+ PainlessValue::Array(arr.iter().map(PainlessValue::from_json).collect())
+ }
Value::Object(o) => PainlessValue::Object(o.clone()),
}
}
@@ -94,7 +96,12 @@ pub struct PainlessCtx<'a> {
impl<'a> PainlessCtx<'a> {
pub fn new(doc: &'a Value, params: &'a Value, score: f32) -> Self {
- Self { doc, params, score, emits: std::cell::RefCell::new(Vec::new()) }
+ Self {
+ doc,
+ params,
+ score,
+ emits: std::cell::RefCell::new(Vec::new()),
+ }
}
pub fn take_emits(&self) -> Vec<PainlessValue> {
std::mem::take(&mut *self.emits.borrow_mut())
@@ -120,28 +127,47 @@ fn tokenize(src: &str) -> Result<Vec<Tok>, String> {
while i < bytes.len() {
let c = bytes[i] as char;
// Skip whitespace
- if c.is_whitespace() { i += 1; continue; }
+ if c.is_whitespace() {
+ i += 1;
+ continue;
+ }
// Comments
if c == '/' && i + 1 < bytes.len() && bytes[i + 1] as char == '/' {
- while i < bytes.len() && bytes[i] as char != '\n' { i += 1; }
+ while i < bytes.len() && bytes[i] as char != '\n' {
+ i += 1;
+ }
continue;
}
if c == '/' && i + 1 < bytes.len() && bytes[i + 1] as char == '*' {
i += 2;
- while i + 1 < bytes.len() && !(bytes[i] as char == '*' && bytes[i + 1] as char == '/') { i += 1; }
+ while i + 1 < bytes.len() && !(bytes[i] as char == '*' && bytes[i + 1] as char == '/') {
+ i += 1;
+ }
i += 2;
continue;
}
// Number literal
- if c.is_ascii_digit() || (c == '.' && i + 1 < bytes.len() && (bytes[i + 1] as char).is_ascii_digit()) {
+ if c.is_ascii_digit()
+ || (c == '.' && i + 1 < bytes.len() && (bytes[i + 1] as char).is_ascii_digit())
+ {
let start = i;
while i < bytes.len() {
let cc = bytes[i] as char;
- if cc.is_ascii_digit() || cc == '.' || cc == 'e' || cc == 'E' || cc == '-' || cc == '+' {
+ if cc.is_ascii_digit()
+ || cc == '.'
+ || cc == 'e'
+ || cc == 'E'
+ || cc == '-'
+ || cc == '+'
+ {
// Allow signed exponent
- if (cc == '-' || cc == '+') && !matches!(bytes[i - 1] as char, 'e' | 'E') { break; }
+ if (cc == '-' || cc == '+') && !matches!(bytes[i - 1] as char, 'e' | 'E') {
+ break;
+ }
i += 1;
- } else { break; }
+ } else {
+ break;
+ }
}
// Strip trailing 'L'/'F'/'D' type suffix Painless allows.
let s_end = i;
@@ -156,8 +182,12 @@ fn tokenize(src: &str) -> Result<Vec<Tok>, String> {
}
}
// strip possibly trailing "L" already in string for safety
- s_clean = s_clean.trim_end_matches(|c: char| matches!(c, 'L' | 'l' | 'F' | 'f' | 'D' | 'd')).to_string();
- let n: f64 = s_clean.parse().map_err(|e| format!("bad number {s_clean}: {e}"))?;
+ s_clean = s_clean
+ .trim_end_matches(['L', 'l', 'F', 'f', 'D', 'd'])
+ .to_string();
+ let n: f64 = s_clean
+ .parse()
+ .map_err(|e| format!("bad number {s_clean}: {e}"))?;
out.push(Tok::Number(n));
continue;
}
@@ -167,9 +197,15 @@ fn tokenize(src: &str) -> Result<Vec<Tok>, String> {
i += 1;
let start = i;
while i < bytes.len() && bytes[i] as char != quote {
- if bytes[i] as char == '\\' && i + 1 < bytes.len() { i += 2; } else { i += 1; }
+ if bytes[i] as char == '\\' && i + 1 < bytes.len() {
+ i += 2;
+ } else {
+ i += 1;
+ }
+ }
+ if i >= bytes.len() {
+ return Err("unterminated string".into());
}
- if i >= bytes.len() { return Err("unterminated string".into()); }
let raw = &src[start..i];
i += 1;
// Basic escape handling.
@@ -200,14 +236,19 @@ fn tokenize(src: &str) -> Result<Vec<Tok>, String> {
let start = i;
while i < bytes.len() {
let cc = bytes[i] as char;
- if cc.is_alphanumeric() || cc == '_' || cc == '$' { i += 1; } else { break; }
+ if cc.is_alphanumeric() || cc == '_' || cc == '$' {
+ i += 1;
+ } else {
+ break;
+ }
}
let s = &src[start..i];
match s {
- "if" | "else" | "return" | "true" | "false" | "null" |
- "double" | "int" | "long" | "float" | "boolean" | "String" |
- "def" | "var" | "for" | "while" | "break" | "continue" |
- "new" | "instanceof" => out.push(Tok::Keyword(s.to_string())),
+ "if" | "else" | "return" | "true" | "false" | "null" | "double" | "int"
+ | "long" | "float" | "boolean" | "String" | "def" | "var" | "for" | "while"
+ | "break" | "continue" | "new" | "instanceof" => {
+ out.push(Tok::Keyword(s.to_string()))
+ }
_ => out.push(Tok::Ident(s.to_string())),
}
continue;
@@ -215,14 +256,52 @@ fn tokenize(src: &str) -> Result<Vec<Tok>, String> {
// Multi-char punctuation
if i + 1 < bytes.len() {
let two: String = format!("{}{}", c, bytes[i + 1] as char);
- if matches!(two.as_str(), "==" | "!=" | "<=" | ">=" | "&&" | "||" | "->" | "+=" | "-=" | "*=" | "/=" | "%=" | "++" | "--") {
+ if matches!(
+ two.as_str(),
+ "==" | "!="
+ | "<="
+ | ">="
+ | "&&"
+ | "||"
+ | "->"
+ | "+="
+ | "-="
+ | "*="
+ | "/="
+ | "%="
+ | "++"
+ | "--"
+ ) {
out.push(Tok::PunctMulti(two));
i += 2;
continue;
}
}
// Single-char punctuation
- if matches!(c, '(' | ')' | '{' | '}' | '[' | ']' | ',' | ';' | '.' | ':' | '?' | '+' | '-' | '*' | '/' | '%' | '<' | '>' | '=' | '!' | '&' | '|') {
+ if matches!(
+ c,
+ '(' | ')'
+ | '{'
+ | '}'
+ | '['
+ | ']'
+ | ','
+ | ';'
+ | '.'
+ | ':'
+ | '?'
+ | '+'
+ | '-'
+ | '*'
+ | '/'
+ | '%'
+ | '<'
+ | '>'
+ | '='
+ | '!'
+ | '&'
+ | '|'
+ ) {
out.push(Tok::Punct(c));
i += 1;
continue;
@@ -270,9 +349,19 @@ struct Parser<'a> {
}
impl<'a> Parser<'a> {
- fn new(toks: &'a [Tok]) -> Self { Self { toks, pos: 0 } }
- fn peek(&self) -> Option<&Tok> { self.toks.get(self.pos) }
- fn eat(&mut self) -> Option<Tok> { let t = self.toks.get(self.pos).cloned(); if t.is_some() { self.pos += 1; } t }
+ fn new(toks: &'a [Tok]) -> Self {
+ Self { toks, pos: 0 }
+ }
+ fn peek(&self) -> Option<&Tok> {
+ self.toks.get(self.pos)
+ }
+ fn eat(&mut self) -> Option<Tok> {
+ let t = self.toks.get(self.pos).cloned();
+ if t.is_some() {
+ self.pos += 1;
+ }
+ t
+ }
fn expect_punct(&mut self, c: char) -> Result<(), String> {
match self.eat() {
Some(Tok::Punct(p)) if p == c => Ok(()),
@@ -281,13 +370,19 @@ impl<'a> Parser<'a> {
}
fn match_punct(&mut self, c: char) -> bool {
if let Some(Tok::Punct(p)) = self.peek() {
- if *p == c { self.pos += 1; return true; }
+ if *p == c {
+ self.pos += 1;
+ return true;
+ }
}
false
}
fn match_keyword(&mut self, kw: &str) -> bool {
if let Some(Tok::Keyword(s)) = self.peek() {
- if s == kw { self.pos += 1; return true; }
+ if s == kw {
+ self.pos += 1;
+ return true;
+ }
}
false
}
@@ -307,7 +402,9 @@ impl<'a> Parser<'a> {
let then_body = self.parse_block_or_stmt()?;
let else_body = if self.match_keyword("else") {
self.parse_block_or_stmt()?
- } else { Vec::new() };
+ } else {
+ Vec::new()
+ };
return Ok(Stmt::If(cond, then_body, else_body));
}
if self.match_keyword("return") {
@@ -327,7 +424,10 @@ impl<'a> Parser<'a> {
}
// Variable decl: `<type> NAME = expr;`
if let Some(Tok::Keyword(kw)) = self.peek().cloned() {
- if matches!(kw.as_str(), "double" | "int" | "long" | "float" | "boolean" | "String" | "def" | "var") {
+ if matches!(
+ kw.as_str(),
+ "double" | "int" | "long" | "float" | "boolean" | "String" | "def" | "var"
+ ) {
self.pos += 1;
let name = match self.eat() {
Some(Tok::Ident(n)) => n,
@@ -349,7 +449,9 @@ impl<'a> Parser<'a> {
if self.match_punct('{') {
let mut out = Vec::new();
while let Some(t) = self.peek() {
- if matches!(t, Tok::Punct('}')) { break; }
+ if matches!(t, Tok::Punct('}')) {
+ break;
+ }
out.push(self.parse_stmt()?);
}
self.expect_punct('}')?;
@@ -379,23 +481,37 @@ impl<'a> Parser<'a> {
let then_e = self.parse_assign()?;
self.expect_punct(':')?;
let else_e = self.parse_assign()?;
- return Ok(Expr::Ternary(Box::new(cond), Box::new(then_e), Box::new(else_e)));
+ return Ok(Expr::Ternary(
+ Box::new(cond),
+ Box::new(then_e),
+ Box::new(else_e),
+ ));
}
Ok(cond)
}
fn parse_or(&mut self) -> Result<Expr, String> {
let mut lhs = self.parse_and()?;
while let Some(Tok::PunctMulti(op)) = self.peek() {
- if op == "||" { self.pos += 1; let rhs = self.parse_and()?; lhs = Expr::Binary("||".into(), Box::new(lhs), Box::new(rhs)); }
- else { break; }
+ if op == "||" {
+ self.pos += 1;
+ let rhs = self.parse_and()?;
+ lhs = Expr::Binary("||".into(), Box::new(lhs), Box::new(rhs));
+ } else {
+ break;
+ }
}
Ok(lhs)
}
fn parse_and(&mut self) -> Result<Expr, String> {
let mut lhs = self.parse_eq()?;
while let Some(Tok::PunctMulti(op)) = self.peek() {
- if op == "&&" { self.pos += 1; let rhs = self.parse_eq()?; lhs = Expr::Binary("&&".into(), Box::new(lhs), Box::new(rhs)); }
- else { break; }
+ if op == "&&" {
+ self.pos += 1;
+ let rhs = self.parse_eq()?;
+ lhs = Expr::Binary("&&".into(), Box::new(lhs), Box::new(rhs));
+ } else {
+ break;
+ }
}
Ok(lhs)
}
@@ -490,18 +606,25 @@ impl<'a> Parser<'a> {
let idx = self.parse_expr()?;
self.expect_punct(']')?;
e = Expr::Index(Box::new(e), Box::new(idx));
- } else { break; }
+ } else {
+ break;
+ }
}
Ok(e)
}
fn parse_args(&mut self, end: char) -> Result<Vec<Expr>, String> {
let mut out: Vec<Expr> = Vec::new();
if let Some(Tok::Punct(c)) = self.peek() {
- if *c == end { self.pos += 1; return Ok(out); }
+ if *c == end {
+ self.pos += 1;
+ return Ok(out);
+ }
}
loop {
out.push(self.parse_expr()?);
- if self.match_punct(',') { continue; }
+ if self.match_punct(',') {
+ continue;
+ }
break;
}
match self.eat() {
@@ -548,16 +671,28 @@ pub fn eval_painless(src: &str, ctx: &PainlessCtx) -> Result<PainlessValue, Stri
let mut last: PainlessValue = PainlessValue::Null;
for stmt in &stmts {
match exec_stmt(stmt, ctx, &mut env)? {
- ExecOutcome::Return(v) => { ret = Some(v); break; }
- ExecOutcome::Value(v) => { last = v; }
+ ExecOutcome::Return(v) => {
+ ret = Some(v);
+ break;
+ }
+ ExecOutcome::Value(v) => {
+ last = v;
+ }
}
}
Ok(ret.unwrap_or(last))
}
-enum ExecOutcome { Return(PainlessValue), Value(PainlessValue) }
+enum ExecOutcome {
+ Return(PainlessValue),
+ Value(PainlessValue),
+}
-fn exec_stmt(s: &Stmt, ctx: &PainlessCtx, env: &mut HashMap<String, PainlessValue>) -> Result<ExecOutcome, String> {
+fn exec_stmt(
+ s: &Stmt,
+ ctx: &PainlessCtx,
+ env: &mut HashMap<String, PainlessValue>,
+) -> Result<ExecOutcome, String> {
match s {
Stmt::Return(opt) => {
let v = match opt {
@@ -590,14 +725,20 @@ fn exec_stmt(s: &Stmt, ctx: &PainlessCtx, env: &mut HashMap<String, PainlessValu
}
}
-fn eval_expr(e: &Expr, ctx: &PainlessCtx, env: &mut HashMap<String, PainlessValue>) -> Result<PainlessValue, String> {
+fn eval_expr(
+ e: &Expr,
+ ctx: &PainlessCtx,
+ env: &mut HashMap<String, PainlessValue>,
+) -> Result<PainlessValue, String> {
match e {
Expr::Number(n) => Ok(PainlessValue::Number(*n)),
Expr::String(s) => Ok(PainlessValue::String(s.clone())),
Expr::Bool(b) => Ok(PainlessValue::Bool(*b)),
Expr::Null => Ok(PainlessValue::Null),
Expr::Ident(name) => {
- if let Some(v) = env.get(name) { return Ok(v.clone()); }
+ if let Some(v) = env.get(name) {
+ return Ok(v.clone());
+ }
match name.as_str() {
"_score" => Ok(PainlessValue::Number(ctx.score as f64)),
"doc" => Ok(PainlessValue::Null), // marker; resolved via Member/Index
@@ -622,33 +763,38 @@ fn eval_expr(e: &Expr, ctx: &PainlessCtx, env: &mut HashMap<String, PainlessValu
// Short-circuit && ||
if op == "&&" {
let av = eval_expr(a, ctx, env)?;
- if !av.as_bool() { return Ok(PainlessValue::Bool(false)); }
+ if !av.as_bool() {
+ return Ok(PainlessValue::Bool(false));
+ }
return Ok(PainlessValue::Bool(eval_expr(b, ctx, env)?.as_bool()));
}
if op == "||" {
let av = eval_expr(a, ctx, env)?;
- if av.as_bool() { return Ok(PainlessValue::Bool(true)); }
+ if av.as_bool() {
+ return Ok(PainlessValue::Bool(true));
+ }
return Ok(PainlessValue::Bool(eval_expr(b, ctx, env)?.as_bool()));
}
let av = eval_expr(a, ctx, env)?;
let bv = eval_expr(b, ctx, env)?;
// String concatenation for `+`.
- if op == "+" {
- if matches!(av, PainlessValue::String(_)) || matches!(bv, PainlessValue::String(_)) {
- let sa = match &av {
- PainlessValue::String(s) => s.clone(),
- PainlessValue::Number(n) => format_num(*n),
- PainlessValue::Bool(b) => b.to_string(),
- _ => "null".to_string(),
- };
- let sb = match &bv {
- PainlessValue::String(s) => s.clone(),
- PainlessValue::Number(n) => format_num(*n),
- PainlessValue::Bool(b) => b.to_string(),
- _ => "null".to_string(),
- };
- return Ok(PainlessValue::String(format!("{sa}{sb}")));
- }
+ if op == "+"
+ && (matches!(av, PainlessValue::String(_))
+ || matches!(bv, PainlessValue::String(_)))
+ {
+ let sa = match &av {
+ PainlessValue::String(s) => s.clone(),
+ PainlessValue::Number(n) => format_num(*n),
+ PainlessValue::Bool(b) => b.to_string(),
+ _ => "null".to_string(),
+ };
+ let sb = match &bv {
+ PainlessValue::String(s) => s.clone(),
+ PainlessValue::Number(n) => format_num(*n),
+ PainlessValue::Bool(b) => b.to_string(),
+ _ => "null".to_string(),
+ };
+ return Ok(PainlessValue::String(format!("{sa}{sb}")));
}
let an = av.as_f64().unwrap_or(0.0);
let bn = bv.as_f64().unwrap_or(0.0);
@@ -656,8 +802,20 @@ fn eval_expr(e: &Expr, ctx: &PainlessCtx, env: &mut HashMap<String, PainlessValu
"+" => an + bn,
"-" => an - bn,
"*" => an * bn,
- "/" => if bn == 0.0 { f64::NAN } else { an / bn },
- "%" => if bn == 0.0 { f64::NAN } else { an % bn },
+ "/" => {
+ if bn == 0.0 {
+ f64::NAN
+ } else {
+ an / bn
+ }
+ }
+ "%" => {
+ if bn == 0.0 {
+ f64::NAN
+ } else {
+ an % bn
+ }
+ }
"<" => return Ok(PainlessValue::Bool(an < bn)),
"<=" => return Ok(PainlessValue::Bool(an <= bn)),
">" => return Ok(PainlessValue::Bool(an > bn)),
@@ -670,7 +828,11 @@ fn eval_expr(e: &Expr, ctx: &PainlessCtx, env: &mut HashMap<String, PainlessValu
}
Expr::Ternary(c, t, f) => {
let cv = eval_expr(c, ctx, env)?;
- if cv.as_bool() { eval_expr(t, ctx, env) } else { eval_expr(f, ctx, env) }
+ if cv.as_bool() {
+ eval_expr(t, ctx, env)
+ } else {
+ eval_expr(f, ctx, env)
+ }
}
Expr::Index(base, idx) => {
// Special-case `doc['field']` / `params['x']`.
@@ -724,7 +886,10 @@ fn eval_expr(e: &Expr, ctx: &PainlessCtx, env: &mut HashMap<String, PainlessValu
}
if name == "Math" {
let argvs: Vec<PainlessValue> = match args {
- Some(args) => args.iter().map(|a| eval_expr(a, ctx, env)).collect::<Result<_, _>>()?,
+ Some(args) => args
+ .iter()
+ .map(|a| eval_expr(a, ctx, env))
+ .collect::<Result<_, _>>()?,
None => Vec::new(),
};
return math_call(member, &argvs);
@@ -775,7 +940,10 @@ fn eval_expr(e: &Expr, ctx: &PainlessCtx, env: &mut HashMap<String, PainlessValu
Err(format!("unsupported member access .{}", member))
}
Expr::Call(name, args) => {
- let argvs: Vec<PainlessValue> = args.iter().map(|a| eval_expr(a, ctx, env)).collect::<Result<_, _>>()?;
+ let argvs: Vec<PainlessValue> = args
+ .iter()
+ .map(|a| eval_expr(a, ctx, env))
+ .collect::<Result<_, _>>()?;
global_call(name, &argvs, ctx)
}
}
@@ -826,7 +994,10 @@ fn resolve_doc_member(
"value" => {
// Return first scalar.
match raw {
- Value::Array(arr) => Ok(arr.first().map(|v| PainlessValue::from_json(v)).unwrap_or(PainlessValue::Number(0.0))),
+ Value::Array(arr) => Ok(arr
+ .first()
+ .map(PainlessValue::from_json)
+ .unwrap_or(PainlessValue::Number(0.0))),
Value::Number(n) => Ok(PainlessValue::Number(n.as_f64().unwrap_or(0.0))),
Value::String(s) => Ok(PainlessValue::String(s)),
Value::Bool(b) => Ok(PainlessValue::Bool(b)),
@@ -873,7 +1044,10 @@ fn get_doc_value(doc: &Value, field: &str) -> Value {
for p in parts.iter() {
if let Value::Object(obj) = &sub {
sub = obj.get(*p).cloned().unwrap_or(Value::Null);
- } else { sub = Value::Null; break; }
+ } else {
+ sub = Value::Null;
+ break;
+ }
}
sub
})
@@ -907,7 +1081,11 @@ fn math_call(name: &str, args: &[PainlessValue]) -> Result<PainlessValue, String
Ok(PainlessValue::Number(r))
}
-fn global_call(name: &str, args: &[PainlessValue], ctx: &PainlessCtx) -> Result<PainlessValue, String> {
+fn global_call(
+ name: &str,
+ args: &[PainlessValue],
+ ctx: &PainlessCtx,
+) -> Result<PainlessValue, String> {
match name {
"emit" => {
// Runtime-field emit — records each call's value into the
@@ -924,7 +1102,9 @@ fn global_call(name: &str, args: &[PainlessValue], ctx: &PainlessCtx) -> Result<
return Err(format!("dotProduct expects 2 args, got {}", args.len()));
}
let query: Vec<f64> = match &args[0] {
- PainlessValue::Array(arr) => arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect(),
+ PainlessValue::Array(arr) => {
+ arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect()
+ }
_ => return Err("dotProduct arg 0 must be array".into()),
};
let doc_vec: Vec<f64> = match &args[1] {
@@ -936,19 +1116,29 @@ fn global_call(name: &str, args: &[PainlessValue], ctx: &PainlessCtx) -> Result<
_ => Vec::new(),
}
}
- PainlessValue::Array(arr) => arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect(),
+ PainlessValue::Array(arr) => {
+ arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect()
+ }
_ => return Err("dotProduct arg 1 must be field name or array".into()),
};
if query.len() != doc_vec.len() {
- return Err(format!("dim mismatch: {} vs {}", query.len(), doc_vec.len()));
+ return Err(format!(
+ "dim mismatch: {} vs {}",
+ query.len(),
+ doc_vec.len()
+ ));
}
let dot: f64 = query.iter().zip(doc_vec.iter()).map(|(a, b)| a * b).sum();
Ok(PainlessValue::Number(dot))
}
"cosineSimilarity" => {
- if args.len() != 2 { return Err("cosineSimilarity expects 2 args".into()); }
+ if args.len() != 2 {
+ return Err("cosineSimilarity expects 2 args".into());
+ }
let q: Vec<f64> = match &args[0] {
- PainlessValue::Array(arr) => arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect(),
+ PainlessValue::Array(arr) => {
+ arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect()
+ }
_ => return Err("cosineSimilarity arg 0 must be array".into()),
};
let d: Vec<f64> = match &args[1] {
@@ -959,20 +1149,32 @@ fn global_call(name: &str, args: &[PainlessValue], ctx: &PainlessCtx) -> Result<
_ => Vec::new(),
}
}
- PainlessValue::Array(arr) => arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect(),
+ PainlessValue::Array(arr) => {
+ arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect()
+ }
_ => return Err("cosineSimilarity arg 1 must be field name".into()),
};
- if q.len() != d.len() { return Err("dim mismatch".into()); }
+ if q.len() != d.len() {
+ return Err("dim mismatch".into());
+ }
let dot: f64 = q.iter().zip(&d).map(|(a, b)| a * b).sum();
let nq: f64 = q.iter().map(|v| v * v).sum::<f64>().sqrt();
let nd: f64 = d.iter().map(|v| v * v).sum::<f64>().sqrt();
let denom = nq * nd;
- Ok(PainlessValue::Number(if denom > 0.0 { dot / denom } else { 0.0 }))
+ Ok(PainlessValue::Number(if denom > 0.0 {
+ dot / denom
+ } else {
+ 0.0
+ }))
}
"l1norm" | "l1Norm" => {
- if args.len() != 2 { return Err("l1norm expects 2 args".into()); }
+ if args.len() != 2 {
+ return Err("l1norm expects 2 args".into());
+ }
let q: Vec<f64> = match &args[0] {
- PainlessValue::Array(arr) => arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect(),
+ PainlessValue::Array(arr) => {
+ arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect()
+ }
_ => return Err("l1norm arg 0 must be array".into()),
};
let d: Vec<f64> = match &args[1] {
@@ -989,9 +1191,13 @@ fn global_call(name: &str, args: &[PainlessValue], ctx: &PainlessCtx) -> Result<
Ok(PainlessValue::Number(s))
}
"l2norm" | "l2Norm" => {
- if args.len() != 2 { return Err("l2norm expects 2 args".into()); }
+ if args.len() != 2 {
+ return Err("l2norm expects 2 args".into());
+ }
let q: Vec<f64> = match &args[0] {
- PainlessValue::Array(arr) => arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect(),
+ PainlessValue::Array(arr) => {
+ arr.iter().map(|v| v.as_f64().unwrap_or(0.0)).collect()
+ }
_ => return Err("l2norm arg 0 must be array".into()),
};
let d: Vec<f64> = match &args[1] {
@@ -1004,11 +1210,18 @@ fn global_call(name: &str, args: &[PainlessValue], ctx: &PainlessCtx) -> Result<
}
_ => return Err("l2norm arg 1 must be field name".into()),
};
- let s: f64 = q.iter().zip(&d).map(|(a, b)| (a - b).powi(2)).sum::<f64>().sqrt();
+ let s: f64 = q
+ .iter()
+ .zip(&d)
+ .map(|(a, b)| (a - b).powi(2))
+ .sum::<f64>()
+ .sqrt();
Ok(PainlessValue::Number(s))
}
"sigmoid" => {
- if args.len() != 1 { return Err("sigmoid expects 1 arg".into()); }
+ if args.len() != 1 {
+ return Err("sigmoid expects 1 arg".into());
+ }
let x = args[0].as_f64().unwrap_or(0.0);
Ok(PainlessValue::Number(1.0 / (1.0 + (-x).exp())))
}
@@ -1029,7 +1242,11 @@ mod tests {
fn doc_value_times_param() {
let doc = json!({"num_likes": 150});
let params = json!({"multiplier": 10});
- let v = eval_painless("doc['num_likes'].value * params.multiplier", &ctx(&doc, &params, 0.0)).unwrap();
+ let v = eval_painless(
+ "doc['num_likes'].value * params.multiplier",
+ &ctx(&doc, &params, 0.0),
+ )
+ .unwrap();
assert!((v.as_f64().unwrap() - 1500.0).abs() < 1e-9);
}
@@ -1045,7 +1262,8 @@ mod tests {
fn ternary_dot_product() {
let doc = json!({"vec": [1.0, 2.0, 3.0]});
let params = json!({"q": [1.0, 0.0, -1.0]});
- let src = "double s = dotProduct(params.q, 'vec'); return s < 0 ? 1.0 / (1.0 - s) : s + 1.0;";
+ let src =
+ "double s = dotProduct(params.q, 'vec'); return s < 0 ? 1.0 / (1.0 - s) : s + 1.0;";
let v = eval_painless(src, &ctx(&doc, &params, 0.0)).unwrap();
// dot = 1*1 + 2*0 + 3*-1 = -2 → 1/(1-(-2)) = 1/3
assert!((v.as_f64().unwrap() - (1.0 / 3.0)).abs() < 1e-6);
@@ -1055,7 +1273,11 @@ mod tests {
fn if_return() {
let doc = json!({"x": 10});
let params = json!({});
- let v = eval_painless("if (doc['x'].value > 5) { return 100; } return 0;", &ctx(&doc, &params, 0.0)).unwrap();
+ let v = eval_painless(
+ "if (doc['x'].value > 5) { return 100; } return 0;",
+ &ctx(&doc, &params, 0.0),
+ )
+ .unwrap();
assert!((v.as_f64().unwrap() - 100.0).abs() < 1e-9);
}
diff --git a/engine/crates/xerj-engine/src/rbac.rs b/engine/crates/xerj-engine/src/rbac.rs
index 732878f8a..8a3b17e8f 100644
--- a/engine/crates/xerj-engine/src/rbac.rs
+++ b/engine/crates/xerj-engine/src/rbac.rs
@@ -47,8 +47,16 @@ pub struct Role {
}
impl Role {
- pub fn new(name: impl Into<String>, privileges: HashSet<Privilege>, indices: Vec<String>) -> Self {
- Self { name: name.into(), privileges, indices }
+ pub fn new(
+ name: impl Into<String>,
+ privileges: HashSet<Privilege>,
+ indices: Vec<String>,
+ ) -> Self {
+ Self {
+ name: name.into(),
+ privileges,
+ indices,
+ }
}
/// Does this role apply to the named index? Glob: "*" matches
@@ -84,7 +92,9 @@ impl RoleStore {
for r in default_roles() {
roles.insert(r.name.clone(), r);
}
- Arc::new(Self { roles: RwLock::new(roles) })
+ Arc::new(Self {
+ roles: RwLock::new(roles),
+ })
}
pub fn put(&self, role: Role) {
@@ -109,7 +119,17 @@ fn default_roles() -> Vec<Role> {
vec![
Role::new(
"admin",
- [ReadIndex, WriteIndex, AdminIndex, SnapshotCreate, SnapshotRestore, SecurityAdmin, AuditRead].into_iter().collect(),
+ [
+ ReadIndex,
+ WriteIndex,
+ AdminIndex,
+ SnapshotCreate,
+ SnapshotRestore,
+ SecurityAdmin,
+ AuditRead,
+ ]
+ .into_iter()
+ .collect(),
vec!["*".into()],
),
Role::new(
@@ -117,11 +137,7 @@ fn default_roles() -> Vec<Role> {
[ReadIndex, WriteIndex].into_iter().collect(),
vec!["*".into()],
),
- Role::new(
- "read",
- [ReadIndex].into_iter().collect(),
- vec!["*".into()],
- ),
+ Role::new("read", [ReadIndex].into_iter().collect(), vec!["*".into()]),
Role::new(
"read_only_index",
[ReadIndex].into_iter().collect(),
diff --git a/engine/crates/xerj-engine/src/sql.rs b/engine/crates/xerj-engine/src/sql.rs
index 09e961add..fbabc83ec 100644
--- a/engine/crates/xerj-engine/src/sql.rs
+++ b/engine/crates/xerj-engine/src/sql.rs
@@ -76,35 +76,58 @@ fn tokenise(sql: &str) -> Vec<Token> {
let mut i = 0;
while i < chars.len() {
match chars[i] {
- ' ' | '\t' | '\n' | '\r' => { i += 1; }
- ',' => { tokens.push(Token::Comma); i += 1; }
- '(' => { tokens.push(Token::LParen); i += 1; }
- ')' => { tokens.push(Token::RParen); i += 1; }
- '*' => { tokens.push(Token::Star); i += 1; }
+ ' ' | '\t' | '\n' | '\r' => {
+ i += 1;
+ }
+ ',' => {
+ tokens.push(Token::Comma);
+ i += 1;
+ }
+ '(' => {
+ tokens.push(Token::LParen);
+ i += 1;
+ }
+ ')' => {
+ tokens.push(Token::RParen);
+ i += 1;
+ }
+ '*' => {
+ tokens.push(Token::Star);
+ i += 1;
+ }
'>' => {
if i + 1 < chars.len() && chars[i + 1] == '=' {
- tokens.push(Token::Ge); i += 2;
+ tokens.push(Token::Ge);
+ i += 2;
} else {
- tokens.push(Token::Gt); i += 1;
+ tokens.push(Token::Gt);
+ i += 1;
}
}
'<' => {
if i + 1 < chars.len() && chars[i + 1] == '=' {
- tokens.push(Token::Le); i += 2;
+ tokens.push(Token::Le);
+ i += 2;
} else if i + 1 < chars.len() && chars[i + 1] == '>' {
- tokens.push(Token::Ne); i += 2;
+ tokens.push(Token::Ne);
+ i += 2;
} else {
- tokens.push(Token::Lt); i += 1;
+ tokens.push(Token::Lt);
+ i += 1;
}
}
'!' => {
if i + 1 < chars.len() && chars[i + 1] == '=' {
- tokens.push(Token::Ne); i += 2;
+ tokens.push(Token::Ne);
+ i += 2;
} else {
i += 1;
}
}
- '=' => { tokens.push(Token::Eq); i += 1; }
+ '=' => {
+ tokens.push(Token::Eq);
+ i += 1;
+ }
'\'' | '"' => {
let quote = chars[i];
i += 1;
@@ -118,12 +141,18 @@ fn tokenise(sql: &str) -> Vec<Token> {
}
i += 1;
}
- if i < chars.len() { i += 1; } // consume closing quote
+ if i < chars.len() {
+ i += 1;
+ } // consume closing quote
tokens.push(Token::Str(s));
}
- c if c.is_ascii_digit() || (c == '-' && i + 1 < chars.len() && chars[i + 1].is_ascii_digit()) => {
+ c if c.is_ascii_digit()
+ || (c == '-' && i + 1 < chars.len() && chars[i + 1].is_ascii_digit()) =>
+ {
let start = i;
- if chars[i] == '-' { i += 1; }
+ if chars[i] == '-' {
+ i += 1;
+ }
while i < chars.len() && (chars[i].is_ascii_digit() || chars[i] == '.') {
i += 1;
}
@@ -133,13 +162,17 @@ fn tokenise(sql: &str) -> Vec<Token> {
}
c if c.is_alphabetic() || c == '_' || c == '.' => {
let start = i;
- while i < chars.len() && (chars[i].is_alphanumeric() || chars[i] == '_' || chars[i] == '.') {
+ while i < chars.len()
+ && (chars[i].is_alphanumeric() || chars[i] == '_' || chars[i] == '.')
+ {
i += 1;
}
let word: String = chars[start..i].iter().collect();
tokens.push(Token::Word(word));
}
- _ => { i += 1; }
+ _ => {
+ i += 1;
+ }
}
}
tokens
@@ -165,7 +198,9 @@ pub fn parse_sql(sql: &str) -> Result<SqlQuery, String> {
// SELECT
match peek_word(pos) {
- Some(w) if w == "SELECT" => { pos += 1; }
+ Some(w) if w == "SELECT" => {
+ pos += 1;
+ }
_ => return Err("Expected SELECT".to_string()),
}
@@ -188,7 +223,12 @@ pub fn parse_sql(sql: &str) -> Result<SqlQuery, String> {
}
Some(Token::Word(w)) => {
let upper = w.to_uppercase();
- if upper == "FROM" || upper == "GROUP" || upper == "ORDER" || upper == "LIMIT" || upper == "WHERE" {
+ if upper == "FROM"
+ || upper == "GROUP"
+ || upper == "ORDER"
+ || upper == "LIMIT"
+ || upper == "WHERE"
+ {
break;
}
if upper == "COUNT" {
@@ -243,7 +283,11 @@ pub fn parse_sql(sql: &str) -> Result<SqlQuery, String> {
_ => break,
}
// Consume optional comma.
- if let Some(Token::Comma) = peek(pos) { pos += 1; } else { break; }
+ if let Some(Token::Comma) = peek(pos) {
+ pos += 1;
+ } else {
+ break;
+ }
}
if fields.is_empty() && agg_functions.is_empty() {
return Err("No fields in SELECT".to_string());
@@ -251,13 +295,19 @@ pub fn parse_sql(sql: &str) -> Result<SqlQuery, String> {
// FROM
match peek_word(pos) {
- Some(w) if w == "FROM" => { pos += 1; }
+ Some(w) if w == "FROM" => {
+ pos += 1;
+ }
_ => return Err("Expected FROM".to_string()),
}
// Index name.
let index = match peek(pos) {
- Some(Token::Word(w)) => { let s = w.clone(); pos += 1; s }
+ Some(Token::Word(w)) => {
+ let s = w.clone();
+ pos += 1;
+ s
+ }
_ => return Err("Expected index name after FROM".to_string()),
};
@@ -273,20 +323,21 @@ pub fn parse_sql(sql: &str) -> Result<SqlQuery, String> {
let mut group_by: Vec<String> = Vec::new();
if peek_word(pos).as_deref() == Some("GROUP") {
pos += 1;
- if peek_word(pos).as_deref() == Some("BY") { pos += 1; }
- loop {
- match peek(pos) {
- Some(Token::Word(field)) => {
- let upper = field.to_uppercase();
- if upper == "ORDER" || upper == "LIMIT" || upper == "HAVING" {
- break;
- }
- group_by.push(field.clone());
- pos += 1;
- }
- _ => break,
+ if peek_word(pos).as_deref() == Some("BY") {
+ pos += 1;
+ }
+ while let Some(Token::Word(field)) = peek(pos) {
+ let upper = field.to_uppercase();
+ if upper == "ORDER" || upper == "LIMIT" || upper == "HAVING" {
+ break;
+ }
+ group_by.push(field.clone());
+ pos += 1;
+ if let Some(Token::Comma) = peek(pos) {
+ pos += 1;
+ } else {
+ break;
}
- if let Some(Token::Comma) = peek(pos) { pos += 1; } else { break; }
}
}
@@ -300,10 +351,21 @@ pub fn parse_sql(sql: &str) -> Result<SqlQuery, String> {
_ => {}
}
match tok {
- Token::Word(_) | Token::Num(_) | Token::Str(_)
- | Token::Eq | Token::Ne | Token::Gt | Token::Ge
- | Token::Lt | Token::Le | Token::LParen | Token::RParen
- | Token::Comma | Token::Star => { pos += 1; }
+ Token::Word(_)
+ | Token::Num(_)
+ | Token::Str(_)
+ | Token::Eq
+ | Token::Ne
+ | Token::Gt
+ | Token::Ge
+ | Token::Lt
+ | Token::Le
+ | Token::LParen
+ | Token::RParen
+ | Token::Comma
+ | Token::Star => {
+ pos += 1;
+ }
}
}
}
@@ -312,28 +374,35 @@ pub fn parse_sql(sql: &str) -> Result<SqlQuery, String> {
let mut sort: Vec<SortField> = Vec::new();
if peek_word(pos).as_deref() == Some("ORDER") {
pos += 1;
- if peek_word(pos).as_deref() == Some("BY") { pos += 1; }
- loop {
- match peek(pos) {
- Some(Token::Word(field)) => {
- let field = field.clone();
+ if peek_word(pos).as_deref() == Some("BY") {
+ pos += 1;
+ }
+ while let Some(Token::Word(field)) = peek(pos) {
+ let field = field.clone();
+ pos += 1;
+ let order = match peek_word(pos).as_deref() {
+ Some("DESC") => {
+ pos += 1;
+ SortOrder::Desc
+ }
+ Some("ASC") => {
pos += 1;
- let order = match peek_word(pos).as_deref() {
- Some("DESC") => { pos += 1; SortOrder::Desc }
- Some("ASC") => { pos += 1; SortOrder::Asc }
- _ => SortOrder::Asc,
- };
- sort.push(SortField {
- field,
- order,
- mode: xerj_query::sort::SortMode::default(),
- missing: xerj_query::sort::SortMissing::default(),
- format: None,
- });
+ SortOrder::Asc
}
- _ => break,
+ _ => SortOrder::Asc,
+ };
+ sort.push(SortField {
+ field,
+ order,
+ mode: xerj_query::sort::SortMode::default(),
+ missing: xerj_query::sort::SortMissing::default(),
+ format: None,
+ });
+ if let Some(Token::Comma) = peek(pos) {
+ pos += 1;
+ } else {
+ break;
}
- if let Some(Token::Comma) = peek(pos) { pos += 1; } else { break; }
}
}
@@ -342,12 +411,23 @@ pub fn parse_sql(sql: &str) -> Result<SqlQuery, String> {
if peek_word(pos).as_deref() == Some("LIMIT") {
pos += 1;
match peek(pos) {
- Some(Token::Num(n)) => { limit = Some(*n as usize); pos += 1; }
+ Some(Token::Num(n)) => {
+ limit = Some(*n as usize);
+ }
_ => return Err("Expected number after LIMIT".to_string()),
}
}
- Ok(SqlQuery { index, fields, query, limit, sort, distinct, group_by, agg_functions })
+ Ok(SqlQuery {
+ index,
+ fields,
+ query,
+ limit,
+ sort,
+ distinct,
+ group_by,
+ agg_functions,
+ })
}
// ── Condition parser (recursive descent) ─────────────────────────────────────
@@ -422,37 +502,79 @@ fn parse_condition(tokens: &[Token], pos: &mut usize) -> Result<QueryNode, Strin
if let Some(Token::LParen) = tokens.get(*pos) {
*pos += 1;
let inner = parse_or_expr(tokens, pos)?;
- if let Some(Token::RParen) = tokens.get(*pos) { *pos += 1; }
+ if let Some(Token::RParen) = tokens.get(*pos) {
+ *pos += 1;
+ }
return Ok(inner);
}
// field op value
let field = match tokens.get(*pos) {
- Some(Token::Word(f)) => { let f = f.clone(); *pos += 1; f }
+ Some(Token::Word(f)) => {
+ let f = f.clone();
+ *pos += 1;
+ f
+ }
_ => return Err(format!("Expected field name at pos {}", pos)),
};
let op = match tokens.get(*pos) {
- Some(Token::Eq) => { *pos += 1; "eq" }
- Some(Token::Ne) => { *pos += 1; "ne" }
- Some(Token::Gt) => { *pos += 1; "gt" }
- Some(Token::Ge) => { *pos += 1; "gte" }
- Some(Token::Lt) => { *pos += 1; "lt" }
- Some(Token::Le) => { *pos += 1; "lte" }
- Some(Token::Word(w)) if w.to_uppercase() == "LIKE" => { *pos += 1; "like" }
- Some(Token::Word(w)) if w.to_uppercase() == "NOT" => {
+ Some(Token::Eq) => {
+ *pos += 1;
+ "eq"
+ }
+ Some(Token::Ne) => {
+ *pos += 1;
+ "ne"
+ }
+ Some(Token::Gt) => {
+ *pos += 1;
+ "gt"
+ }
+ Some(Token::Ge) => {
+ *pos += 1;
+ "gte"
+ }
+ Some(Token::Lt) => {
+ *pos += 1;
+ "lt"
+ }
+ Some(Token::Le) => {
+ *pos += 1;
+ "lte"
+ }
+ Some(Token::Word(w)) if w.to_uppercase() == "LIKE" => {
+ *pos += 1;
+ "like"
+ }
+ Some(Token::Word(w)) if w.to_uppercase() == "NOT" => {
// NOT LIKE
*pos += 1;
if let Some(Token::Word(w2)) = tokens.get(*pos) {
- if w2.to_uppercase() == "LIKE" { *pos += 1; "not_like" } else { "ne" }
- } else { "ne" }
+ if w2.to_uppercase() == "LIKE" {
+ *pos += 1;
+ "not_like"
+ } else {
+ "ne"
+ }
+ } else {
+ "ne"
+ }
}
_ => return Err(format!("Expected operator at pos {}", pos)),
};
let value = match tokens.get(*pos) {
- Some(Token::Str(s)) => { let v = serde_json::Value::String(s.clone()); *pos += 1; v }
- Some(Token::Num(n)) => { let v = serde_json::json!(*n); *pos += 1; v }
+ Some(Token::Str(s)) => {
+ let v = serde_json::Value::String(s.clone());
+ *pos += 1;
+ v
+ }
+ Some(Token::Num(n)) => {
+ let v = serde_json::json!(*n);
+ *pos += 1;
+ v
+ }
Some(Token::Word(w)) => {
let upper = w.to_uppercase();
let v = if upper == "TRUE" {
@@ -464,7 +586,8 @@ fn parse_condition(tokens: &[Token], pos: &mut usize) -> Result<QueryNode, Strin
} else {
serde_json::Value::String(w.clone())
};
- *pos += 1; v
+ *pos += 1;
+ v
}
_ => return Err(format!("Expected value at pos {}", pos)),
};
@@ -482,32 +605,56 @@ fn parse_condition(tokens: &[Token], pos: &mut usize) -> Result<QueryNode, Strin
analyzer: None,
}
} else {
- QueryNode::Term { field: field.clone(), value: value.clone(), boost: None }
+ QueryNode::Term {
+ field: field.clone(),
+ value: value.clone(),
+ boost: None,
+ }
}
}
"ne" => QueryNode::Bool {
must: vec![],
filter: vec![],
should: vec![],
- must_not: vec![QueryNode::Term { field: field.clone(), value: value.clone(), boost: None }],
+ must_not: vec![QueryNode::Term {
+ field: field.clone(),
+ value: value.clone(),
+ boost: None,
+ }],
minimum_should_match: None,
},
- "gt" => make_range(&field, None, None, Some(value), None),
+ "gt" => make_range(&field, None, None, Some(value), None),
"gte" => make_range(&field, None, None, None, Some(value)),
- "lt" => make_range(&field, Some(value), None, None, None),
+ "lt" => make_range(&field, Some(value), None, None, None),
"lte" => make_range(&field, None, Some(value), None, None),
"like" => {
// Convert SQL LIKE pattern (% → *, _ → ?) to wildcard query.
- let pattern = value.as_str().unwrap_or("*").replace('%', "*").replace('_', "?");
- QueryNode::Wildcard { field: field.clone(), value: pattern, boost: None }
+ let pattern = value
+ .as_str()
+ .unwrap_or("*")
+ .replace('%', "*")
+ .replace('_', "?");
+ QueryNode::Wildcard {
+ field: field.clone(),
+ value: pattern,
+ boost: None,
+ }
}
"not_like" => {
- let pattern = value.as_str().unwrap_or("*").replace('%', "*").replace('_', "?");
+ let pattern = value
+ .as_str()
+ .unwrap_or("*")
+ .replace('%', "*")
+ .replace('_', "?");
QueryNode::Bool {
must: vec![],
filter: vec![],
should: vec![],
- must_not: vec![QueryNode::Wildcard { field: field.clone(), value: pattern, boost: None }],
+ must_not: vec![QueryNode::Wildcard {
+ field: field.clone(),
+ value: pattern,
+ boost: None,
+ }],
minimum_should_match: None,
}
}
diff --git a/engine/crates/xerj-engine/src/turbo_ingest.rs b/engine/crates/xerj-engine/src/turbo_ingest.rs
index 0fd94a39e..27f0403a9 100644
--- a/engine/crates/xerj-engine/src/turbo_ingest.rs
+++ b/engine/crates/xerj-engine/src/turbo_ingest.rs
@@ -28,7 +28,6 @@
//! turbo_fast_analyzer = false # skip stemming/stopwords for max speed
//! ```
-use rayon::prelude::*;
use serde_json::Value;
use std::sync::Arc;
@@ -58,13 +57,13 @@ pub fn simd_lowercase(input: &[u8]) -> Vec<u8> {
if all_ascii {
// Fast path: ASCII lowercase in bulk.
for &b in chunk {
- output.push(if b >= b'A' && b <= b'Z' { b + 32 } else { b });
+ output.push(if b.is_ascii_uppercase() { b + 32 } else { b });
}
} else {
// Slow path: multi-byte sequences — pass through unchanged.
// Uppercase ASCII bytes that happen to appear are still lowercased.
for &b in chunk {
- output.push(if b >= b'A' && b <= b'Z' { b + 32 } else { b });
+ output.push(if b.is_ascii_uppercase() { b + 32 } else { b });
}
}
i += 16;
@@ -72,7 +71,7 @@ pub fn simd_lowercase(input: &[u8]) -> Vec<u8> {
// Tail: remaining bytes.
for &b in &input[i..] {
- output.push(if b >= b'A' && b <= b'Z' { b + 32 } else { b });
+ output.push(if b.is_ascii_uppercase() { b + 32 } else { b });
}
output
@@ -91,14 +90,37 @@ pub fn simd_find_word_boundaries(input: &[u8]) -> Vec<(usize, usize)> {
for (i, &b) in input.iter().enumerate() {
let is_sep = matches!(
b,
- b' ' | b'\t' | b'\n' | b'\r'
- | b',' | b'.' | b'!' | b'?'
- | b';' | b':' | b'"' | b'\''
- | b'(' | b')' | b'[' | b']'
- | b'{' | b'}' | b'/' | b'\\'
- | b'|' | b'@' | b'#' | b'%'
- | b'^' | b'&' | b'*' | b'+'
- | b'=' | b'<' | b'>' | b'~'
+ b' ' | b'\t'
+ | b'\n'
+ | b'\r'
+ | b','
+ | b'.'
+ | b'!'
+ | b'?'
+ | b';'
+ | b':'
+ | b'"'
+ | b'\''
+ | b'('
+ | b')'
+ | b'['
+ | b']'
+ | b'{'
+ | b'}'
+ | b'/'
+ | b'\\'
+ | b'|'
+ | b'@'
+ | b'#'
+ | b'%'
+ | b'^'
+ | b'&'
+ | b'*'
+ | b'+'
+ | b'='
+ | b'<'
+ | b'>'
+ | b'~'
| b'`'
);
@@ -151,7 +173,7 @@ impl FastTokenizer {
.iter()
.filter(|&&(s, e)| {
let len = e - s;
- len >= 2 && len <= 40
+ (2..=40).contains(&len)
})
.map(|&(s, e)| {
// SAFETY: `lower` contains only bytes that were either:
@@ -242,6 +264,10 @@ pub struct TurboIngestPipeline {
/// round-trip that v13 was paying.
buffer: Vec<(String, Value, Arc<[u8]>)>,
/// Whether to use parallel tokenisation.
+ ///
+ /// Retained for the public `new(batch_size, parallel)` constructor API;
+ /// unread since M5.9 dropped pre-tokenisation from the flush hot path.
+ #[allow(dead_code)]
parallel: bool,
}
@@ -381,7 +407,11 @@ impl BatchWalWriter {
}
// Estimate total size so we can pre-allocate one contiguous buffer.
- let total_cap: usize = self.entries.iter().map(|(id, src)| 8 + id.len() + src.len()).sum();
+ let total_cap: usize = self
+ .entries
+ .iter()
+ .map(|(id, src)| 8 + id.len() + src.len())
+ .sum();
let mut buf = Vec::with_capacity(total_cap);
for (id, src) in &self.entries {
@@ -477,10 +507,26 @@ mod tests {
#[test]
fn pipeline_auto_flushes_at_batch_size() {
let mut pipeline = TurboIngestPipeline::new(3, false);
- assert!(pipeline.push("id1".into(), serde_json::json!({"text": "hello"}), empty_bytes()).is_none());
- assert!(pipeline.push("id2".into(), serde_json::json!({"text": "world"}), empty_bytes()).is_none());
+ assert!(pipeline
+ .push(
+ "id1".into(),
+ serde_json::json!({"text": "hello"}),
+ empty_bytes()
+ )
+ .is_none());
+ assert!(pipeline
+ .push(
+ "id2".into(),
+ serde_json::json!({"text": "world"}),
+ empty_bytes()
+ )
+ .is_none());
let results = pipeline
- .push("id3".into(), serde_json::json!({"text": "foo bar"}), empty_bytes())
+ .push(
+ "id3".into(),
+ serde_json::json!({"text": "foo bar"}),
+ empty_bytes(),
+ )
.expect("should auto-flush at batch_size=3");
assert_eq!(results.len(), 3);
assert_eq!(pipeline.buffered(), 0);
@@ -492,8 +538,16 @@ mod tests {
// tokens are now empty Vecs. Verify the pipeline still
// produces the right number of results with correct ids.
let mut pipeline = TurboIngestPipeline::new(100, false);
- pipeline.push("id1".into(), serde_json::json!({"body": "quick brown fox"}), empty_bytes());
- pipeline.push("id2".into(), serde_json::json!({"body": "lazy dog"}), empty_bytes());
+ pipeline.push(
+ "id1".into(),
+ serde_json::json!({"body": "quick brown fox"}),
+ empty_bytes(),
+ );
+ pipeline.push(
+ "id2".into(),
+ serde_json::json!({"body": "lazy dog"}),
+ empty_bytes(),
+ );
let results = pipeline.flush();
assert_eq!(results.len(), 2);
assert_eq!(results[0].id, "id1");
diff --git a/engine/crates/xerj-engine/tests/battle_test.rs b/engine/crates/xerj-engine/tests/battle_test.rs
index 7fe2d0362..71600b020 100644
--- a/engine/crates/xerj-engine/tests/battle_test.rs
+++ b/engine/crates/xerj-engine/tests/battle_test.rs
@@ -65,8 +65,7 @@ fn dataset_dir() -> PathBuf {
}
// CARGO_MANIFEST_DIR points to crates/xerj-engine.
// Walk up until we hit the workspace root (has Cargo.lock).
- let manifest = std::env::var("CARGO_MANIFEST_DIR")
- .unwrap_or_else(|_| ".".into());
+ let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
let mut dir = PathBuf::from(manifest);
loop {
if dir.join("Cargo.lock").exists() {
@@ -74,7 +73,7 @@ fn dataset_dir() -> PathBuf {
}
match dir.parent() {
Some(p) => dir = p.to_path_buf(),
- None => break,
+ None => break,
}
}
// Fallback: relative to crate root (works if run from workspace root)
@@ -100,7 +99,12 @@ fn load_ndjson(filename: &str) -> Result<Vec<Value>, String> {
continue;
}
let val: Value = serde_json::from_str(trimmed).map_err(|e| {
- format!("JSON parse error in {} line {}: {}", filename, lineno + 1, e)
+ format!(
+ "JSON parse error in {} line {}: {}",
+ filename,
+ lineno + 1,
+ e
+ )
})?;
docs.push(val);
}
@@ -197,7 +201,10 @@ fn lat_rows(label: &str, lats: &[Duration]) {
row(&format!("{} p50", label), &format!("{:.2?}", ps[0]));
row(&format!("{} p95", label), &format!("{:.2?}", ps[1]));
row(&format!("{} p99", label), &format!("{:.2?}", ps[2]));
- row(&format!("{} min/max", label), &format!("{:.2?} / {:.2?}", lats[0], lats[lats.len() - 1]));
+ row(
+ &format!("{} min/max", label),
+ &format!("{:.2?} / {:.2?}", lats[0], lats[lats.len() - 1]),
+ );
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -213,19 +220,31 @@ async fn battle_bulk_ingest_all() {
// Load datasets
let access_docs = match load_ndjson("access_logs.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let product_docs = match load_ndjson("products.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let error_docs = match load_ndjson("error_logs.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let article_docs = match load_ndjson("articles.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let total_docs = access_docs.len() + product_docs.len() + error_docs.len() + article_docs.len();
@@ -241,22 +260,60 @@ async fn battle_bulk_ingest_all() {
let wall_start = Instant::now();
let (n, t) = bulk_index(&engine, "access-logs", &access_docs).await;
- row("access-logs indexed:", &format!("{} docs in {:.2?} ({:.0} docs/s)", n, t, n as f64 / t.as_secs_f64()));
+ row(
+ "access-logs indexed:",
+ &format!(
+ "{} docs in {:.2?} ({:.0} docs/s)",
+ n,
+ t,
+ n as f64 / t.as_secs_f64()
+ ),
+ );
let (n, t) = bulk_index(&engine, "products", &product_docs).await;
- row("products indexed:", &format!("{} docs in {:.2?} ({:.0} docs/s)", n, t, n as f64 / t.as_secs_f64()));
+ row(
+ "products indexed:",
+ &format!(
+ "{} docs in {:.2?} ({:.0} docs/s)",
+ n,
+ t,
+ n as f64 / t.as_secs_f64()
+ ),
+ );
let (n, t) = bulk_index(&engine, "error-logs", &error_docs).await;
- row("error-logs indexed:", &format!("{} docs in {:.2?} ({:.0} docs/s)", n, t, n as f64 / t.as_secs_f64()));
+ row(
+ "error-logs indexed:",
+ &format!(
+ "{} docs in {:.2?} ({:.0} docs/s)",
+ n,
+ t,
+ n as f64 / t.as_secs_f64()
+ ),
+ );
let (n, t) = bulk_index(&engine, "articles", &article_docs).await;
- row("articles indexed:", &format!("{} docs in {:.2?} ({:.0} docs/s)", n, t, n as f64 / t.as_secs_f64()));
+ row(
+ "articles indexed:",
+ &format!(
+ "{} docs in {:.2?} ({:.0} docs/s)",
+ n,
+ t,
+ n as f64 / t.as_secs_f64()
+ ),
+ );
let wall = wall_start.elapsed();
separator();
row("TOTAL wall-clock time:", &format!("{:.2?}", wall));
- row("Overall throughput:", &format!("{:.0} docs/sec", total_docs as f64 / wall.as_secs_f64()));
- row("Avg latency per doc:", &format!("{:.3} ms", wall.as_millis() as f64 / total_docs as f64));
+ row(
+ "Overall throughput:",
+ &format!("{:.0} docs/sec", total_docs as f64 / wall.as_secs_f64()),
+ );
+ row(
+ "Avg latency per doc:",
+ &format!("{:.3} ms", wall.as_millis() as f64 / total_docs as f64),
+ );
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -270,7 +327,10 @@ async fn battle_log_error_search() {
let docs = match load_ndjson("error_logs.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let (engine, _dir) = battle_engine();
@@ -278,7 +338,10 @@ async fn battle_log_error_search() {
let (indexed, ingest_time) = bulk_index(&engine, "error-logs", &docs).await;
let idx = engine.get_index("error-logs").unwrap();
- row("Docs indexed:", &format!("{} in {:.2?}", indexed, ingest_time));
+ row(
+ "Docs indexed:",
+ &format!("{} in {:.2?}", indexed, ingest_time),
+ );
separator();
const RUNS: usize = 200;
@@ -287,7 +350,8 @@ async fn battle_log_error_search() {
let req_error = parse_request(&json!({
"query": { "term": { "level": "ERROR" } },
"size": 50
- })).unwrap();
+ }))
+ .unwrap();
let result = idx.search(&req_error).await.unwrap();
let error_count = result.total.value;
row("ERROR doc count (term query):", &error_count.to_string());
@@ -300,9 +364,13 @@ async fn battle_log_error_search() {
let req_err_fatal = parse_request(&json!({
"query": { "terms": { "level": ["ERROR", "FATAL"] } },
"size": 100
- })).unwrap();
+ }))
+ .unwrap();
let result2 = idx.search(&req_err_fatal).await.unwrap();
- row("ERROR+FATAL count (terms query):", &result2.total.value.to_string());
+ row(
+ "ERROR+FATAL count (terms query):",
+ &result2.total.value.to_string(),
+ );
let lats2 = measure_search!(idx, req_err_fatal, RUNS);
lat_rows("ERROR+FATAL terms query", &lats2);
@@ -321,10 +389,17 @@ async fn battle_log_error_search() {
}
},
"size": 50
- })).unwrap();
+ }))
+ .unwrap();
let result3 = idx.search(&req_conn_error).await.unwrap();
- row("Connection timeout errors (bool must+filter):", &result3.total.value.to_string());
- assert!(result3.total.value > 0, "Expected at least one connection timeout error");
+ row(
+ "Connection timeout errors (bool must+filter):",
+ &result3.total.value.to_string(),
+ );
+ assert!(
+ result3.total.value > 0,
+ "Expected at least one connection timeout error"
+ );
let lats3 = measure_search!(idx, req_conn_error, RUNS);
lat_rows("Bool must+filter (conn errors)", &lats3);
@@ -343,9 +418,13 @@ async fn battle_log_error_search() {
}
},
"size": 50
- })).unwrap();
+ }))
+ .unwrap();
let result4 = idx.search(&req_timeout).await.unwrap();
- row("Timeout messages, not FATAL:", &result4.total.value.to_string());
+ row(
+ "Timeout messages, not FATAL:",
+ &result4.total.value.to_string(),
+ );
let lats4 = measure_search!(idx, req_timeout, RUNS);
lat_rows("Bool must+must_not (timeout)", &lats4);
@@ -362,7 +441,10 @@ async fn battle_product_fulltext_search() {
let docs = match load_ndjson("products.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let (engine, _dir) = battle_engine();
@@ -370,7 +452,10 @@ async fn battle_product_fulltext_search() {
let (indexed, ingest_time) = bulk_index(&engine, "products", &docs).await;
let idx = engine.get_index("products").unwrap();
- row("Docs indexed:", &format!("{} in {:.2?}", indexed, ingest_time));
+ row(
+ "Docs indexed:",
+ &format!("{} in {:.2?}", indexed, ingest_time),
+ );
separator();
const RUNS: usize = 500;
@@ -383,10 +468,17 @@ async fn battle_product_fulltext_search() {
}
},
"size": 10
- })).unwrap();
+ }))
+ .unwrap();
let res = idx.search(&req_premium).await.unwrap();
- row("match 'premium quality' in name hits:", &res.total.value.to_string());
- assert!(res.total.value > 0, "Expected hits for 'premium' or 'quality' in product names");
+ row(
+ "match 'premium quality' in name hits:",
+ &res.total.value.to_string(),
+ );
+ assert!(
+ res.total.value > 0,
+ "Expected hits for 'premium' or 'quality' in product names"
+ );
assert!(res.hits.len() <= 10);
let lats = measure_search!(idx, req_premium, RUNS);
@@ -399,9 +491,13 @@ async fn battle_product_fulltext_search() {
"match": { "description": "durable excellent craftsmanship" }
},
"size": 20
- })).unwrap();
+ }))
+ .unwrap();
let res2 = idx.search(&req_desc).await.unwrap();
- row("match 'durable excellent craftsmanship':", &res2.total.value.to_string());
+ row(
+ "match 'durable excellent craftsmanship':",
+ &res2.total.value.to_string(),
+ );
let lats2 = measure_search!(idx, req_desc, RUNS);
lat_rows("match (description)", &lats2);
@@ -422,14 +518,26 @@ async fn battle_product_fulltext_search() {
},
"size": 20,
"sort": [{ "price": "asc" }]
- })).unwrap();
+ }))
+ .unwrap();
let res3 = idx.search(&req_bool).await.unwrap();
- row("Bool + range + sort (in-stock $50-$500):", &res3.total.value.to_string());
+ row(
+ "Bool + range + sort (in-stock $50-$500):",
+ &res3.total.value.to_string(),
+ );
// Verify sort order if we got results
if res3.hits.len() >= 2 {
- let p0 = res3.hits[0].source.get("price").and_then(|v| v.as_f64()).unwrap_or(0.0);
- let p1 = res3.hits[1].source.get("price").and_then(|v| v.as_f64()).unwrap_or(0.0);
+ let p0 = res3.hits[0]
+ .source
+ .get("price")
+ .and_then(|v| v.as_f64())
+ .unwrap_or(0.0);
+ let p1 = res3.hits[1]
+ .source
+ .get("price")
+ .and_then(|v| v.as_f64())
+ .unwrap_or(0.0);
assert!(p0 <= p1, "price sort asc violated: {} > {}", p0, p1);
row("Sort correctness (price asc):", "OK");
}
@@ -442,9 +550,13 @@ async fn battle_product_fulltext_search() {
let req_prefix = parse_request(&json!({
"query": { "prefix": { "category": "electr" } },
"size": 50
- })).unwrap();
+ }))
+ .unwrap();
let res4 = idx.search(&req_prefix).await.unwrap();
- row("Prefix 'electr' on category:", &res4.total.value.to_string());
+ row(
+ "Prefix 'electr' on category:",
+ &res4.total.value.to_string(),
+ );
let lats4 = measure_search!(idx, req_prefix, RUNS);
lat_rows("Prefix query (autocomplete)", &lats4);
@@ -461,15 +573,23 @@ async fn battle_geo_distance_search() {
let docs = match load_ndjson("products.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let (engine, _dir) = battle_engine();
- engine.create_index("products-geo", Schema::empty()).unwrap();
+ engine
+ .create_index("products-geo", Schema::empty())
+ .unwrap();
let (indexed, ingest_time) = bulk_index(&engine, "products-geo", &docs).await;
let idx = engine.get_index("products-geo").unwrap();
- row("Docs indexed:", &format!("{} in {:.2?}", indexed, ingest_time));
+ row(
+ "Docs indexed:",
+ &format!("{} in {:.2?}", indexed, ingest_time),
+ );
separator();
const RUNS: usize = 200;
@@ -483,7 +603,8 @@ async fn battle_geo_distance_search() {
}
},
"size": 100
- })).unwrap();
+ }))
+ .unwrap();
let res = idx.search(&req_nyc_50km).await.unwrap();
row("Within 50 km of NYC:", &res.total.value.to_string());
@@ -500,7 +621,8 @@ async fn battle_geo_distance_search() {
}
},
"size": 100
- })).unwrap();
+ }))
+ .unwrap();
let res2 = idx.search(&req_nyc_500km).await.unwrap();
row("Within 500 km of NYC:", &res2.total.value.to_string());
// More results at wider radius
@@ -531,7 +653,8 @@ async fn battle_geo_distance_search() {
}
},
"size": 50
- })).unwrap();
+ }))
+ .unwrap();
let res3 = idx.search(&req_geo_stock).await.unwrap();
row("200 km + in_stock=true:", &res3.total.value.to_string());
@@ -550,16 +673,24 @@ async fn battle_aggregations() {
let error_docs = match load_ndjson("error_logs.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let product_docs = match load_ndjson("products.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let (engine, _dir) = battle_engine();
engine.create_index("agg-errors", Schema::empty()).unwrap();
- engine.create_index("agg-products", Schema::empty()).unwrap();
+ engine
+ .create_index("agg-products", Schema::empty())
+ .unwrap();
let (en, et) = bulk_index(&engine, "agg-errors", &error_docs).await;
let (pn, pt) = bulk_index(&engine, "agg-products", &product_docs).await;
@@ -587,7 +718,8 @@ async fn battle_aggregations() {
"terms": { "field": "hostname", "size": 10 }
}
}
- })).unwrap();
+ }))
+ .unwrap();
let agg_res = err_idx.search(&req_top_services).await.unwrap();
let aggs = agg_res.aggs.as_ref().expect("aggs present");
@@ -617,7 +749,8 @@ async fn battle_aggregations() {
"stats": { "field": "duration_ms" }
}
}
- })).unwrap();
+ }))
+ .unwrap();
let res2 = err_idx.search(&req_duration_stats).await.unwrap();
let aggs2 = res2.aggs.as_ref().expect("aggs present");
@@ -658,20 +791,32 @@ async fn battle_aggregations() {
"stats": { "field": "rating" }
}
}
- })).unwrap();
+ }))
+ .unwrap();
let res3 = prod_idx.search(&req_faceted).await.unwrap();
let aggs3 = res3.aggs.as_ref().expect("aggs present");
- let cat_buckets = aggs3["by_category"]["buckets"].as_array().expect("category buckets");
- row("Category buckets (in-stock):", &cat_buckets.len().to_string());
+ let cat_buckets = aggs3["by_category"]["buckets"]
+ .as_array()
+ .expect("category buckets");
+ row(
+ "Category buckets (in-stock):",
+ &cat_buckets.len().to_string(),
+ );
assert!(!cat_buckets.is_empty(), "Expected category buckets");
- let price_buckets = aggs3["price_ranges"]["buckets"].as_array().expect("price range buckets");
+ let price_buckets = aggs3["price_ranges"]["buckets"]
+ .as_array()
+ .expect("price range buckets");
row("Price range buckets:", &price_buckets.len().to_string());
if let Some(avg_rating) = aggs3["rating_stats"]["avg"].as_f64() {
row("Avg product rating:", &format!("{:.2}", avg_rating));
- assert!((1.0..=5.0).contains(&avg_rating), "Rating out of range: {}", avg_rating);
+ assert!(
+ (1.0..=5.0).contains(&avg_rating),
+ "Rating out of range: {}",
+ avg_rating
+ );
}
let lats3 = measure_search!(prod_idx, req_faceted, RUNS);
@@ -693,11 +838,17 @@ async fn battle_aggregations() {
}
}
}
- })).unwrap();
+ }))
+ .unwrap();
let res4 = prod_idx.search(&req_composite).await.unwrap();
let aggs4 = res4.aggs.as_ref().expect("aggs present");
- let comp_buckets = aggs4["by_brand_cat"]["buckets"].as_array().expect("composite buckets");
- row("Composite (category x brand) buckets:", &comp_buckets.len().to_string());
+ let comp_buckets = aggs4["by_brand_cat"]["buckets"]
+ .as_array()
+ .expect("composite buckets");
+ row(
+ "Composite (category x brand) buckets:",
+ &comp_buckets.len().to_string(),
+ );
let lats4 = measure_search!(prod_idx, req_composite, RUNS);
lat_rows("Composite agg (brand x category)", &lats4);
@@ -714,7 +865,10 @@ async fn battle_article_search_with_highlight() {
let docs = match load_ndjson("articles.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let (engine, _dir) = battle_engine();
@@ -722,7 +876,10 @@ async fn battle_article_search_with_highlight() {
let (indexed, ingest_time) = bulk_index(&engine, "articles", &docs).await;
let idx = engine.get_index("articles").unwrap();
- row("Docs indexed:", &format!("{} in {:.2?}", indexed, ingest_time));
+ row(
+ "Docs indexed:",
+ &format!("{} in {:.2?}", indexed, ingest_time),
+ );
separator();
const RUNS: usize = 300;
@@ -741,15 +898,25 @@ async fn battle_article_search_with_highlight() {
}
}
}
- })).unwrap();
+ }))
+ .unwrap();
let res = idx.search(&req_hl).await.unwrap();
- row("'memory safety performance' hits:", &res.total.value.to_string());
- assert!(res.total.value > 0, "Expected FTS hits for 'memory safety performance'");
+ row(
+ "'memory safety performance' hits:",
+ &res.total.value.to_string(),
+ );
+ assert!(
+ res.total.value > 0,
+ "Expected FTS hits for 'memory safety performance'"
+ );
// Verify highlights are present and contain the em tags
let hits_with_highlight = res.hits.iter().filter(|h| h.highlight.is_some()).count();
- row("Hits with highlight fragments:", &hits_with_highlight.to_string());
+ row(
+ "Hits with highlight fragments:",
+ &hits_with_highlight.to_string(),
+ );
if let Some(first_with_hl) = res.hits.iter().find(|h| h.highlight.is_some()) {
let hl = first_with_hl.highlight.as_ref().unwrap();
if let Some(frags) = hl.get("body") {
@@ -774,9 +941,13 @@ async fn battle_article_search_with_highlight() {
}
},
"size": 10
- })).unwrap();
+ }))
+ .unwrap();
let res2 = idx.search(&req_multi).await.unwrap();
- row("multi_match 'distributed systems':", &res2.total.value.to_string());
+ row(
+ "multi_match 'distributed systems':",
+ &res2.total.value.to_string(),
+ );
let lats2 = measure_search!(idx, req_multi, RUNS);
lat_rows("multi_match (title+body)", &lats2);
@@ -788,9 +959,13 @@ async fn battle_article_search_with_highlight() {
"match_phrase": { "body": "memory safety" }
},
"size": 20
- })).unwrap();
+ }))
+ .unwrap();
let res3 = idx.search(&req_phrase).await.unwrap();
- row("match_phrase 'memory safety':", &res3.total.value.to_string());
+ row(
+ "match_phrase 'memory safety':",
+ &res3.total.value.to_string(),
+ );
let lats3 = measure_search!(idx, req_phrase, RUNS);
lat_rows("match_phrase", &lats3);
@@ -815,9 +990,13 @@ async fn battle_article_search_with_highlight() {
},
"size": 20,
"sort": [{ "views": "desc" }, { "_score": "desc" }]
- })).unwrap();
+ }))
+ .unwrap();
let res4 = idx.search(&req_complex).await.unwrap();
- row("Complex bool (performance+security):", &res4.total.value.to_string());
+ row(
+ "Complex bool (performance+security):",
+ &res4.total.value.to_string(),
+ );
let lats4 = measure_search!(idx, req_complex, RUNS);
lat_rows("Complex bool + sort", &lats4);
@@ -834,7 +1013,10 @@ async fn battle_access_log_analytics() {
let docs = match load_ndjson("access_logs.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let (engine, _dir) = battle_engine();
@@ -842,7 +1024,10 @@ async fn battle_access_log_analytics() {
let (indexed, ingest_time) = bulk_index(&engine, "access-logs", &docs).await;
let idx = engine.get_index("access-logs").unwrap();
- row("Docs indexed:", &format!("{} in {:.2?}", indexed, ingest_time));
+ row(
+ "Docs indexed:",
+ &format!("{} in {:.2?}", indexed, ingest_time),
+ );
separator();
const RUNS: usize = 200;
@@ -851,7 +1036,8 @@ async fn battle_access_log_analytics() {
let req_500 = parse_request(&json!({
"query": { "term": { "status": "500" } },
"size": 100
- })).unwrap();
+ }))
+ .unwrap();
let res_500 = idx.search(&req_500).await.unwrap();
row("HTTP 500 errors:", &res_500.total.value.to_string());
@@ -877,7 +1063,8 @@ async fn battle_access_log_analytics() {
"stats": { "field": "response_time_ms" }
}
}
- })).unwrap();
+ }))
+ .unwrap();
let res2 = idx.search(&req_non200).await.unwrap();
row("Non-200 total:", &res2.total.value.to_string());
@@ -897,15 +1084,29 @@ async fn battle_access_log_analytics() {
},
"size": 50,
"sort": [{ "response_time_ms": "desc" }]
- })).unwrap();
+ }))
+ .unwrap();
let res3 = idx.search(&req_slow).await.unwrap();
row("Slow requests (>1000ms):", &res3.total.value.to_string());
// Verify sort order descending
if res3.hits.len() >= 2 {
- let t0 = res3.hits[0].source.get("response_time_ms").and_then(|v| v.as_u64()).unwrap_or(0);
- let t1 = res3.hits[1].source.get("response_time_ms").and_then(|v| v.as_u64()).unwrap_or(0);
- assert!(t0 >= t1, "response_time_ms sort desc violated: {} < {}", t0, t1);
+ let t0 = res3.hits[0]
+ .source
+ .get("response_time_ms")
+ .and_then(|v| v.as_u64())
+ .unwrap_or(0);
+ let t1 = res3.hits[1]
+ .source
+ .get("response_time_ms")
+ .and_then(|v| v.as_u64())
+ .unwrap_or(0);
+ assert!(
+ t0 >= t1,
+ "response_time_ms sort desc violated: {} < {}",
+ t0,
+ t1
+ );
row("Sort correctness (response_time desc):", "OK");
}
@@ -917,10 +1118,17 @@ async fn battle_access_log_analytics() {
let req_exists = parse_request(&json!({
"query": { "exists": { "field": "request_id" } },
"size": 0
- })).unwrap();
+ }))
+ .unwrap();
let res4 = idx.search(&req_exists).await.unwrap();
- row("Docs with request_id field (exists):", &res4.total.value.to_string());
- assert_eq!(res4.total.value, indexed as u64, "All docs should have request_id");
+ row(
+ "Docs with request_id field (exists):",
+ &res4.total.value.to_string(),
+ );
+ assert_eq!(
+ res4.total.value, indexed as u64,
+ "All docs should have request_id"
+ );
let lats4 = measure_search!(idx, req_exists, RUNS);
lat_rows("Exists query", &lats4);
@@ -941,10 +1149,13 @@ async fn battle_access_log_analytics() {
"stats": { "field": "response_time_ms" }
}
}
- })).unwrap();
+ }))
+ .unwrap();
let res5 = idx.search(&req_method_breakdown).await.unwrap();
let aggs5 = res5.aggs.as_ref().expect("aggs present");
- let method_buckets = aggs5["by_method"]["buckets"].as_array().expect("method buckets");
+ let method_buckets = aggs5["by_method"]["buckets"]
+ .as_array()
+ .expect("method buckets");
row("HTTP method buckets:", &method_buckets.len().to_string());
for b in method_buckets.iter().take(5) {
let key = b["key"].as_str().unwrap_or("?");
@@ -967,26 +1178,42 @@ async fn battle_correctness_spot_checks() {
let access_docs = match load_ndjson("access_logs.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let product_docs = match load_ndjson("products.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let error_docs = match load_ndjson("error_logs.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let article_docs = match load_ndjson("articles.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let (engine, _dir) = battle_engine();
engine.create_index("chk-access", Schema::empty()).unwrap();
- engine.create_index("chk-products", Schema::empty()).unwrap();
+ engine
+ .create_index("chk-products", Schema::empty())
+ .unwrap();
engine.create_index("chk-errors", Schema::empty()).unwrap();
- engine.create_index("chk-articles", Schema::empty()).unwrap();
+ engine
+ .create_index("chk-articles", Schema::empty())
+ .unwrap();
let (an, _) = bulk_index(&engine, "chk-access", &access_docs).await;
let (pn, _) = bulk_index(&engine, "chk-products", &product_docs).await;
@@ -1001,88 +1228,211 @@ async fn battle_correctness_spot_checks() {
// ── Access logs ────────────────────────────────────────────────────────────
// match_all must return the full dataset count
- let r = acc.search(&parse_request(&json!({"query":{"match_all":{}},"size":0})).unwrap()).await.unwrap();
- assert_eq!(r.total.value, an as u64, "access-logs: match_all count mismatch");
- row("access-logs match_all total:", &format!("{} (PASS)", r.total.value));
+ let r = acc
+ .search(&parse_request(&json!({"query":{"match_all":{}},"size":0})).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(
+ r.total.value, an as u64,
+ "access-logs: match_all count mismatch"
+ );
+ row(
+ "access-logs match_all total:",
+ &format!("{} (PASS)", r.total.value),
+ );
// 200 + non-200 must equal total
- let r200 = acc.search(&parse_request(&json!({"query":{"term":{"status":200}},"size":0})).unwrap()).await.unwrap();
- let rnon = acc.search(&parse_request(&json!({"query":{"bool":{"must_not":[{"term":{"status":200}}]}},"size":0})).unwrap()).await.unwrap();
- assert_eq!(r200.total.value + rnon.total.value, an as u64, "200 + non-200 != total");
+ let r200 = acc
+ .search(&parse_request(&json!({"query":{"term":{"status":200}},"size":0})).unwrap())
+ .await
+ .unwrap();
+ let rnon = acc
+ .search(
+ &parse_request(
+ &json!({"query":{"bool":{"must_not":[{"term":{"status":200}}]}},"size":0}),
+ )
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ r200.total.value + rnon.total.value,
+ an as u64,
+ "200 + non-200 != total"
+ );
row("access-logs 200+non200==total:", "PASS");
// ── Products ───────────────────────────────────────────────────────────────
// match_all
- let r = prod.search(&parse_request(&json!({"query":{"match_all":{}},"size":0})).unwrap()).await.unwrap();
- assert_eq!(r.total.value, pn as u64, "products: match_all count mismatch");
- row("products match_all total:", &format!("{} (PASS)", r.total.value));
+ let r = prod
+ .search(&parse_request(&json!({"query":{"match_all":{}},"size":0})).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(
+ r.total.value, pn as u64,
+ "products: match_all count mismatch"
+ );
+ row(
+ "products match_all total:",
+ &format!("{} (PASS)", r.total.value),
+ );
// in_stock=true + in_stock=false == total
- let rin = prod.search(&parse_request(&json!({"query":{"term":{"in_stock":true}},"size":0})).unwrap()).await.unwrap();
- let rout = prod.search(&parse_request(&json!({"query":{"term":{"in_stock":false}},"size":0})).unwrap()).await.unwrap();
- assert_eq!(rin.total.value + rout.total.value, pn as u64, "in_stock true+false != total");
+ let rin = prod
+ .search(&parse_request(&json!({"query":{"term":{"in_stock":true}},"size":0})).unwrap())
+ .await
+ .unwrap();
+ let rout = prod
+ .search(&parse_request(&json!({"query":{"term":{"in_stock":false}},"size":0})).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(
+ rin.total.value + rout.total.value,
+ pn as u64,
+ "in_stock true+false != total"
+ );
row("products in_stock true+false==total:", "PASS");
// price range [0, ∞) == total
- let rprice = prod.search(&parse_request(&json!({"query":{"range":{"price":{"gte":0}}},"size":0})).unwrap()).await.unwrap();
- assert_eq!(rprice.total.value, pn as u64, "products: price range gte 0 != total");
+ let rprice = prod
+ .search(&parse_request(&json!({"query":{"range":{"price":{"gte":0}}},"size":0})).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(
+ rprice.total.value, pn as u64,
+ "products: price range gte 0 != total"
+ );
row("products price >=0 range==total:", "PASS");
// rating agg avg within [1,5]
let ragg = prod.search(&parse_request(&json!({"query":{"match_all":{}},"size":0,"aggs":{"avg_r":{"avg":{"field":"rating"}}}})).unwrap()).await.unwrap();
- if let Some(avg) = ragg.aggs.as_ref().and_then(|a| a["avg_r"]["value"].as_f64()) {
- assert!((1.0..=5.0).contains(&avg), "avg rating {} out of [1,5]", avg);
- row("products avg rating in [1,5]:", &format!("{:.2} (PASS)", avg));
+ if let Some(avg) = ragg
+ .aggs
+ .as_ref()
+ .and_then(|a| a["avg_r"]["value"].as_f64())
+ {
+ assert!(
+ (1.0..=5.0).contains(&avg),
+ "avg rating {} out of [1,5]",
+ avg
+ );
+ row(
+ "products avg rating in [1,5]:",
+ &format!("{:.2} (PASS)", avg),
+ );
}
// ── Error logs ─────────────────────────────────────────────────────────────
- let r = err.search(&parse_request(&json!({"query":{"match_all":{}},"size":0})).unwrap()).await.unwrap();
- assert_eq!(r.total.value, en as u64, "error-logs: match_all count mismatch");
- row("error-logs match_all total:", &format!("{} (PASS)", r.total.value));
+ let r = err
+ .search(&parse_request(&json!({"query":{"match_all":{}},"size":0})).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(
+ r.total.value, en as u64,
+ "error-logs: match_all count mismatch"
+ );
+ row(
+ "error-logs match_all total:",
+ &format!("{} (PASS)", r.total.value),
+ );
// All log levels must sum to total
let levels = ["INFO", "WARN", "ERROR", "FATAL"];
let mut level_sum = 0u64;
for level in &levels {
- let rl = err.search(&parse_request(&json!({"query":{"term":{"level":level}},"size":0})).unwrap()).await.unwrap();
+ let rl = err
+ .search(&parse_request(&json!({"query":{"term":{"level":level}},"size":0})).unwrap())
+ .await
+ .unwrap();
level_sum += rl.total.value;
- row(&format!("error-logs level={}:", level), &rl.total.value.to_string());
+ row(
+ &format!("error-logs level={}:", level),
+ &rl.total.value.to_string(),
+ );
}
- assert_eq!(level_sum, en as u64, "level sum {} != total {}", level_sum, en);
+ assert_eq!(
+ level_sum, en as u64,
+ "level sum {} != total {}",
+ level_sum, en
+ );
row("error-logs sum(levels)==total:", "PASS");
// terms agg by level — bucket sum must equal total
- let ragg = err.search(&parse_request(&json!({
- "query": { "match_all": {} },
- "size": 0,
- "aggs": { "by_level": { "terms": { "field": "level", "size": 10 } } }
- })).unwrap()).await.unwrap();
- let agg_sum: u64 = ragg.aggs.as_ref()
+ let ragg = err
+ .search(
+ &parse_request(&json!({
+ "query": { "match_all": {} },
+ "size": 0,
+ "aggs": { "by_level": { "terms": { "field": "level", "size": 10 } } }
+ }))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let agg_sum: u64 = ragg
+ .aggs
+ .as_ref()
.and_then(|a| a["by_level"]["buckets"].as_array())
- .map(|b| b.iter().map(|bk| bk["doc_count"].as_u64().unwrap_or(0)).sum())
+ .map(|b| {
+ b.iter()
+ .map(|bk| bk["doc_count"].as_u64().unwrap_or(0))
+ .sum()
+ })
.unwrap_or(0);
- row("error-logs terms agg bucket sum:", &format!("{} vs {} total", agg_sum, en));
+ row(
+ "error-logs terms agg bucket sum:",
+ &format!("{} vs {} total", agg_sum, en),
+ );
// ── Articles ───────────────────────────────────────────────────────────────
- let r = art.search(&parse_request(&json!({"query":{"match_all":{}},"size":0})).unwrap()).await.unwrap();
- assert_eq!(r.total.value, artn as u64, "articles: match_all count mismatch");
- row("articles match_all total:", &format!("{} (PASS)", r.total.value));
+ let r = art
+ .search(&parse_request(&json!({"query":{"match_all":{}},"size":0})).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(
+ r.total.value, artn as u64,
+ "articles: match_all count mismatch"
+ );
+ row(
+ "articles match_all total:",
+ &format!("{} (PASS)", r.total.value),
+ );
// word_count range [0,∞) == total
- let rwc = art.search(&parse_request(&json!({"query":{"range":{"word_count":{"gte":0}}},"size":0})).unwrap()).await.unwrap();
- assert_eq!(rwc.total.value, artn as u64, "articles: word_count range != total");
+ let rwc = art
+ .search(
+ &parse_request(&json!({"query":{"range":{"word_count":{"gte":0}}},"size":0})).unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ rwc.total.value, artn as u64,
+ "articles: word_count range != total"
+ );
row("articles word_count >=0 == total:", "PASS");
// FTS hits for a common word that should appear in all articles
- let rfts = art.search(&parse_request(&json!({
- "query": { "match": { "body": "performance" } },
- "size": 0
- })).unwrap()).await.unwrap();
- row("articles FTS 'performance' hits:", &rfts.total.value.to_string());
- assert!(rfts.total.value > 0, "Expected FTS hits for 'performance' in articles");
+ let rfts = art
+ .search(
+ &parse_request(&json!({
+ "query": { "match": { "body": "performance" } },
+ "size": 0
+ }))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ row(
+ "articles FTS 'performance' hits:",
+ &rfts.total.value.to_string(),
+ );
+ assert!(
+ rfts.total.value > 0,
+ "Expected FTS hits for 'performance' in articles"
+ );
separator();
row("ALL CORRECTNESS CHECKS:", "PASSED");
@@ -1099,16 +1449,24 @@ async fn battle_complex_bool_queries() {
let error_docs = match load_ndjson("error_logs.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let product_docs = match load_ndjson("products.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let (engine, _dir) = battle_engine();
engine.create_index("bool-errors", Schema::empty()).unwrap();
- engine.create_index("bool-products", Schema::empty()).unwrap();
+ engine
+ .create_index("bool-products", Schema::empty())
+ .unwrap();
let (en, _) = bulk_index(&engine, "bool-errors", &error_docs).await;
let (pn, _) = bulk_index(&engine, "bool-products", &product_docs).await;
@@ -1144,9 +1502,13 @@ async fn battle_complex_bool_queries() {
{ "duration_ms": "desc" },
{ "_score": "desc" }
]
- })).unwrap();
+ }))
+ .unwrap();
let res = err_idx.search(&req_complex_log).await.unwrap();
- row("Complex log bool (database+service filter):", &res.total.value.to_string());
+ row(
+ "Complex log bool (database+service filter):",
+ &res.total.value.to_string(),
+ );
let lats = measure_search!(err_idx, req_complex_log, RUNS);
lat_rows("Complex bool (must+filter+must_not+sort)", &lats);
@@ -1173,9 +1535,13 @@ async fn battle_complex_bool_queries() {
}
},
"size": 50
- })).unwrap();
+ }))
+ .unwrap();
let res2 = err_idx.search(&req_nested_bool).await.unwrap();
- row("Nested bool (connection OR timeout, ERROR/FATAL):", &res2.total.value.to_string());
+ row(
+ "Nested bool (connection OR timeout, ERROR/FATAL):",
+ &res2.total.value.to_string(),
+ );
let lats2 = measure_search!(err_idx, req_nested_bool, RUNS);
lat_rows("Nested bool (should inside must)", &lats2);
@@ -1203,14 +1569,26 @@ async fn battle_complex_bool_queries() {
},
"size": 20,
"sort": [{ "rating": "desc" }, { "price": "asc" }]
- })).unwrap();
+ }))
+ .unwrap();
let res3 = prod_idx.search(&req_product_complex).await.unwrap();
- row("Product complex bool (quality+rating+price):", &res3.total.value.to_string());
+ row(
+ "Product complex bool (quality+rating+price):",
+ &res3.total.value.to_string(),
+ );
// Sort verification: rating descending
if res3.hits.len() >= 2 {
- let r0 = res3.hits[0].source.get("rating").and_then(|v| v.as_f64()).unwrap_or(0.0);
- let r1 = res3.hits[1].source.get("rating").and_then(|v| v.as_f64()).unwrap_or(0.0);
+ let r0 = res3.hits[0]
+ .source
+ .get("rating")
+ .and_then(|v| v.as_f64())
+ .unwrap_or(0.0);
+ let r1 = res3.hits[1]
+ .source
+ .get("rating")
+ .and_then(|v| v.as_f64())
+ .unwrap_or(0.0);
assert!(r0 >= r1, "rating sort desc violated: {} < {}", r0, r1);
row("Sort correctness (rating desc, price asc):", "OK");
}
@@ -1224,12 +1602,14 @@ async fn battle_complex_bool_queries() {
"query": { "match_all": {} },
"from": 0, "size": 10,
"sort": [{ "price": "asc" }]
- })).unwrap();
+ }))
+ .unwrap();
let req_page2 = parse_request(&json!({
"query": { "match_all": {} },
"from": 10, "size": 10,
"sort": [{ "price": "asc" }]
- })).unwrap();
+ }))
+ .unwrap();
let p1 = prod_idx.search(&req_page1).await.unwrap();
let p2 = prod_idx.search(&req_page2).await.unwrap();
@@ -1256,16 +1636,24 @@ async fn battle_concurrent_search_load() {
let error_docs = match load_ndjson("error_logs.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let product_docs = match load_ndjson("products.ndjson") {
Ok(d) => d,
- Err(e) => { println!("SKIP: {}", e); return; }
+ Err(e) => {
+ println!("SKIP: {}", e);
+ return;
+ }
};
let (engine, _dir) = battle_engine();
engine.create_index("conc-errors", Schema::empty()).unwrap();
- engine.create_index("conc-products", Schema::empty()).unwrap();
+ engine
+ .create_index("conc-products", Schema::empty())
+ .unwrap();
let (en, _) = bulk_index(&engine, "conc-errors", &error_docs).await;
let (pn, _) = bulk_index(&engine, "conc-products", &product_docs).await;
@@ -1314,7 +1702,7 @@ async fn battle_concurrent_search_load() {
let t0 = Instant::now();
// Alternate between the two indices
- if qi % 2 == 0 {
+ if qi.is_multiple_of(2) {
let _ = err_c.search(&req).await;
} else {
let _ = prd_c.search(&req).await;
diff --git a/engine/crates/xerj-engine/tests/chaos_tests.rs b/engine/crates/xerj-engine/tests/chaos_tests.rs
index a63c4e360..308346a4e 100644
--- a/engine/crates/xerj-engine/tests/chaos_tests.rs
+++ b/engine/crates/xerj-engine/tests/chaos_tests.rs
@@ -57,17 +57,9 @@ fn now_ts() -> String {
fn print_header(test_name: &str) {
println!();
- println!(
- "┌─────────────────────────────────────────────────────────────────┐"
- );
- println!(
- "│ [{ts}] {name:<52}│",
- ts = now_ts(),
- name = test_name
- );
- println!(
- "└─────────────────────────────────────────────────────────────────┘"
- );
+ println!("┌─────────────────────────────────────────────────────────────────┐");
+ println!("│ [{ts}] {name:<52}│", ts = now_ts(), name = test_name);
+ println!("└─────────────────────────────────────────────────────────────────┘");
}
fn print_summary(results: &[TestResult]) {
@@ -178,9 +170,7 @@ async fn chaos_crash_during_writes() {
// Phase 1: index N docs then "crash" (drop without flush).
{
let engine = make_engine(&dir);
- engine
- .create_index("crash_idx", Schema::empty())
- .unwrap();
+ engine.create_index("crash_idx", Schema::empty()).unwrap();
let idx = engine.get_index("crash_idx").unwrap();
for i in 0..crash_after {
idx.index_document(
@@ -204,9 +194,13 @@ async fn chaos_crash_during_writes() {
let elapsed_ms = iter_start.elapsed().as_millis();
// Allow up to 0.5% loss for batched WAL sync (same as ES translog batched mode)
- let min_recovered = (crash_after as f64 * 0.995) as u64;
- let ok = recovered >= min_recovered;
- if ok { pass += 1; } else { fail += 1; }
+ let min_recovered = (crash_after as f64 * 0.995) as u64;
+ let ok = recovered >= min_recovered;
+ if ok {
+ pass += 1;
+ } else {
+ fail += 1;
+ }
let rate = (recovered as f64 / crash_after as f64) * 100.0;
println!(
@@ -219,7 +213,11 @@ async fn chaos_crash_during_writes() {
rec = recovered,
rate = rate,
ms = elapsed_ms,
- status = if ok { "PASS" } else { "FAIL ← data loss detected" },
+ status = if ok {
+ "PASS"
+ } else {
+ "FAIL ← data loss detected"
+ },
);
}
}
@@ -245,14 +243,16 @@ async fn chaos_crash_during_writes() {
drop(engine);
}
let engine2 = make_engine_at(dir.path());
- let idx2 = engine2
- .get_index("crash_idx")
- .expect("index must reopen");
+ let idx2 = engine2.get_index("crash_idx").expect("index must reopen");
let recovered = count_all(&idx2).await;
// Allow up to 0.5% loss for batched WAL sync (same as ES translog batched mode)
let min_recovered = (crash_after as f64 * 0.995) as u64;
let ok = recovered >= min_recovered;
- if ok { pass += 1; } else { fail += 1; }
+ if ok {
+ pass += 1;
+ } else {
+ fail += 1;
+ }
println!(
" [{ts}] iter={it:>2} crash_after={n:>6} recovered={rec} \
elapsed={ms}ms {status}",
@@ -299,6 +299,9 @@ async fn chaos_wal_corruption() {
apply: fn(&mut Vec<u8>),
}
+ // ptr_arg: signature must stay `fn(&mut Vec<u8>)` to match the CorruptCase.apply
+ // field type shared with corrupt_truncate50, which needs Vec::truncate.
+ #[allow(clippy::ptr_arg)]
fn corrupt_last10(data: &mut Vec<u8>) {
let len = data.len();
if len >= 10 {
@@ -307,6 +310,9 @@ async fn chaos_wal_corruption() {
}
}
}
+ // ptr_arg: signature must stay `fn(&mut Vec<u8>)` to match the CorruptCase.apply
+ // field type shared with corrupt_truncate50, which needs Vec::truncate.
+ #[allow(clippy::ptr_arg)]
fn corrupt_middle(data: &mut Vec<u8>) {
let mid = data.len() / 2;
let end = (mid + 64).min(data.len());
@@ -314,6 +320,9 @@ async fn chaos_wal_corruption() {
*b ^= 0xAA;
}
}
+ // ptr_arg: signature must stay `fn(&mut Vec<u8>)` to match the CorruptCase.apply
+ // field type shared with corrupt_truncate50, which needs Vec::truncate.
+ #[allow(clippy::ptr_arg)]
fn corrupt_header(data: &mut Vec<u8>) {
// Overwrite the 4-byte "ZWAL" magic at offset 0.
if data.len() >= 4 {
@@ -327,6 +336,9 @@ async fn chaos_wal_corruption() {
let half = data.len() / 2;
data.truncate(half);
}
+ // ptr_arg: signature must stay `fn(&mut Vec<u8>)` to match the CorruptCase.apply
+ // field type shared with corrupt_truncate50, which needs Vec::truncate.
+ #[allow(clippy::ptr_arg)]
fn corrupt_zero4kb(data: &mut Vec<u8>) {
let start = (data.len() / 4).min(data.len().saturating_sub(4096));
let end = (start + 4096).min(data.len());
@@ -336,11 +348,26 @@ async fn chaos_wal_corruption() {
}
let cases: &[CorruptCase] = &[
- CorruptCase { name: "last-10-bytes", apply: corrupt_last10 },
- CorruptCase { name: "middle-64-bytes", apply: corrupt_middle },
- CorruptCase { name: "header-magic", apply: corrupt_header },
- CorruptCase { name: "truncate-50pct", apply: corrupt_truncate50 },
- CorruptCase { name: "zero-4kb-block", apply: corrupt_zero4kb },
+ CorruptCase {
+ name: "last-10-bytes",
+ apply: corrupt_last10,
+ },
+ CorruptCase {
+ name: "middle-64-bytes",
+ apply: corrupt_middle,
+ },
+ CorruptCase {
+ name: "header-magic",
+ apply: corrupt_header,
+ },
+ CorruptCase {
+ name: "truncate-50pct",
+ apply: corrupt_truncate50,
+ },
+ CorruptCase {
+ name: "zero-4kb-block",
+ apply: corrupt_zero4kb,
+ },
];
for case in cases {
@@ -371,12 +398,7 @@ async fn chaos_wal_corruption() {
.ok()
.and_then(|rd| {
rd.flatten()
- .find(|e| {
- e.path()
- .extension()
- .map(|x| x == "wal")
- .unwrap_or(false)
- })
+ .find(|e| e.path().extension().map(|x| x == "wal").unwrap_or(false))
})
.map(|e| e.path());
@@ -422,7 +444,11 @@ async fn chaos_wal_corruption() {
// Pass = engine opened without panic (data loss is acceptable for some corruption types).
let ok = engine_opened;
- if ok { pass += 1; } else { fail += 1; }
+ if ok {
+ pass += 1;
+ } else {
+ fail += 1;
+ }
println!(
" [{ts}] iter={it} corruption={name:<20} opened={op} \
@@ -435,7 +461,11 @@ async fn chaos_wal_corruption() {
total = doc_count,
loss = loss_pct,
ms = iter_start.elapsed().as_millis(),
- status = if ok { "PASS" } else { "FAIL ← engine crashed/panicked" },
+ status = if ok {
+ "PASS"
+ } else {
+ "FAIL ← engine crashed/panicked"
+ },
);
}
@@ -473,9 +503,7 @@ async fn chaos_rapid_restart_loop() {
// Seed data and flush so it lives in a segment (survives WAL-free reopens).
{
let engine = make_engine(&dir);
- engine
- .create_index("restart_idx", Schema::empty())
- .unwrap();
+ engine.create_index("restart_idx", Schema::empty()).unwrap();
let idx = engine.get_index("restart_idx").unwrap();
for i in 0..initial_docs {
idx.index_document(
@@ -505,15 +533,18 @@ async fn chaos_rapid_restart_loop() {
// Use match_all — works across both memtable and flushed segments
let search_req =
- parse_request(&json!({ "query": { "match_all": {} }, "size": 5 }))
- .unwrap();
+ parse_request(&json!({ "query": { "match_all": {} }, "size": 5 })).unwrap();
let search_result = idx.search(&search_req).await.unwrap();
- let search_works = search_result.total.value > 0 && search_result.hits.len() > 0;
+ let search_works = search_result.total.value > 0 && !search_result.hits.is_empty();
let restart_ms = iter_start.elapsed().as_millis();
let iter_ok = doc_count == initial_docs && search_works;
- if iter_ok { pass += 1; } else { fail += 1; }
+ if iter_ok {
+ pass += 1;
+ } else {
+ fail += 1;
+ }
// Print every 10th iteration or on failure.
if i % 10 == 0 || !iter_ok {
@@ -565,9 +596,7 @@ async fn chaos_concurrent_readwrite_stress() {
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine
- .create_index("stress_idx", Schema::empty())
- .unwrap();
+ engine.create_index("stress_idx", Schema::empty()).unwrap();
let idx = Arc::new(engine.get_index("stress_idx").unwrap());
let write_errors = Arc::new(AtomicU64::new(0));
@@ -601,9 +630,8 @@ async fn chaos_concurrent_readwrite_stress() {
}
// Spawn readers.
- let search_req = Arc::new(
- parse_request(&json!({ "query": { "match_all": {} }, "size": 1 })).unwrap(),
- );
+ let search_req =
+ Arc::new(parse_request(&json!({ "query": { "match_all": {} }, "size": 1 })).unwrap());
let mut reader_handles = Vec::new();
for _ in 0..reader_tasks {
let idx_c = Arc::clone(&idx);
@@ -613,15 +641,23 @@ async fn chaos_concurrent_readwrite_stress() {
reader_handles.push(tokio::spawn(async move {
for _ in 0..searches_per_reader {
match idx_c.search(&req_c).await {
- Ok(_) => { ok_c.fetch_add(1, Ordering::Relaxed); }
- Err(_) => { err_c.fetch_add(1, Ordering::Relaxed); }
+ Ok(_) => {
+ ok_c.fetch_add(1, Ordering::Relaxed);
+ }
+ Err(_) => {
+ err_c.fetch_add(1, Ordering::Relaxed);
+ }
}
}
}));
}
- for h in writer_handles { h.await.expect("writer task panicked"); }
- for h in reader_handles { h.await.expect("reader task panicked"); }
+ for h in writer_handles {
+ h.await.expect("writer task panicked");
+ }
+ for h in reader_handles {
+ h.await.expect("reader task panicked");
+ }
let expected_docs = (writer_tasks * docs_per_writer) as u64;
let actual_docs = count_all(&idx).await;
@@ -635,15 +671,23 @@ async fn chaos_concurrent_readwrite_stress() {
println!(
" [{ts}] total_writes={tw} actual_docs={ad} write_errors={we}",
- ts = now_ts(), tw = expected_docs, ad = actual_docs, we = w_err
+ ts = now_ts(),
+ tw = expected_docs,
+ ad = actual_docs,
+ we = w_err
);
println!(
" [{ts}] reads_ok={rok} read_errors={re} duration={d:.1}s",
- ts = now_ts(), rok = r_ok, re = r_err, d = duration.as_secs_f64()
+ ts = now_ts(),
+ rok = r_ok,
+ re = r_err,
+ d = duration.as_secs_f64()
);
println!(
" [{ts}] write_throughput={wt:.0} docs/s read_throughput={rt:.0} searches/s",
- ts = now_ts(), wt = write_tput, rt = read_tput
+ ts = now_ts(),
+ wt = write_tput,
+ rt = read_tput
);
let all_ok = actual_docs == expected_docs && w_err == 0 && r_err == 0;
@@ -680,9 +724,7 @@ async fn chaos_large_documents() {
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine
- .create_index("large_idx", Schema::empty())
- .unwrap();
+ engine.create_index("large_idx", Schema::empty()).unwrap();
let idx = engine.get_index("large_idx").unwrap();
let sizes: &[(usize, &str)] = &[
@@ -724,7 +766,11 @@ async fn chaos_large_documents() {
.unwrap_or(false);
let iter_ok = index_ok && source_verified;
- if iter_ok { pass += 1; } else { fail += 1; }
+ if iter_ok {
+ pass += 1;
+ } else {
+ fail += 1;
+ }
println!(
" [{ts}] size={label:<6} bytes={b:>9} index_ms={im:>6} \
@@ -777,9 +823,7 @@ async fn chaos_flush_during_writes() {
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine
- .create_index("flush_idx", Schema::empty())
- .unwrap();
+ engine.create_index("flush_idx", Schema::empty()).unwrap();
let idx = engine.get_index("flush_idx").unwrap();
let mut flush_count = 0u64;
@@ -831,9 +875,8 @@ async fn chaos_flush_during_writes() {
ar = docs_after,
);
- let all_ok = flush_count == expected_flushes
- && docs_before == total_docs
- && docs_after == total_docs;
+ let all_ok =
+ flush_count == expected_flushes && docs_before == total_docs && docs_after == total_docs;
println!(
" RESULT: {} duration={d:.1}s",
if all_ok { "PASS" } else { "FAIL" },
@@ -846,7 +889,11 @@ async fn chaos_flush_during_writes() {
fail: if all_ok { 0 } else { 1 },
duration,
});
- assert_eq!(flush_count, expected_flushes, "expected {} flushes", expected_flushes);
+ assert_eq!(
+ flush_count, expected_flushes,
+ "expected {} flushes",
+ expected_flushes
+ );
assert_eq!(docs_before, total_docs, "docs before restart mismatch");
assert_eq!(docs_after, total_docs, "docs after restart mismatch");
}
@@ -922,7 +969,12 @@ async fn chaos_delete_during_writes() {
let even_spot_check = [0u64, 2, 100, 500, 1998];
let mut even_present = 0u64;
for &n in &even_spot_check {
- if idx.get_document(&format!("orig-{}", n)).await.unwrap().is_some() {
+ if idx
+ .get_document(&format!("orig-{}", n))
+ .await
+ .unwrap()
+ .is_some()
+ {
even_present += 1;
}
}
@@ -931,7 +983,12 @@ async fn chaos_delete_during_writes() {
let odd_spot_check = [1u64, 3, 101, 501, 1999];
let mut odd_absent = 0u64;
for &n in &odd_spot_check {
- if idx.get_document(&format!("orig-{}", n)).await.unwrap().is_none() {
+ if idx
+ .get_document(&format!("orig-{}", n))
+ .await
+ .unwrap()
+ .is_none()
+ {
odd_absent += 1;
}
}
@@ -979,8 +1036,16 @@ async fn chaos_delete_during_writes() {
fail: u64::from(!all_ok),
duration,
});
- assert_eq!(even_present, even_spot_check.len() as u64, "even originals missing");
- assert_eq!(odd_absent, odd_spot_check.len() as u64, "odd docs not deleted");
+ assert_eq!(
+ even_present,
+ even_spot_check.len() as u64,
+ "even originals missing"
+ );
+ assert_eq!(
+ odd_absent,
+ odd_spot_check.len() as u64,
+ "odd docs not deleted"
+ );
assert!(new_doc_ok, "new doc missing after concurrent writes");
}
@@ -1004,15 +1069,16 @@ async fn chaos_memory_pressure() {
// ~1KB body per doc.
let body_1kb: String = "abcdefghij".repeat(102);
- println!(" Indexing {} docs (~1KB each = ~{}MB total)…", doc_count, doc_count / 1024);
+ println!(
+ " Indexing {} docs (~1KB each = ~{}MB total)…",
+ doc_count,
+ doc_count / 1024
+ );
let ingest_start = Instant::now();
for i in 0..doc_count {
- idx.index_document(
- Some(format!("m{}", i)),
- json!({ "n": i, "body": body_1kb }),
- )
- .await
- .unwrap();
+ idx.index_document(Some(format!("m{}", i)), json!({ "n": i, "body": body_1kb }))
+ .await
+ .unwrap();
if i % 10_000 == 0 && i > 0 {
println!(
" [{ts}] ingested {i}/{doc_count} rss={rss:.0}MB",
@@ -1028,7 +1094,11 @@ async fn chaos_memory_pressure() {
);
let peak_rss_mb = read_rss_mb();
- println!(" [{ts}] peak_rss_mb={rss:.0}", ts = now_ts(), rss = peak_rss_mb);
+ println!(
+ " [{ts}] peak_rss_mb={rss:.0}",
+ ts = now_ts(),
+ rss = peak_rss_mb
+ );
// 100 concurrent aggregation queries.
let idx_arc = Arc::new(idx);
@@ -1052,12 +1122,18 @@ async fn chaos_memory_pressure() {
let ok_c = Arc::clone(&q_ok);
handles.push(tokio::spawn(async move {
match idx_c.search(&req_c).await {
- Ok(_) => { ok_c.fetch_add(1, Ordering::Relaxed); }
- Err(_) => { err_c.fetch_add(1, Ordering::Relaxed); }
+ Ok(_) => {
+ ok_c.fetch_add(1, Ordering::Relaxed);
+ }
+ Err(_) => {
+ err_c.fetch_add(1, Ordering::Relaxed);
+ }
}
}));
}
- for h in handles { h.await.expect("query task panicked"); }
+ for h in handles {
+ h.await.expect("query task panicked");
+ }
let queries_ok = q_ok.load(Ordering::Relaxed);
let queries_err = q_err.load(Ordering::Relaxed);
@@ -1089,13 +1165,20 @@ async fn chaos_memory_pressure() {
duration,
});
assert_eq!(queries_err, 0, "{} aggregation queries failed", queries_err);
- assert_eq!(queries_ok, concurrent_queries as u64, "not all queries completed");
+ assert_eq!(
+ queries_ok, concurrent_queries as u64,
+ "not all queries completed"
+ );
}
fn read_rss_mb() -> f64 {
// /proc/self/statm: size resident shared text lib data dt (pages, 4KB each).
if let Ok(s) = std::fs::read_to_string("/proc/self/statm") {
- if let Some(rss_pages) = s.split_whitespace().nth(1).and_then(|v| v.parse::<u64>().ok()) {
+ if let Some(rss_pages) = s
+ .split_whitespace()
+ .nth(1)
+ .and_then(|v| v.parse::<u64>().ok())
+ {
return (rss_pages * 4096) as f64 / (1024.0 * 1024.0);
}
}
@@ -1114,9 +1197,7 @@ async fn chaos_schema_evolution() {
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine
- .create_index("schema_idx", Schema::empty())
- .unwrap();
+ engine.create_index("schema_idx", Schema::empty()).unwrap();
let idx = engine.get_index("schema_idx").unwrap();
// Batch 1: fields {a, b, c}
@@ -1170,13 +1251,34 @@ async fn chaos_schema_evolution() {
query: Value,
}
let field_searches = [
- FieldSearch { field: "a", query: json!({"range": {"a": {"gte": 0, "lte": 100}}}) },
- FieldSearch { field: "b", query: json!({"match": {"b": "batch1"}}) },
- FieldSearch { field: "c", query: json!({"range": {"c": {"gte": 0, "lte": 5}}}) },
- FieldSearch { field: "d", query: json!({"match": {"d": "batch2"}}) },
- FieldSearch { field: "e", query: json!({"range": {"e": {"gte": 0, "lte": 50}}}) },
- FieldSearch { field: "f", query: json!({"match": {"f": "batch3"}}) },
- FieldSearch { field: "g", query: json!({"range": {"g": {"gte": 0.0, "lte": 10.0}}}) },
+ FieldSearch {
+ field: "a",
+ query: json!({"range": {"a": {"gte": 0, "lte": 100}}}),
+ },
+ FieldSearch {
+ field: "b",
+ query: json!({"match": {"b": "batch1"}}),
+ },
+ FieldSearch {
+ field: "c",
+ query: json!({"range": {"c": {"gte": 0, "lte": 5}}}),
+ },
+ FieldSearch {
+ field: "d",
+ query: json!({"match": {"d": "batch2"}}),
+ },
+ FieldSearch {
+ field: "e",
+ query: json!({"range": {"e": {"gte": 0, "lte": 50}}}),
+ },
+ FieldSearch {
+ field: "f",
+ query: json!({"match": {"f": "batch3"}}),
+ },
+ FieldSearch {
+ field: "g",
+ query: json!({"range": {"g": {"gte": 0.0, "lte": 10.0}}}),
+ },
];
let mut fields_ok = 0usize;
@@ -1186,7 +1288,11 @@ async fn chaos_schema_evolution() {
let req = parse_request(&json!({ "query": fs.query, "size": 5 })).unwrap();
let result = idx.search(&req).await.unwrap();
let ok = result.total.value > 0;
- if ok { fields_ok += 1; } else { fields_fail += 1; }
+ if ok {
+ fields_ok += 1;
+ } else {
+ fields_fail += 1;
+ }
println!(
" [{ts}] field={f:<3} hits={h:>5} {status}",
ts = now_ts(),
@@ -1223,7 +1329,11 @@ async fn chaos_schema_evolution() {
duration,
});
assert_eq!(total_docs, 1500, "expected 1500 docs");
- assert_eq!(fields_fail, 0, "schema evolution: {} fields returned no hits", fields_fail);
+ assert_eq!(
+ fields_fail, 0,
+ "schema evolution: {} fields returned no hits",
+ fields_fail
+ );
}
// ══════════════════════════════════════════════════════════════════════════════
@@ -1259,7 +1369,10 @@ async fn chaos_data_integrity() {
let dir = TempDir::new().unwrap();
// ── Phase 1: index and verify before restart ──────────────────────────────
- println!(" Phase 1: indexing {} docs and verifying checksums pre-restart…", doc_count);
+ println!(
+ " Phase 1: indexing {} docs and verifying checksums pre-restart…",
+ doc_count
+ );
let (pre_verified, pre_mismatches) = {
let engine = make_engine(&dir);
engine
@@ -1278,9 +1391,7 @@ async fn chaos_data_integrity() {
for (id, src) in &docs {
match idx.get_document(id).await {
Ok(Some(got)) => {
- if fnv64(got.to_string().as_bytes())
- == fnv64(src.to_string().as_bytes())
- {
+ if fnv64(got.to_string().as_bytes()) == fnv64(src.to_string().as_bytes()) {
v += 1;
} else {
m += 1;
@@ -1301,7 +1412,11 @@ async fn chaos_data_integrity() {
total = doc_count,
m = pre_mismatches,
);
- assert_eq!(pre_mismatches, 0, "pre-restart checksum failures: {}", pre_mismatches);
+ assert_eq!(
+ pre_mismatches, 0,
+ "pre-restart checksum failures: {}",
+ pre_mismatches
+ );
// ── Phase 2: reopen via WAL replay and re-verify ──────────────────────────
println!(" Phase 2: reopening engine (WAL replay) and re-verifying checksums…");
@@ -1338,8 +1453,7 @@ async fn chaos_data_integrity() {
miss = post_missing,
);
- let all_ok =
- post_mismatches == 0 && post_missing == 0 && post_verified == doc_count;
+ let all_ok = post_mismatches == 0 && post_missing == 0 && post_verified == doc_count;
println!(
" RESULT: {} duration={d:.1}s",
if all_ok { "PASS" } else { "FAIL" },
@@ -1363,7 +1477,18 @@ async fn chaos_data_integrity() {
print_summary(&results);
}
- assert_eq!(post_mismatches, 0, "checksum mismatches after WAL replay: {}", post_mismatches);
- assert_eq!(post_missing, 0, "docs missing after WAL replay: {}", post_missing);
- assert_eq!(post_verified, doc_count, "not all docs verified after restart");
+ assert_eq!(
+ post_mismatches, 0,
+ "checksum mismatches after WAL replay: {}",
+ post_mismatches
+ );
+ assert_eq!(
+ post_missing, 0,
+ "docs missing after WAL replay: {}",
+ post_missing
+ );
+ assert_eq!(
+ post_verified, doc_count,
+ "not all docs verified after restart"
+ );
}
diff --git a/engine/crates/xerj-engine/tests/multi_match_scoring.rs b/engine/crates/xerj-engine/tests/multi_match_scoring.rs
index 39aa9d14b..c58c9cbd1 100644
--- a/engine/crates/xerj-engine/tests/multi_match_scoring.rs
+++ b/engine/crates/xerj-engine/tests/multi_match_scoring.rs
@@ -50,34 +50,58 @@ async fn multi_match_scores_nonzero_and_best_fields_takes_field_max() {
// ── Memtable state: scores must not be zero ──────────────────────────
let r = idx
- .search(&req(json!({"multi_match": {"query": "golf", "fields": ["msg", "note"]}})))
+ .search(&req(
+ json!({"multi_match": {"query": "golf", "fields": ["msg", "note"]}}),
+ ))
.await
.unwrap();
assert_eq!(r.total.value, 9, "memtable multi_match union");
for h in &r.hits {
- assert!(h.score > 0.0, "memtable multi_match hit {} scored 0.0", h.id);
+ assert!(
+ h.score > 0.0,
+ "memtable multi_match hit {} scored 0.0",
+ h.id
+ );
}
// ── Segment state: BM25 scores, ES best_fields semantics ─────────────
idx.flush().await.unwrap();
- let match_msg = idx.search(&req(json!({"match": {"msg": "golf"}}))).await.unwrap();
- let match_note = idx.search(&req(json!({"match": {"note": "golf"}}))).await.unwrap();
- let msg_scores: HashMap<&str, f32> =
- match_msg.hits.iter().map(|h| (h.id.as_str(), h.score)).collect();
- let note_scores: HashMap<&str, f32> =
- match_note.hits.iter().map(|h| (h.id.as_str(), h.score)).collect();
+ let match_msg = idx
+ .search(&req(json!({"match": {"msg": "golf"}})))
+ .await
+ .unwrap();
+ let match_note = idx
+ .search(&req(json!({"match": {"note": "golf"}})))
+ .await
+ .unwrap();
+ let msg_scores: HashMap<&str, f32> = match_msg
+ .hits
+ .iter()
+ .map(|h| (h.id.as_str(), h.score))
+ .collect();
+ let note_scores: HashMap<&str, f32> = match_note
+ .hits
+ .iter()
+ .map(|h| (h.id.as_str(), h.score))
+ .collect();
assert!(match_msg.hits.iter().all(|h| h.score > 0.0));
// Single-field multi_match must score IDENTICALLY to the match query.
let mm_single = idx
- .search(&req(json!({"multi_match": {"query": "golf", "fields": ["msg"]}})))
+ .search(&req(
+ json!({"multi_match": {"query": "golf", "fields": ["msg"]}}),
+ ))
.await
.unwrap();
assert_eq!(mm_single.total.value, match_msg.total.value);
for h in &mm_single.hits {
let expected = msg_scores.get(h.id.as_str()).copied().unwrap_or(0.0);
- assert!(h.score > 0.0, "single-field multi_match hit {} scored 0.0", h.id);
+ assert!(
+ h.score > 0.0,
+ "single-field multi_match hit {} scored 0.0",
+ h.id
+ );
assert!(
(h.score - expected).abs() < 1e-5,
"hit {}: multi_match(msg) score {} != match(msg) score {}",
@@ -89,7 +113,9 @@ async fn multi_match_scores_nonzero_and_best_fields_takes_field_max() {
// Multi-field best_fields: score = max(per-field match scores).
let mm = idx
- .search(&req(json!({"multi_match": {"query": "golf", "fields": ["msg", "note"]}})))
+ .search(&req(
+ json!({"multi_match": {"query": "golf", "fields": ["msg", "note"]}}),
+ ))
.await
.unwrap();
assert_eq!(mm.total.value, 9, "segment multi_match union");
@@ -97,7 +123,11 @@ async fn multi_match_scores_nonzero_and_best_fields_takes_field_max() {
let ms = msg_scores.get(h.id.as_str()).copied().unwrap_or(0.0);
let ns = note_scores.get(h.id.as_str()).copied().unwrap_or(0.0);
let expected = ms.max(ns);
- assert!(h.score > 0.0, "multi-field multi_match hit {} scored 0.0", h.id);
+ assert!(
+ h.score > 0.0,
+ "multi-field multi_match hit {} scored 0.0",
+ h.id
+ );
assert!(
(h.score - expected).abs() < 1e-5,
"hit {}: best_fields score {} != max(msg {}, note {})",
diff --git a/engine/crates/xerj-engine/tests/perf_benchmark.rs b/engine/crates/xerj-engine/tests/perf_benchmark.rs
index d306383af..5e3de696e 100644
--- a/engine/crates/xerj-engine/tests/perf_benchmark.rs
+++ b/engine/crates/xerj-engine/tests/perf_benchmark.rs
@@ -40,7 +40,15 @@ fn bench_engine() -> (Engine, TempDir) {
/// Generate a realistic document body for document index `i`.
fn make_doc(i: usize) -> Value {
- let categories = ["technology", "science", "politics", "sports", "arts", "health", "finance"];
+ let categories = [
+ "technology",
+ "science",
+ "politics",
+ "sports",
+ "arts",
+ "health",
+ "finance",
+ ];
let cat = categories[i % categories.len()];
let score = (i % 100) as f64 / 10.0;
json!({
@@ -60,7 +68,7 @@ fn make_doc(i: usize) -> Value {
"category": cat,
"score": score,
"tags": [cat, format!("tag{}", i % 10)],
- "published": i % 3 != 0,
+ "published": !i.is_multiple_of(3),
"price": (i % 500) as f64 + 0.99,
})
}
@@ -154,12 +162,27 @@ async fn perf_search_latency() {
.unwrap();
}
let index_time = index_start.elapsed();
- print_kv("Index phase:", &format!("{:.2?} ({:.0} docs/sec)", index_time, INDEX_N as f64 / index_time.as_secs_f64()));
+ print_kv(
+ "Index phase:",
+ &format!(
+ "{:.2?} ({:.0} docs/sec)",
+ index_time,
+ INDEX_N as f64 / index_time.as_secs_f64()
+ ),
+ );
// Query terms cycling over categories and body keywords
let terms = [
- "technology", "science", "politics", "sports", "arts",
- "alpha", "beta", "gamma", "delta", "omega",
+ "technology",
+ "science",
+ "politics",
+ "sports",
+ "arts",
+ "alpha",
+ "beta",
+ "gamma",
+ "delta",
+ "omega",
];
// Warm up (not measured)
@@ -236,9 +259,8 @@ async fn perf_bulk_throughput() {
let mut ndjson = String::with_capacity(BATCH_SIZE * 200);
for j in 0..BATCH_SIZE {
let doc_idx = batch * BATCH_SIZE + j;
- let action = format!(
- "{{\"index\":{{\"_index\":\"bulk_bench\",\"_id\":\"{doc_idx}\"}}}}\n"
- );
+ let action =
+ format!("{{\"index\":{{\"_index\":\"bulk_bench\",\"_id\":\"{doc_idx}\"}}}}\n");
let doc_str = serde_json::to_string(&make_doc(doc_idx)).unwrap() + "\n";
ndjson.push_str(&action);
ndjson.push_str(&doc_str);
@@ -256,10 +278,16 @@ async fn perf_bulk_throughput() {
let total_mb = total_bytes as f64 / 1_048_576.0;
print_kv("Documents indexed:", &format!("{}", docs_indexed));
- print_kv("Batches:", &format!("{} × {} docs", total_batches, BATCH_SIZE));
+ print_kv(
+ "Batches:",
+ &format!("{} × {} docs", total_batches, BATCH_SIZE),
+ );
print_kv("Total payload size:", &format!("{:.1} MB", total_mb));
print_kv("Total time:", &format!("{:.2?}", elapsed));
- print_kv("Throughput (docs):", &format!("{:.0} docs/sec", docs_per_sec));
+ print_kv(
+ "Throughput (docs):",
+ &format!("{:.0} docs/sec", docs_per_sec),
+ );
print_kv("Throughput (data):", &format!("{:.1} MB/sec", mb_per_sec));
println!();
}
@@ -387,8 +415,16 @@ async fn perf_concurrent_search() {
let idx = std::sync::Arc::new(idx);
let terms = std::sync::Arc::new([
- "technology", "science", "politics", "sports", "arts",
- "alpha", "beta", "gamma", "delta", "epsilon",
+ "technology",
+ "science",
+ "politics",
+ "sports",
+ "arts",
+ "alpha",
+ "beta",
+ "gamma",
+ "delta",
+ "epsilon",
]);
// Spawn concurrent tasks
@@ -439,7 +475,10 @@ async fn perf_concurrent_search() {
print_kv("p95:", &format!("{:.2?}", pcts[1]));
print_kv("p99:", &format!("{:.2?}", pcts[2]));
print_kv("min:", &format!("{:.2?}", all_latencies[0]));
- print_kv("max:", &format!("{:.2?}", all_latencies[all_latencies.len() - 1]));
+ print_kv(
+ "max:",
+ &format!("{:.2?}", all_latencies[all_latencies.len() - 1]),
+ );
println!();
}
@@ -468,9 +507,13 @@ async fn perf_memory_footprint() {
{
// SAFETY: sysconf(_SC_PAGESIZE) is always safe; it reads a kernel
// constant and never touches user memory.
- extern "C" { fn sysconf(name: i32) -> i64; }
+ extern "C" {
+ fn sysconf(name: i32) -> i64;
+ }
let ps = unsafe { sysconf(30) }; // 30 == _SC_PAGESIZE
- if ps > 0 { return ps as u64; }
+ if ps > 0 {
+ return ps as u64;
+ }
}
4096
}
@@ -507,11 +550,23 @@ async fn perf_memory_footprint() {
print_kv("Documents indexed:", &format!("{}", N));
print_kv("Indexing time:", &format!("{:.2?}", elapsed));
- print_kv("RSS before:", &format!("{:.1} MB", baseline_rss as f64 / 1_048_576.0));
- print_kv("RSS after:", &format!("{:.1} MB", rss_after as f64 / 1_048_576.0));
- print_kv("RSS delta:", &format!("{:.1} MB", rss_delta as f64 / 1_048_576.0));
+ print_kv(
+ "RSS before:",
+ &format!("{:.1} MB", baseline_rss as f64 / 1_048_576.0),
+ );
+ print_kv(
+ "RSS after:",
+ &format!("{:.1} MB", rss_after as f64 / 1_048_576.0),
+ );
+ print_kv(
+ "RSS delta:",
+ &format!("{:.1} MB", rss_delta as f64 / 1_048_576.0),
+ );
print_kv("RSS bytes/doc:", &format!("{}", rss_per_doc));
- print_kv("Engine-reported memtable:", &format!("{:.1} MB", engine_reported_bytes as f64 / 1_048_576.0));
+ print_kv(
+ "Engine-reported memtable:",
+ &format!("{:.1} MB", engine_reported_bytes as f64 / 1_048_576.0),
+ );
print_kv("Engine bytes/doc:", &format!("{}", engine_per_doc));
println!();
}
@@ -543,7 +598,9 @@ async fn perf_turbo_indexing_throughput() {
// ── Standard path ─────────────────────────────────────────────────────
let (engine_std, _dir_std) = bench_engine();
- engine_std.create_index("std_bench", Schema::empty()).unwrap();
+ engine_std
+ .create_index("std_bench", Schema::empty())
+ .unwrap();
let idx_std = engine_std.get_index("std_bench").unwrap();
let std_start = Instant::now();
@@ -565,7 +622,8 @@ async fn perf_turbo_indexing_throughput() {
let idx_turbo = engine_turbo.get_index("turbo_bench").unwrap();
let turbo_start = Instant::now();
- let mut batch: Vec<(String, serde_json::Value, std::sync::Arc<[u8]>)> = Vec::with_capacity(TURBO_BATCH);
+ let mut batch: Vec<(String, serde_json::Value, std::sync::Arc<[u8]>)> =
+ Vec::with_capacity(TURBO_BATCH);
for i in 0..N {
let empty_bytes: std::sync::Arc<[u8]> = std::sync::Arc::from(&[][..]);
diff --git a/engine/crates/xerj-engine/tests/product_experience.rs b/engine/crates/xerj-engine/tests/product_experience.rs
index 245b90f72..78e00a588 100644
--- a/engine/crates/xerj-engine/tests/product_experience.rs
+++ b/engine/crates/xerj-engine/tests/product_experience.rs
@@ -42,12 +42,7 @@ fn make_engine_with_config(dir: &TempDir, configure: impl FnOnce(&mut Config)) -
Engine::new(config).expect("engine::new")
}
-fn print_result(
- journey: &str,
- xerj_desc: &str,
- es_desc: &str,
- winner_line: &str,
-) {
+fn print_result(journey: &str, xerj_desc: &str, es_desc: &str, winner_line: &str) {
let width = 52usize;
let border = "=".repeat(width);
let pad = |s: &str| {
@@ -112,7 +107,10 @@ async fn journey_first_time_user() {
"First-Time User",
&format!("{:.2?} from zero to search results", total_time),
"60+ seconds (JVM startup alone)",
- &format!("xerj ({:.0}x faster to first value)", 60_000.0 / total_time.as_millis().max(1) as f64),
+ &format!(
+ "xerj ({:.0}x faster to first value)",
+ 60_000.0 / total_time.as_millis().max(1) as f64
+ ),
);
assert!(
@@ -121,7 +119,10 @@ async fn journey_first_time_user() {
total_time
);
assert!(result.total.value > 0, "Must find at least one document");
- assert_eq!(result.total.value, 100, "All 100 docs should match 'various topics'");
+ assert_eq!(
+ result.total.value, 100,
+ "All 100 docs should match 'various topics'"
+ );
}
// ═════════════════════════════════════════════════════════════════════════════
@@ -147,7 +148,9 @@ async fn journey_crash_recovery() {
config.server.data_dir = data_path.clone();
let engine = Engine::new(config).unwrap();
- engine.create_index("critical-data", Schema::empty()).unwrap();
+ engine
+ .create_index("critical-data", Schema::empty())
+ .unwrap();
let idx = engine.get_index("critical-data").unwrap();
for i in 0..doc_count {
@@ -302,7 +305,12 @@ async fn journey_log_analytics() {
engine.create_index("app-logs", Schema::empty()).unwrap();
let idx = engine.get_index("app-logs").unwrap();
- let services = ["auth-service", "payment-service", "user-service", "api-gateway"];
+ let services = [
+ "auth-service",
+ "payment-service",
+ "user-service",
+ "api-gateway",
+ ];
let levels = ["INFO", "WARN", "ERROR", "DEBUG"];
let messages = [
"request processed successfully",
@@ -320,9 +328,9 @@ async fn journey_log_analytics() {
for i in 0..5_000usize {
let service = services[i % services.len()];
// Skew: errors cluster in payment-service and api-gateway.
- let level = if service == "payment-service" && i % 5 == 0 {
- "ERROR"
- } else if service == "api-gateway" && i % 8 == 0 {
+ let level = if (service == "payment-service" && i % 5 == 0)
+ || (service == "api-gateway" && i % 8 == 0)
+ {
"ERROR"
} else {
levels[i % 3] // INFO/WARN/ERROR cycling but less ERROR
@@ -380,7 +388,10 @@ async fn journey_log_analytics() {
.await
.unwrap();
- assert!(payment_errors.total.value > 0, "Must find payment-service errors");
+ assert!(
+ payment_errors.total.value > 0,
+ "Must find payment-service errors"
+ );
// Query 3: Full-text search across log messages — find connection issues.
let connection_issues = idx
@@ -442,7 +453,10 @@ async fn journey_es_migration() {
// ES contract: _id, result, _version fields in response.
assert_eq!(resp.id, "1", "response._id must match supplied ID");
- assert_eq!(resp.result, "created", "result must be 'created' on first write");
+ assert_eq!(
+ resp.result, "created",
+ "result must be 'created' on first write"
+ );
assert_eq!(resp.version, 1, "_version must start at 1");
// POST /{index}/_search — same query DSL as ES.
@@ -458,11 +472,17 @@ async fn journey_es_migration() {
.await
.unwrap();
- assert_eq!(result.total.value, 1, "Search must find the indexed document");
+ assert_eq!(
+ result.total.value, 1,
+ "Search must find the indexed document"
+ );
// ES response format: hits.total.value and hits.hits[].
let hit = &result.hits[0];
assert_eq!(hit.id, "1");
- assert!(hit.source.get("user").is_some(), "Source fields must be returned");
+ assert!(
+ hit.source.get("user").is_some(),
+ "Source fields must be returned"
+ );
// POST /_bulk — NDJSON bulk format (identical to ES).
let bulk_body = r#"{"index":{"_index":"my-index","_id":"2"}}
@@ -474,10 +494,20 @@ async fn journey_es_migration() {
"#;
let bulk_result = process_bulk(&engine, Some("my-index"), bulk_body).await;
- assert!(!bulk_result.errors, "Bulk operation must succeed without errors");
- assert_eq!(bulk_result.items.len(), 3, "All 3 bulk items must be processed");
+ assert!(
+ !bulk_result.errors,
+ "Bulk operation must succeed without errors"
+ );
+ assert_eq!(
+ bulk_result.items.len(),
+ 3,
+ "All 3 bulk items must be processed"
+ );
for item in &bulk_result.items {
- assert!(item.status == 200 || item.status == 201, "Each bulk item must return HTTP 200 or 201");
+ assert!(
+ item.status == 200 || item.status == 201,
+ "Each bulk item must return HTTP 200 or 201"
+ );
}
// Verify all docs are searchable immediately after bulk.
@@ -511,7 +541,10 @@ async fn journey_es_migration() {
)
.await
.unwrap();
- assert_eq!(recent.total.value, 3, "Range query must find 3 docs from 2024");
+ assert_eq!(
+ recent.total.value, 3,
+ "Range query must find 3 docs from 2024"
+ );
print_result(
"ES Migration",
@@ -543,7 +576,7 @@ async fn journey_security_by_default() {
// Admin key starts empty — it's auto-generated at server startup time.
// (In tests we verify the flag, not the runtime key generation.)
// The point: you can't accidentally run an open cluster.
- let config_with_auth = Config::from_str(
+ let config_with_auth = Config::from_toml_str(
r#"
[auth]
enabled = true
@@ -555,7 +588,7 @@ async fn journey_security_by_default() {
assert_eq!(config_with_auth.auth.admin_api_key, "test-key-abc123");
// Verify: disabling auth is a deliberate choice, not the default.
- let insecure_config = Config::from_str(
+ let insecure_config = Config::from_toml_str(
r#"
[auth]
enabled = false
@@ -568,7 +601,9 @@ async fn journey_security_by_default() {
);
// Verify: the default config is valid (usable out of the box).
- Config::default().validate().expect("Default config must be valid");
+ Config::default()
+ .validate()
+ .expect("Default config must be valid");
print_result(
"Security by Default",
@@ -599,7 +634,9 @@ async fn journey_upgrade() {
config.server.data_dir = data_path.clone();
let engine = Engine::new(config).unwrap();
- engine.create_index("user-profiles", Schema::empty()).unwrap();
+ engine
+ .create_index("user-profiles", Schema::empty())
+ .unwrap();
let idx = engine.get_index("user-profiles").unwrap();
for i in 0..doc_count {
@@ -660,15 +697,23 @@ async fn journey_upgrade() {
)
.await
.unwrap();
- println!(" After upgrade: {} docs found, first 3: {:?}",
+ println!(
+ " After upgrade: {} docs found, first 3: {:?}",
enterprise.total.value,
- enterprise.hits.iter().map(|h| &h.id).collect::<Vec<_>>());
- assert!(enterprise.total.value >= doc_count as u64 / 2,
- "Most docs must survive upgrade, found {}", enterprise.total.value);
+ enterprise.hits.iter().map(|h| &h.id).collect::<Vec<_>>()
+ );
+ assert!(
+ enterprise.total.value >= doc_count as u64 / 2,
+ "Most docs must survive upgrade, found {}",
+ enterprise.total.value
+ );
print_result(
"Zero-Downtime Upgrade",
- &format!("restart in {:.2?}; all {} docs immediately available", startup_time, doc_count),
+ &format!(
+ "restart in {:.2?}; all {} docs immediately available",
+ startup_time, doc_count
+ ),
"major version upgrades require reindex (days/weeks); rolling restarts",
"xerj (restart and go; no reindex ever)",
);
@@ -724,19 +769,31 @@ async fn journey_zero_config() {
.await
.unwrap();
- assert_eq!(result.total.value, 1, "Search must work with zero configuration");
+ assert_eq!(
+ result.total.value, 1,
+ "Search must work with zero configuration"
+ );
// Count user-facing settings.
// xerj: 38 settings (5+2+3+5+5+3+1+6+2+4+3 - 1 auto-generated).
// ES: 3,000+ settings across elasticsearch.yml, jvm.options, log4j2.properties.
let xerj_settings: usize = 5 + 2 + 3 + 5 + 5 + 3 + 1 + 6 + 2 + 4 + 3 - 1; // = 38
- assert_eq!(xerj_settings, 38, "xerj must have exactly 38 user-facing settings");
+ assert_eq!(
+ xerj_settings, 38,
+ "xerj must have exactly 38 user-facing settings"
+ );
print_result(
"Zero Configuration",
- &format!("{} settings total; works out of the box with defaults", xerj_settings),
+ &format!(
+ "{} settings total; works out of the box with defaults",
+ xerj_settings
+ ),
"3,000+ settings; requires JVM tuning, cluster config, network setup",
- &format!("xerj ({:.0}x fewer settings to learn)", 3000.0 / xerj_settings as f64),
+ &format!(
+ "xerj ({:.0}x fewer settings to learn)",
+ 3000.0 / xerj_settings as f64
+ ),
);
}
@@ -807,13 +864,13 @@ async fn journey_realtime_search() {
}
let all = idx
- .search(
- &parse_request(&json!({ "query": { "match_all": {} }, "size": 0 }))
- .unwrap(),
- )
+ .search(&parse_request(&json!({ "query": { "match_all": {} }, "size": 0 })).unwrap())
.await
.unwrap();
- assert_eq!(all.total.value, 10, "All 10 documents must be searchable immediately");
+ assert_eq!(
+ all.total.value, 10,
+ "All 10 documents must be searchable immediately"
+ );
print_result(
"Real-Time Search",
diff --git a/engine/crates/xerj-engine/tests/shard_router_write_path.rs b/engine/crates/xerj-engine/tests/shard_router_write_path.rs
index 0d07124ca..24ec1c600 100644
--- a/engine/crates/xerj-engine/tests/shard_router_write_path.rs
+++ b/engine/crates/xerj-engine/tests/shard_router_write_path.rs
@@ -89,14 +89,12 @@ async fn router_forwards_remote_shards() {
counts[&None]
);
assert!(
- counts[&Some("peer-b".to_string())] >= 150
- && counts[&Some("peer-b".to_string())] <= 350,
+ counts[&Some("peer-b".to_string())] >= 150 && counts[&Some("peer-b".to_string())] <= 350,
"peer-b got {} hits, expected 150..350",
counts[&Some("peer-b".to_string())]
);
assert!(
- counts[&Some("peer-c".to_string())] >= 150
- && counts[&Some("peer-c".to_string())] <= 350,
+ counts[&Some("peer-c".to_string())] >= 150 && counts[&Some("peer-c".to_string())] <= 350,
"peer-c got {} hits, expected 150..350",
counts[&Some("peer-c".to_string())]
);
diff --git a/engine/crates/xerj-fts/src/analyzer.rs b/engine/crates/xerj-fts/src/analyzer.rs
index 7504bb9ec..2913ad68d 100644
--- a/engine/crates/xerj-fts/src/analyzer.rs
+++ b/engine/crates/xerj-fts/src/analyzer.rs
@@ -14,8 +14,8 @@ use regex::Regex;
use rust_stemmers::{Algorithm, Stemmer};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
-use unicode_segmentation::UnicodeSegmentation;
use tracing;
+use unicode_segmentation::UnicodeSegmentation;
// ── Core token type ──────────────────────────────────────────────────────────
@@ -109,10 +109,7 @@ impl AnalyzerPipeline {
/// Convenience: return just the token texts (used for query term extraction).
pub fn analyze_to_terms(&self, input: &str) -> Vec<String> {
- self.analyze(input)
- .into_iter()
- .map(|t| t.text)
- .collect()
+ self.analyze(input).into_iter().map(|t| t.text).collect()
}
}
@@ -125,21 +122,14 @@ pub struct StandardTokenizer;
impl Tokenizer for StandardTokenizer {
fn tokenize(&self, input: &str) -> Vec<Token> {
let mut tokens = Vec::new();
- let mut position: u32 = 0;
- for word in input.unicode_words() {
+ for (position, word) in input.unicode_words().enumerate() {
// Find the byte offset of this word in the original string.
// SAFETY: `unicode_words()` returns sub-slices of `input`.
let start = word.as_ptr() as usize - input.as_ptr() as usize;
let end = start + word.len();
- tokens.push(Token::new(
- word,
- position,
- start as u32,
- end as u32,
- ));
- position += 1;
+ tokens.push(Token::new(word, position as u32, start as u32, end as u32));
}
tokens
@@ -312,8 +302,17 @@ impl Tokenizer for NGramTokenizer {
let text: String = chars[start_char..end_char].iter().collect();
// Compute byte offsets.
let byte_start: usize = chars[..start_char].iter().map(|c| c.len_utf8()).sum();
- let byte_end: usize = byte_start + chars[start_char..end_char].iter().map(|c| c.len_utf8()).sum::<usize>();
- tokens.push(Token::new(text, position, byte_start as u32, byte_end as u32));
+ let byte_end: usize = byte_start
+ + chars[start_char..end_char]
+ .iter()
+ .map(|c| c.len_utf8())
+ .sum::<usize>();
+ tokens.push(Token::new(
+ text,
+ position,
+ byte_start as u32,
+ byte_end as u32,
+ ));
position += 1;
}
}
@@ -343,16 +342,14 @@ impl Tokenizer for EdgeNGramTokenizer {
fn tokenize(&self, input: &str) -> Vec<Token> {
let chars: Vec<char> = input.chars().collect();
let mut tokens = Vec::new();
- let mut position: u32 = 0;
- for gram_size in self.min_gram..=self.max_gram {
+ for (position, gram_size) in (self.min_gram..=self.max_gram).enumerate() {
if gram_size > chars.len() {
break;
}
let text: String = chars[..gram_size].iter().collect();
let byte_end: usize = chars[..gram_size].iter().map(|c| c.len_utf8()).sum();
- tokens.push(Token::new(text, position, 0, byte_end as u32));
- position += 1;
+ tokens.push(Token::new(text, position as u32, 0, byte_end as u32));
}
tokens
}
@@ -387,14 +384,17 @@ impl PatternTokenizer {
impl Tokenizer for PatternTokenizer {
fn tokenize(&self, input: &str) -> Vec<Token> {
let mut tokens = Vec::new();
- let mut position: u32 = 0;
- for mat in self.pattern.split(input).filter(|s| !s.is_empty()) {
+ for (position, mat) in self
+ .pattern
+ .split(input)
+ .filter(|s| !s.is_empty())
+ .enumerate()
+ {
// Compute byte offsets by finding the substring in the original input.
let start = mat.as_ptr() as usize - input.as_ptr() as usize;
let end = start + mat.len();
- tokens.push(Token::new(mat, position, start as u32, end as u32));
- position += 1;
+ tokens.push(Token::new(mat, position as u32, start as u32, end as u32));
}
tokens
}
@@ -430,15 +430,19 @@ impl SynonymFilter {
if let Some((lhs, rhs)) = rule.split_once("=>") {
// Explicit: lhs terms map to rhs terms.
let inputs: Vec<String> = lhs.split(',').map(|s| s.trim().to_lowercase()).collect();
- let outputs: Vec<String> = rhs.split(',').map(|s| s.trim().to_lowercase()).collect();
+ let outputs: Vec<String> =
+ rhs.split(',').map(|s| s.trim().to_lowercase()).collect();
for input in inputs {
- map.entry(input).or_default().extend(outputs.iter().cloned());
+ map.entry(input)
+ .or_default()
+ .extend(outputs.iter().cloned());
}
} else {
// Equivalence: all terms expand to the full set.
let terms: Vec<String> = rule.split(',').map(|s| s.trim().to_lowercase()).collect();
for term in &terms {
- let others: Vec<String> = terms.iter().filter(|t| *t != term).cloned().collect();
+ let others: Vec<String> =
+ terms.iter().filter(|t| *t != term).cloned().collect();
map.entry(term.clone()).or_default().extend(others);
}
}
@@ -505,37 +509,41 @@ fn fold_to_ascii(s: &str) -> String {
fn push_ascii_fold(c: char, buf: &mut String) {
match c {
// A
- 'À'|'Á'|'Â'|'Ã'|'Ä'|'Å'|'à'|'á'|'â'|'ã'|'ä'|'å' => buf.push('a'),
+ 'À' | 'Á' | 'Â' | 'Ã' | 'Ä' | 'Å' | 'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' => {
+ buf.push('a')
+ }
// AE
- 'Æ'|'æ' => buf.push_str("ae"),
+ 'Æ' | 'æ' => buf.push_str("ae"),
// C
- 'Ç'|'ç' => buf.push('c'),
+ 'Ç' | 'ç' => buf.push('c'),
// D
- 'Ð'|'ð' => buf.push('d'),
+ 'Ð' | 'ð' => buf.push('d'),
// E
- 'È'|'É'|'Ê'|'Ë'|'è'|'é'|'ê'|'ë' => buf.push('e'),
+ 'È' | 'É' | 'Ê' | 'Ë' | 'è' | 'é' | 'ê' | 'ë' => buf.push('e'),
// G
- 'Ğ'|'ğ' => buf.push('g'),
+ 'Ğ' | 'ğ' => buf.push('g'),
// I
- 'Ì'|'Í'|'Î'|'Ï'|'ì'|'í'|'î'|'ï' => buf.push('i'),
+ 'Ì' | 'Í' | 'Î' | 'Ï' | 'ì' | 'í' | 'î' | 'ï' => buf.push('i'),
// N
- 'Ñ'|'ñ' => buf.push('n'),
+ 'Ñ' | 'ñ' => buf.push('n'),
// O
- 'Ò'|'Ó'|'Ô'|'Õ'|'Ö'|'Ø'|'ò'|'ó'|'ô'|'õ'|'ö'|'ø' => buf.push('o'),
+ 'Ò' | 'Ó' | 'Ô' | 'Õ' | 'Ö' | 'Ø' | 'ò' | 'ó' | 'ô' | 'õ' | 'ö' | 'ø' => {
+ buf.push('o')
+ }
// OE
- 'Œ'|'œ' => buf.push_str("oe"),
+ 'Œ' | 'œ' => buf.push_str("oe"),
// S
- 'Š'|'š' => buf.push('s'),
+ 'Š' | 'š' => buf.push('s'),
// SS
'ß' => buf.push_str("ss"),
// T
- 'Þ'|'þ' => buf.push_str("th"),
+ 'Þ' | 'þ' => buf.push_str("th"),
// U
- 'Ù'|'Ú'|'Û'|'Ü'|'ù'|'ú'|'û'|'ü' => buf.push('u'),
+ 'Ù' | 'Ú' | 'Û' | 'Ü' | 'ù' | 'ú' | 'û' | 'ü' => buf.push('u'),
// Y
- 'Ý'|'ÿ'|'ý' => buf.push('y'),
+ 'Ý' | 'ÿ' | 'ý' => buf.push('y'),
// Z
- 'Ž'|'ž' => buf.push('z'),
+ 'Ž' | 'ž' => buf.push('z'),
// Passthrough
other => buf.push(other),
}
@@ -749,10 +757,19 @@ impl Tokenizer for ThaiTokenizer {
while i < chars.len() && is_thai(chars[i].1) {
i += 1;
}
- let run_end_byte = if i < chars.len() { chars[i].0 } else { input.len() };
+ let run_end_byte = if i < chars.len() {
+ chars[i].0
+ } else {
+ input.len()
+ };
if let Some(word) = input.get(run_start_byte..run_end_byte) {
if !word.is_empty() {
- tokens.push(Token::new(word, position, run_start_byte as u32, run_end_byte as u32));
+ tokens.push(Token::new(
+ word,
+ position,
+ run_start_byte as u32,
+ run_end_byte as u32,
+ ));
position += 1;
}
}
@@ -764,7 +781,11 @@ impl Tokenizer for ThaiTokenizer {
while i < chars.len() && !chars[i].1.is_whitespace() && !is_thai(chars[i].1) {
i += 1;
}
- let run_end_byte = if i < chars.len() { chars[i].0 } else { input.len() };
+ let run_end_byte = if i < chars.len() {
+ chars[i].0
+ } else {
+ input.len()
+ };
if let Some(word) = input.get(run_start_byte..run_end_byte) {
if !word.is_empty() {
tokens.push(Token::new(
@@ -813,9 +834,7 @@ fn nfkc_normalize(s: &str) -> String {
// a best-effort fold: lowercase + ASCII compatibility substitutions.
// The unicode-normalization crate is not a current dependency, so we keep
// this lightweight — the filter still benefits from lowercasing.
- s.chars()
- .flat_map(|c| nfkc_fold_char(c))
- .collect()
+ s.chars().flat_map(nfkc_fold_char).collect()
}
/// Single-character NFKC compatibility fold for the most common cases.
@@ -997,7 +1016,8 @@ impl AnalyzerRegistry {
/// Returns the "standard" analyzer, panicking if it is not registered.
/// This should never panic with a default-constructed registry.
pub fn standard(&self) -> Arc<AnalyzerPipeline> {
- self.get_analyzer("standard").expect("standard analyzer always registered")
+ self.get_analyzer("standard")
+ .expect("standard analyzer always registered")
}
/// Extend this registry with custom analyzer definitions parsed from an
@@ -1044,25 +1064,20 @@ impl AnalyzerRegistry {
let rules: Vec<&str> = filter_def
.get("synonyms")
.and_then(|v| v.as_array())
- .map(|arr| {
- arr.iter()
- .filter_map(|v| v.as_str())
- .collect()
- })
+ .map(|arr| arr.iter().filter_map(|v| v.as_str()).collect())
.unwrap_or_default();
let f = SynonymFilter::new(&rules);
custom_filters.insert(filter_name.clone(), Arc::new(f));
}
"length" => {
- let min = filter_def
- .get("min")
- .and_then(|v| v.as_u64())
- .unwrap_or(2) as usize;
+ let min =
+ filter_def.get("min").and_then(|v| v.as_u64()).unwrap_or(2) as usize;
let max = filter_def
.get("max")
.and_then(|v| v.as_u64())
.unwrap_or(256) as usize;
- custom_filters.insert(filter_name.clone(), Arc::new(LengthFilter::new(min, max)));
+ custom_filters
+ .insert(filter_name.clone(), Arc::new(LengthFilter::new(min, max)));
}
"shingle" => {
let size = filter_def
@@ -1100,10 +1115,7 @@ impl AnalyzerRegistry {
if let Some(tok_map) = analysis.pointer("/tokenizer").and_then(|v| v.as_object()) {
for (tok_name, tok_def) in tok_map {
- let tok_type = tok_def
- .get("type")
- .and_then(|v| v.as_str())
- .unwrap_or("");
+ let tok_type = tok_def.get("type").and_then(|v| v.as_str()).unwrap_or("");
match tok_type {
"ngram" => {
@@ -1115,7 +1127,8 @@ impl AnalyzerRegistry {
.get("max_gram")
.and_then(|v| v.as_u64())
.unwrap_or(2) as usize;
- custom_tokenizers.insert(tok_name.clone(), Arc::new(NGramTokenizer::new(min, max)));
+ custom_tokenizers
+ .insert(tok_name.clone(), Arc::new(NGramTokenizer::new(min, max)));
}
"edge_ngram" => {
let min = tok_def
@@ -1126,7 +1139,10 @@ impl AnalyzerRegistry {
.get("max_gram")
.and_then(|v| v.as_u64())
.unwrap_or(2) as usize;
- custom_tokenizers.insert(tok_name.clone(), Arc::new(EdgeNGramTokenizer::new(min, max)));
+ custom_tokenizers.insert(
+ tok_name.clone(),
+ Arc::new(EdgeNGramTokenizer::new(min, max)),
+ );
}
"pattern" => {
let pattern = tok_def
@@ -1134,7 +1150,9 @@ impl AnalyzerRegistry {
.and_then(|v| v.as_str())
.unwrap_or(r"\W+");
match PatternTokenizer::new(pattern) {
- Ok(t) => { custom_tokenizers.insert(tok_name.clone(), Arc::new(t)); }
+ Ok(t) => {
+ custom_tokenizers.insert(tok_name.clone(), Arc::new(t));
+ }
Err(e) => {
tracing::warn!(
tok_name = tok_name.as_str(),
@@ -1320,74 +1338,464 @@ const ECOMMERCE_SYNONYMS: &[&str] = &[
/// Matches Lucene's `EnglishAnalyzer.ENGLISH_STOP_WORDS_SET` (174 words).
const ENGLISH_STOP_WORDS: &[&str] = &[
- "a", "an", "and", "are", "as", "at", "be", "but", "by", "for",
- "if", "in", "into", "is", "it", "no", "not", "of", "on", "or",
- "such", "that", "the", "their", "then", "there", "these", "they",
- "this", "to", "was", "will", "with",
+ "a",
+ "an",
+ "and",
+ "are",
+ "as",
+ "at",
+ "be",
+ "but",
+ "by",
+ "for",
+ "if",
+ "in",
+ "into",
+ "is",
+ "it",
+ "no",
+ "not",
+ "of",
+ "on",
+ "or",
+ "such",
+ "that",
+ "the",
+ "their",
+ "then",
+ "there",
+ "these",
+ "they",
+ "this",
+ "to",
+ "was",
+ "will",
+ "with",
// Extended Lucene English stop list
- "able", "about", "above", "according", "accordingly", "across",
- "actually", "after", "afterwards", "again", "against", "albeit",
- "all", "allow", "allows", "almost", "alone", "along", "already",
- "also", "although", "always", "am", "among", "amongst", "another",
- "any", "anybody", "anyhow", "anyone", "anything", "anyway",
- "anyways", "anywhere", "apart", "appear", "appreciate", "appropriate",
- "around", "aside", "ask", "asking", "associated", "available",
- "away", "awfully", "became", "because", "become", "becomes",
- "becoming", "been", "before", "beforehand", "behind", "being",
- "below", "beside", "besides", "best", "better", "between", "beyond",
- "both", "brief", "came", "can", "cannot", "cant", "cause", "causes",
- "certain", "certainly", "changes", "clearly", "co", "com", "come",
- "comes", "concerning", "consequently", "consider", "considering",
- "contain", "containing", "contains", "corresponding", "could",
- "course", "currently", "definitely", "described", "despite",
- "did", "different", "does", "doing", "done", "down", "during",
- "each", "eight", "either", "else", "elsewhere", "enough", "entirely",
- "especially", "even", "ever", "every", "everybody", "everyone",
- "everything", "everywhere", "ex", "exactly", "except", "far",
- "few", "fifth", "first", "five", "followed", "following", "follows",
- "former", "formerly", "forth", "four", "from", "further",
- "furthermore", "get", "gets", "given", "go", "goes", "going",
- "gone", "got", "had", "happens", "hardly", "has", "have", "having",
- "he", "hence", "her", "here", "hereafter", "hereby", "herein",
- "hereupon", "hers", "herself", "him", "himself", "his", "hither",
- "hopefully", "how", "howbeit", "however", "i", "ie", "ignored",
- "immediate", "inasmuch", "inc", "indeed", "indicate", "indicated",
- "indicates", "inner", "insofar", "instead", "its", "itself",
- "just", "keep", "kept", "know", "known", "knows", "last", "lately",
- "later", "latter", "latterly", "least", "less", "lest", "let",
- "like", "liked", "likely", "little", "look", "looking", "looks",
- "ltd", "mainly", "many", "may", "maybe", "me", "mean", "meanwhile",
- "merely", "might", "more", "moreover", "most", "mostly", "much",
- "must", "my", "myself", "name", "namely", "nd", "near", "nearly",
- "necessary", "need", "needs", "neither", "never", "nevertheless",
- "new", "next", "nine", "nobody", "none", "noone", "nor", "normally",
- "nothing", "novel", "now", "nowhere", "obviously", "off", "often",
- "oh", "ok", "okay", "old", "once", "one", "ones", "only", "onto",
- "other", "others", "otherwise", "our", "ours", "ourselves", "out",
- "outside", "over", "overall", "own", "particular", "particularly",
- "per", "perhaps", "placed", "please", "plus", "possible", "presumably",
- "probably", "provides", "quite", "rather", "really", "reasonably",
- "regarding", "regardless", "regards", "relatively", "respectively",
- "right", "said", "same", "saw", "say", "saying", "says", "second",
- "secondly", "see", "seeing", "seem", "seemed", "seeming", "seems",
- "seen", "self", "selves", "sensible", "sent", "serious", "seriously",
- "seven", "several", "shall", "she", "should", "since", "six",
- "so", "some", "somebody", "somehow", "someone", "something",
- "sometime", "sometimes", "somewhat", "somewhere", "soon", "sorry",
- "specified", "specify", "specifying", "still", "sub", "sup",
- "sure", "take", "taken", "tell", "tends", "th", "than", "thank",
- "thanks", "third", "thorough", "thoroughly", "though", "three",
- "through", "throughout", "thru", "thus", "together", "too", "took",
- "toward", "towards", "tried", "tries", "truly", "try", "trying",
- "twice", "two", "un", "under", "unfortunately", "unless", "unlikely",
- "until", "unto", "upon", "us", "use", "used", "useful", "uses",
- "using", "usually", "value", "various", "very", "via", "viz",
- "vs", "want", "wants", "we", "well", "went", "were", "what",
- "whatever", "when", "whence", "whenever", "where", "whereafter",
- "whereas", "whereby", "wherein", "whereupon", "wherever", "whether",
- "which", "while", "whither", "who", "whoever", "whole", "whom",
- "whose", "why", "within", "without", "wonder", "would", "yes",
- "yet", "you", "your", "yours", "yourself", "yourselves", "zero",
+ "able",
+ "about",
+ "above",
+ "according",
+ "accordingly",
+ "across",
+ "actually",
+ "after",
+ "afterwards",
+ "again",
+ "against",
+ "albeit",
+ "all",
+ "allow",
+ "allows",
+ "almost",
+ "alone",
+ "along",
+ "already",
+ "also",
+ "although",
+ "always",
+ "am",
+ "among",
+ "amongst",
+ "another",
+ "any",
+ "anybody",
+ "anyhow",
+ "anyone",
+ "anything",
+ "anyway",
+ "anyways",
+ "anywhere",
+ "apart",
+ "appear",
+ "appreciate",
+ "appropriate",
+ "around",
+ "aside",
+ "ask",
+ "asking",
+ "associated",
+ "available",
+ "away",
+ "awfully",
+ "became",
+ "because",
+ "become",
+ "becomes",
+ "becoming",
+ "been",
+ "before",
+ "beforehand",
+ "behind",
+ "being",
+ "below",
+ "beside",
+ "besides",
+ "best",
+ "better",
+ "between",
+ "beyond",
+ "both",
+ "brief",
+ "came",
+ "can",
+ "cannot",
+ "cant",
+ "cause",
+ "causes",
+ "certain",
+ "certainly",
+ "changes",
+ "clearly",
+ "co",
+ "com",
+ "come",
+ "comes",
+ "concerning",
+ "consequently",
+ "consider",
+ "considering",
+ "contain",
+ "containing",
+ "contains",
+ "corresponding",
+ "could",
+ "course",
+ "currently",
+ "definitely",
+ "described",
+ "despite",
+ "did",
+ "different",
+ "does",
+ "doing",
+ "done",
+ "down",
+ "during",
+ "each",
+ "eight",
+ "either",
+ "else",
+ "elsewhere",
+ "enough",
+ "entirely",
+ "especially",
+ "even",
+ "ever",
+ "every",
+ "everybody",
+ "everyone",
+ "everything",
+ "everywhere",
+ "ex",
+ "exactly",
+ "except",
+ "far",
+ "few",
+ "fifth",
+ "first",
+ "five",
+ "followed",
+ "following",
+ "follows",
+ "former",
+ "formerly",
+ "forth",
+ "four",
+ "from",
+ "further",
+ "furthermore",
+ "get",
+ "gets",
+ "given",
+ "go",
+ "goes",
+ "going",
+ "gone",
+ "got",
+ "had",
+ "happens",
+ "hardly",
+ "has",
+ "have",
+ "having",
+ "he",
+ "hence",
+ "her",
+ "here",
+ "hereafter",
+ "hereby",
+ "herein",
+ "hereupon",
+ "hers",
+ "herself",
+ "him",
+ "himself",
+ "his",
+ "hither",
+ "hopefully",
+ "how",
+ "howbeit",
+ "however",
+ "i",
+ "ie",
+ "ignored",
+ "immediate",
+ "inasmuch",
+ "inc",
+ "indeed",
+ "indicate",
+ "indicated",
+ "indicates",
+ "inner",
+ "insofar",
+ "instead",
+ "its",
+ "itself",
+ "just",
+ "keep",
+ "kept",
+ "know",
+ "known",
+ "knows",
+ "last",
+ "lately",
+ "later",
+ "latter",
+ "latterly",
+ "least",
+ "less",
+ "lest",
+ "let",
+ "like",
+ "liked",
+ "likely",
+ "little",
+ "look",
+ "looking",
+ "looks",
+ "ltd",
+ "mainly",
+ "many",
+ "may",
+ "maybe",
+ "me",
+ "mean",
+ "meanwhile",
+ "merely",
+ "might",
+ "more",
+ "moreover",
+ "most",
+ "mostly",
+ "much",
+ "must",
+ "my",
+ "myself",
+ "name",
+ "namely",
+ "nd",
+ "near",
+ "nearly",
+ "necessary",
+ "need",
+ "needs",
+ "neither",
+ "never",
+ "nevertheless",
+ "new",
+ "next",
+ "nine",
+ "nobody",
+ "none",
+ "noone",
+ "nor",
+ "normally",
+ "nothing",
+ "novel",
+ "now",
+ "nowhere",
+ "obviously",
+ "off",
+ "often",
+ "oh",
+ "ok",
+ "okay",
+ "old",
+ "once",
+ "one",
+ "ones",
+ "only",
+ "onto",
+ "other",
+ "others",
+ "otherwise",
+ "our",
+ "ours",
+ "ourselves",
+ "out",
+ "outside",
+ "over",
+ "overall",
+ "own",
+ "particular",
+ "particularly",
+ "per",
+ "perhaps",
+ "placed",
+ "please",
+ "plus",
+ "possible",
+ "presumably",
+ "probably",
+ "provides",
+ "quite",
+ "rather",
+ "really",
+ "reasonably",
+ "regarding",
+ "regardless",
+ "regards",
+ "relatively",
+ "respectively",
+ "right",
+ "said",
+ "same",
+ "saw",
+ "say",
+ "saying",
+ "says",
+ "second",
+ "secondly",
+ "see",
+ "seeing",
+ "seem",
+ "seemed",
+ "seeming",
+ "seems",
+ "seen",
+ "self",
+ "selves",
+ "sensible",
+ "sent",
+ "serious",
+ "seriously",
+ "seven",
+ "several",
+ "shall",
+ "she",
+ "should",
+ "since",
+ "six",
+ "so",
+ "some",
+ "somebody",
+ "somehow",
+ "someone",
+ "something",
+ "sometime",
+ "sometimes",
+ "somewhat",
+ "somewhere",
+ "soon",
+ "sorry",
+ "specified",
+ "specify",
+ "specifying",
+ "still",
+ "sub",
+ "sup",
+ "sure",
+ "take",
+ "taken",
+ "tell",
+ "tends",
+ "th",
+ "than",
+ "thank",
+ "thanks",
+ "third",
+ "thorough",
+ "thoroughly",
+ "though",
+ "three",
+ "through",
+ "throughout",
+ "thru",
+ "thus",
+ "together",
+ "too",
+ "took",
+ "toward",
+ "towards",
+ "tried",
+ "tries",
+ "truly",
+ "try",
+ "trying",
+ "twice",
+ "two",
+ "un",
+ "under",
+ "unfortunately",
+ "unless",
+ "unlikely",
+ "until",
+ "unto",
+ "upon",
+ "us",
+ "use",
+ "used",
+ "useful",
+ "uses",
+ "using",
+ "usually",
+ "value",
+ "various",
+ "very",
+ "via",
+ "viz",
+ "vs",
+ "want",
+ "wants",
+ "we",
+ "well",
+ "went",
+ "were",
+ "what",
+ "whatever",
+ "when",
+ "whence",
+ "whenever",
+ "where",
+ "whereafter",
+ "whereas",
+ "whereby",
+ "wherein",
+ "whereupon",
+ "wherever",
+ "whether",
+ "which",
+ "while",
+ "whither",
+ "who",
+ "whoever",
+ "whole",
+ "whom",
+ "whose",
+ "why",
+ "within",
+ "without",
+ "wonder",
+ "would",
+ "yes",
+ "yet",
+ "you",
+ "your",
+ "yours",
+ "yourself",
+ "yourselves",
+ "zero",
];
// ── Tests ─────────────────────────────────────────────────────────────────────
@@ -1463,7 +1871,8 @@ mod tests {
fn registry_standard_analyzer_e2e() {
let registry = AnalyzerRegistry::default();
let analyzer = registry.get_analyzer("standard").unwrap();
- let terms = analyzer.analyze_to_terms("The quick brown foxes are jumping over the lazy dogs");
+ let terms =
+ analyzer.analyze_to_terms("The quick brown foxes are jumping over the lazy dogs");
// V4 — `standard` now matches ES semantics (lowercase + unicode
// tokenize, no stop words, no stemming). For stemming use the
// `english` analyzer explicitly.
diff --git a/engine/crates/xerj-fts/src/bm25.rs b/engine/crates/xerj-fts/src/bm25.rs
index 1b6101c77..8ad8d3773 100644
--- a/engine/crates/xerj-fts/src/bm25.rs
+++ b/engine/crates/xerj-fts/src/bm25.rs
@@ -58,7 +58,12 @@ impl Bm25Scorer {
/// Creates a scorer with custom k1/b parameters.
pub fn with_params(k1: f32, b: f32, avg_dl: f32, total_docs: u64) -> Self {
- Self { k1, b, avg_dl, total_docs }
+ Self {
+ k1,
+ b,
+ avg_dl,
+ total_docs,
+ }
}
/// Compute IDF for a term.
@@ -288,7 +293,12 @@ mod tests {
let scorer = Bm25Scorer::new(1.0, 100);
for n in 1u64..=100 {
let idf = scorer.idf(n);
- assert!(idf >= 0.0, "IDF must not be negative, got {} for n={}", idf, n);
+ assert!(
+ idf >= 0.0,
+ "IDF must not be negative, got {} for n={}",
+ idf,
+ n
+ );
}
}
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index 4714afd9b..919c3a266 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -25,7 +25,7 @@ use anyhow::{Context, Result};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use fst::{Map, MapBuilder};
use memmap2::Mmap;
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
use std::{
collections::HashMap,
fs::{self, File},
@@ -240,9 +240,7 @@ fn encode_field_meta_v4(
let tp = term_postings
.get(term)
.expect("sorted_terms must match term_postings keys");
- records
- .write_u32::<LittleEndian>(tp.doc_frequency)
- .unwrap();
+ records.write_u32::<LittleEndian>(tp.doc_frequency).unwrap();
records
.write_u64::<LittleEndian>(tp.total_term_frequency)
.unwrap();
@@ -254,18 +252,18 @@ fn encode_field_meta_v4(
.unwrap();
}
let uncompressed_len = records.len() as u32;
- let compressed = zstd::bulk::compress(&records, ZSTD_DURABLE_LEVEL)
- .with_context(|| "ZFM4 zstd compress")?;
- let mut out: Vec<u8> = Vec::with_capacity(
- ZFM3_HEADER_LEN + 4 + 4 + compressed.len(),
- );
+ let compressed =
+ zstd::bulk::compress(&records, ZSTD_DURABLE_LEVEL).with_context(|| "ZFM4 zstd compress")?;
+ let mut out: Vec<u8> = Vec::with_capacity(ZFM3_HEADER_LEN + 4 + 4 + compressed.len());
out.extend_from_slice(META_MAGIC_V4);
out.write_u64::<LittleEndian>(stats.total_docs).unwrap();
- out.write_u64::<LittleEndian>(stats.total_field_length).unwrap();
+ out.write_u64::<LittleEndian>(stats.total_field_length)
+ .unwrap();
out.write_u32::<LittleEndian>(num_terms as u32).unwrap();
out.push(if has_positions { 1u8 } else { 0u8 });
out.write_u32::<LittleEndian>(uncompressed_len).unwrap();
- out.write_u32::<LittleEndian>(compressed.len() as u32).unwrap();
+ out.write_u32::<LittleEndian>(compressed.len() as u32)
+ .unwrap();
out.extend_from_slice(&compressed);
Ok(out)
}
@@ -313,7 +311,10 @@ fn decode_field_meta_binary(bytes: &[u8]) -> Result<FieldMeta> {
));
}
return Ok(FieldMeta {
- stats: FieldStats { total_docs, total_field_length },
+ stats: FieldStats {
+ total_docs,
+ total_field_length,
+ },
terms: HashMap::new(),
has_positions,
fst_value_format: FstValueFormat::MetaByteOffset,
@@ -338,7 +339,10 @@ fn decode_field_meta_binary(bytes: &[u8]) -> Result<FieldMeta> {
));
}
return Ok(FieldMeta {
- stats: FieldStats { total_docs, total_field_length },
+ stats: FieldStats {
+ total_docs,
+ total_field_length,
+ },
// ZFM3 doesn't populate `terms` — lookups go through
// `flat_records` via the FST byte-offset value instead.
terms: HashMap::new(),
@@ -381,15 +385,21 @@ fn decode_field_meta_binary(bytes: &[u8]) -> Result<FieldMeta> {
let total_term_frequency = cur.read_u64::<LittleEndian>()?;
let postings_offset = cur.read_u64::<LittleEndian>()?;
let postings_length = cur.read_u32::<LittleEndian>()?;
- terms.insert(term, SerialTermPostings {
- doc_frequency,
- total_term_frequency,
- postings_offset,
- postings_length,
- });
+ terms.insert(
+ term,
+ SerialTermPostings {
+ doc_frequency,
+ total_term_frequency,
+ postings_offset,
+ postings_length,
+ },
+ );
}
Ok(FieldMeta {
- stats: FieldStats { total_docs, total_field_length },
+ stats: FieldStats {
+ total_docs,
+ total_field_length,
+ },
terms,
has_positions,
fst_value_format: FstValueFormat::PostingsOffset,
@@ -492,7 +502,9 @@ impl FtsIndexWriter {
let field_len = tokens.len() as u64;
// Record norms (capped at u16::MAX)
- field_data.norms.push((doc_id, field_len.min(u16::MAX as u64) as u16));
+ field_data
+ .norms
+ .push((doc_id, field_len.min(u16::MAX as u64) as u16));
// Update field stats
field_data.stats.total_docs += 1;
@@ -500,7 +512,9 @@ impl FtsIndexWriter {
// Accumulate postings
for token in &tokens {
- field_data.postings.add_occurrence(&token.text, doc_id, token.position);
+ field_data
+ .postings
+ .add_occurrence(&token.text, doc_id, token.position);
}
}
}
@@ -525,7 +539,10 @@ impl FtsIndexWriter {
/// Generic over the third tuple element (a source payload the caller
/// keeps alongside for its own use — `serde_json::Value`,
/// `Arc<serde_json::Value>`, …): this method never reads it.
- pub fn add_documents_parallel<S: Sync>(&mut self, docs: &[(String, HashMap<String, String>, S)]) {
+ pub fn add_documents_parallel<S: Sync>(
+ &mut self,
+ docs: &[(String, HashMap<String, String>, S)],
+ ) {
use rayon::prelude::*;
use std::collections::HashMap as StdHashMap;
@@ -577,10 +594,7 @@ impl FtsIndexWriter {
let built: Vec<(String, FieldData)> = per_field_vec
.into_par_iter()
.map(|(field_name, entries)| {
- let cfg = field_configs
- .get(&field_name)
- .cloned()
- .unwrap_or_default();
+ let cfg = field_configs.get(&field_name).cloned().unwrap_or_default();
let analyzer = registry
.get_analyzer(&cfg.analyzer)
.or_else(|| registry.get_analyzer("standard"))
@@ -601,7 +615,8 @@ impl FtsIndexWriter {
for (doc_ord, text) in entries {
let tokens = analyzer.analyze(text);
let field_len = tokens.len() as u64;
- fd.norms.push((doc_ord, field_len.min(u16::MAX as u64) as u16));
+ fd.norms
+ .push((doc_ord, field_len.min(u16::MAX as u64) as u16));
fd.stats.total_docs += 1;
fd.stats.total_field_length += field_len;
for token in &tokens {
@@ -652,13 +667,8 @@ impl FtsIndexWriter {
let results: Vec<Result<(String, FieldStats)>> = fields
.into_par_iter()
.map(|(field_name, stats, field_data)| {
- Self::write_field_static(
- &segment_dir,
- &segment_id,
- &field_name,
- field_data,
- )
- .with_context(|| format!("writing field '{}'", field_name))?;
+ Self::write_field_static(&segment_dir, &segment_id, &field_name, field_data)
+ .with_context(|| format!("writing field '{}'", field_name))?;
Ok((field_name, stats))
})
.collect();
@@ -683,9 +693,8 @@ impl FtsIndexWriter {
// so using `segment_dir.join("segment_id.field_name")` followed by
// `with_extension("fst")` would strip `.field_name` and collapse every
// field to the same file. Build filenames by hand to avoid that.
- let filename = |ext: &str| {
- segment_dir.join(format!("{}.{}.{}", segment_id, field_name, ext))
- };
+ let filename =
+ |ext: &str| segment_dir.join(format!("{}.{}.{}", segment_id, field_name, ext));
let fst_path = filename("fst");
let post_path = filename("post");
let meta_path = filename("meta");
@@ -696,11 +705,8 @@ impl FtsIndexWriter {
let mut term_postings: HashMap<String, TermPostings> = HashMap::new();
// Collect sorted terms (FST requires lexicographic order)
- let mut sorted_terms: Vec<String> = field_data
- .postings
- .terms()
- .map(|s| s.to_owned())
- .collect();
+ let mut sorted_terms: Vec<String> =
+ field_data.postings.terms().map(|s| s.to_owned()).collect();
sorted_terms.sort_unstable();
// Pre-compute per-term (doc_freq, ttf) ONCE. The previous code
@@ -718,10 +724,7 @@ impl FtsIndexWriter {
for term in &sorted_terms {
if let Some((offset, _skip)) = field_data.postings.encode_term(term, &mut post_data) {
// Calculate doc_freq and ttf from the writer's internal stats
- let (doc_freq, ttf) = stats_by_term
- .get(term.as_str())
- .copied()
- .unwrap_or((0, 0));
+ let (doc_freq, ttf) = stats_by_term.get(term.as_str()).copied().unwrap_or((0, 0));
let end_offset = post_data.len() as u64;
let length = (end_offset - offset) as u32;
@@ -780,20 +783,17 @@ impl FtsIndexWriter {
// holds df + ttf + postings_offset + length, so one FST hit →
// one bounded mmap read.
let fst_file = BufWriter::new(
- File::create(&fst_path)
- .with_context(|| format!("creating FST file {:?}", fst_path))?,
+ File::create(&fst_path).with_context(|| format!("creating FST file {:?}", fst_path))?,
);
- let mut fst_builder = MapBuilder::new(fst_file)
- .with_context(|| "creating FST builder")?;
+ let mut fst_builder = MapBuilder::new(fst_file).with_context(|| "creating FST builder")?;
// The i-th sorted term gets record slot `i`, whose byte offset
// inside the flat meta array is `ZFM3_HEADER_LEN + i * 24`. We
// iterate the same sorted_terms list that built the meta body,
// so ordering is consistent.
for (i, term) in sorted_terms.iter().enumerate() {
- if term_postings.get(term).is_some() {
- let meta_offset =
- (ZFM3_HEADER_LEN + i * ZFM3_RECORD_LEN) as u64;
+ if term_postings.contains_key(term) {
+ let meta_offset = (ZFM3_HEADER_LEN + i * ZFM3_RECORD_LEN) as u64;
fst_builder
.insert(term.as_bytes(), meta_offset)
.with_context(|| format!("inserting term '{}' into FST", term))?;
@@ -845,7 +845,8 @@ impl FtsIndexWriter {
// to pay off (long runs of identical norms on low-entropy fields
// like nginx `method` compress ~8×).
let lz4_try = lz4_flex::compress_prepend_size(&dense);
- let (encoding, payload): (u8, &[u8]) = if dense.len() > 1024 && lz4_try.len() < dense.len() {
+ let (encoding, payload): (u8, &[u8]) = if dense.len() > 1024 && lz4_try.len() < dense.len()
+ {
(1, &lz4_try[..])
} else {
(0, &dense[..])
@@ -873,9 +874,13 @@ const NORMS_MAGIC: &[u8; 4] = b"ZNM1";
/// lengths, same as Lucene's `SmallFloat`.
#[inline]
fn norm_u16_to_u8(len: u16) -> u8 {
- if len == 0 { return 0; }
+ if len == 0 {
+ return 0;
+ }
// Clamp short lengths [1..8] to direct encoding (0..7).
- if len < 8 { return (len - 1) as u8 & 0x07; }
+ if len < 8 {
+ return (len - 1) as u8 & 0x07;
+ }
// Logarithmic scale beyond 8.
let l = (len as f64).log2();
let v = ((l - 3.0) * 32.0 + 8.0) as i32;
@@ -885,7 +890,9 @@ fn norm_u16_to_u8(len: u16) -> u8 {
#[inline]
#[allow(dead_code)]
fn norm_u8_to_u16(b: u8) -> u16 {
- if b < 8 { return (b + 1) as u16; }
+ if b < 8 {
+ return (b + 1) as u16;
+ }
let l = ((b - 8) as f64) / 32.0 + 3.0;
let v = l.exp2();
v.min(u16::MAX as f64) as u16
@@ -972,9 +979,8 @@ impl FtsIndexReader {
for &field_name in field_names {
// Build filenames explicitly — see note in `write_field_static`
// for why `with_extension()` would be wrong here.
- let filename = |ext: &str| {
- segment_dir.join(format!("{}.{}.{}", segment_id, field_name, ext))
- };
+ let filename =
+ |ext: &str| segment_dir.join(format!("{}.{}.{}", segment_id, field_name, ext));
let fst_path = filename("fst");
let post_path = filename("post");
let meta_path = filename("meta");
@@ -988,15 +994,13 @@ impl FtsIndexReader {
// ── FST ──────────────────────────────────────────────────
// Prefer mmap; fall back to fs::read if mmap fails (tmpfs etc.).
let fst = match Self::mmap_file(&fst_path) {
- Ok(mmap) => FstData::Mmap(
- Map::new(mmap).with_context(|| "parsing FST map (mmap)")?,
- ),
+ Ok(mmap) => {
+ FstData::Mmap(Map::new(mmap).with_context(|| "parsing FST map (mmap)")?)
+ }
Err(_) => {
let fst_bytes = fs::read(&fst_path)
.with_context(|| format!("reading FST {:?}", fst_path))?;
- FstData::Owned(
- Map::new(fst_bytes).with_context(|| "parsing FST map (owned)")?,
- )
+ FstData::Owned(Map::new(fst_bytes).with_context(|| "parsing FST map (owned)")?)
}
};
@@ -1039,8 +1043,8 @@ impl FtsIndexReader {
// Auto-detect the on-disk format: ZFM1 binary (new) vs
// legacy pretty-JSON (from pre-M4.7 segments). We keep the
// JSON fallback so upgrades don't require a reindex.
- let meta_bytes = fs::read(&meta_path)
- .with_context(|| format!("reading meta {:?}", meta_path))?;
+ let meta_bytes =
+ fs::read(&meta_path).with_context(|| format!("reading meta {:?}", meta_path))?;
let is_binary = meta_bytes.len() >= 4
&& (&meta_bytes[..4] == META_MAGIC_V1
|| &meta_bytes[..4] == META_MAGIC_V2
@@ -1067,15 +1071,21 @@ impl FtsIndexReader {
.with_context(|| "parsing legacy field meta JSON")?;
FieldMeta {
stats: legacy.stats,
- terms: legacy.terms.into_iter().map(|(k, v)| (
- k,
- SerialTermPostings {
- doc_frequency: v.doc_frequency,
- total_term_frequency: v.total_term_frequency,
- postings_offset: v.postings_offset,
- postings_length: v.postings_length,
- },
- )).collect(),
+ terms: legacy
+ .terms
+ .into_iter()
+ .map(|(k, v)| {
+ (
+ k,
+ SerialTermPostings {
+ doc_frequency: v.doc_frequency,
+ total_term_frequency: v.total_term_frequency,
+ postings_offset: v.postings_offset,
+ postings_length: v.postings_length,
+ },
+ )
+ })
+ .collect(),
has_positions: true, // legacy segments always had positions
fst_value_format: FstValueFormat::PostingsOffset,
flat_records: Vec::new(),
@@ -1086,11 +1096,20 @@ impl FtsIndexReader {
fields.insert(
field_name.to_owned(),
- LoadedField { fst, post_data, meta, norms },
+ LoadedField {
+ fst,
+ post_data,
+ meta,
+ norms,
+ },
);
}
- Ok(Self { segment_dir, segment_id, fields })
+ Ok(Self {
+ segment_dir,
+ segment_id,
+ fields,
+ })
}
/// Memory-map a file.
@@ -1101,24 +1120,18 @@ impl FtsIndexReader {
/// immutable until merged away. We only mmap these stable files, so
/// this is safe in practice.
fn mmap_file(path: &Path) -> Result<Mmap> {
- let file = File::open(path)
- .with_context(|| format!("opening {:?} for mmap", path))?;
+ let file = File::open(path).with_context(|| format!("opening {:?} for mmap", path))?;
// Zero-length files would panic — short-circuit.
- let len = file
- .metadata()
- .with_context(|| "stat for mmap")?
- .len();
+ let len = file.metadata().with_context(|| "stat for mmap")?.len();
if len == 0 {
return Err(anyhow::anyhow!("empty file"));
}
- let mmap = unsafe { Mmap::map(&file) }
- .with_context(|| format!("mmap {:?}", path))?;
+ let mmap = unsafe { Mmap::map(&file) }.with_context(|| format!("mmap {:?}", path))?;
Ok(mmap)
}
fn load_norms(path: &Path) -> Result<Vec<(u32, u16)>> {
- let bytes = fs::read(path)
- .with_context(|| format!("opening norms {:?}", path))?;
+ let bytes = fs::read(path).with_context(|| format!("opening norms {:?}", path))?;
// V4 M4.7 compact format starts with `NORMS_MAGIC`; legacy starts
// with a raw u32 count (whose first byte almost never matches 'Z').
if bytes.len() >= 4 && &bytes[..4] == NORMS_MAGIC {
@@ -1137,7 +1150,9 @@ impl FtsIndexReader {
};
if dense.len() != dense_len {
return Err(anyhow::anyhow!(
- "norms: dense length mismatch {} != {}", dense.len(), dense_len
+ "norms: dense length mismatch {} != {}",
+ dense.len(),
+ dense_len
));
}
// Materialise (doc_id, norm) pairs only for live docs so the
@@ -1176,7 +1191,10 @@ impl FtsIndexReader {
// record inside the flat `.meta` array.
let off = fst_value as usize;
let end = off.checked_add(ZFM3_RECORD_LEN)?;
- let rec = loaded.meta.flat_records.get(off.checked_sub(ZFM3_HEADER_LEN)?..end.checked_sub(ZFM3_HEADER_LEN)?)?;
+ let rec = loaded
+ .meta
+ .flat_records
+ .get(off.checked_sub(ZFM3_HEADER_LEN)?..end.checked_sub(ZFM3_HEADER_LEN)?)?;
// rec is exactly 24 bytes: df(4) ttf(8) off(8) len(4).
let mut cur = std::io::Cursor::new(rec);
let doc_frequency = cur.read_u32::<LittleEndian>().ok()?;
@@ -1318,9 +1336,15 @@ mod tests {
let mut writer = FtsIndexWriter::new(dir.path(), "seg0", registry);
let docs: Vec<HashMap<String, String>> = vec![
- [("body".to_owned(), "the quick brown fox".to_owned())].into_iter().collect(),
- [("body".to_owned(), "the lazy dog".to_owned())].into_iter().collect(),
- [("body".to_owned(), "quick fox lazy dog".to_owned())].into_iter().collect(),
+ [("body".to_owned(), "the quick brown fox".to_owned())]
+ .into_iter()
+ .collect(),
+ [("body".to_owned(), "the lazy dog".to_owned())]
+ .into_iter()
+ .collect(),
+ [("body".to_owned(), "quick fox lazy dog".to_owned())]
+ .into_iter()
+ .collect(),
];
for (i, doc) in docs.iter().enumerate() {
@@ -1340,9 +1364,12 @@ mod tests {
assert!(!all_terms.is_empty(), "should have indexed terms");
// "lazi" is the Snowball stem of "lazy"
- let lazi_exists = reader.term_exists("body", "lazi")
- || reader.term_exists("body", "lazy");
- assert!(lazi_exists, "lazy/lazi should be indexed; terms={:?}", all_terms);
+ let lazi_exists = reader.term_exists("body", "lazi") || reader.term_exists("body", "lazy");
+ assert!(
+ lazi_exists,
+ "lazy/lazi should be indexed; terms={:?}",
+ all_terms
+ );
// Field stats round-trip
let fs = reader.field_stats("body").unwrap();
@@ -1364,13 +1391,19 @@ mod tests {
let mut writer = FtsIndexWriter::new(dir.path(), "seg1", registry);
// Use keyword analyzer to avoid stemming surprises
- let mut cfg = FieldIndexConfig::default();
- cfg.analyzer = "whitespace".to_owned();
+ let cfg = FieldIndexConfig {
+ analyzer: "whitespace".to_owned(),
+ ..Default::default()
+ };
writer.configure_field("title", cfg);
- let docs = vec![
- [("title".to_owned(), "hello world".to_owned())].into_iter().collect(),
- [("title".to_owned(), "hello rust".to_owned())].into_iter().collect(),
+ let docs = [
+ [("title".to_owned(), "hello world".to_owned())]
+ .into_iter()
+ .collect(),
+ [("title".to_owned(), "hello rust".to_owned())]
+ .into_iter()
+ .collect(),
];
for (i, doc) in docs.iter().enumerate() {
writer.add_document(i as u32, doc);
@@ -1380,12 +1413,16 @@ mod tests {
let reader = FtsIndexReader::open(dir.path(), "seg1", &["title"]).unwrap();
// "hello" appears in both docs
- let tp = reader.lookup_term("title", "hello").expect("'hello' should be in index");
+ let tp = reader
+ .lookup_term("title", "hello")
+ .expect("'hello' should be in index");
assert_eq!(tp.doc_frequency, 2);
assert_eq!(tp.total_term_frequency, 2);
// "world" appears in 1 doc
- let tp = reader.lookup_term("title", "world").expect("'world' should be in index");
+ let tp = reader
+ .lookup_term("title", "world")
+ .expect("'world' should be in index");
assert_eq!(tp.doc_frequency, 1);
// Postings data retrievable
diff --git a/engine/crates/xerj-fts/src/lib.rs b/engine/crates/xerj-fts/src/lib.rs
index 84ccc0109..b174ed925 100644
--- a/engine/crates/xerj-fts/src/lib.rs
+++ b/engine/crates/xerj-fts/src/lib.rs
@@ -65,14 +65,13 @@ pub mod search;
pub use analyzer::{
AnalyzerPipeline, AnalyzerRegistry, AsciiFoldingFilter, CharFilter, CjkTokenizer,
EdgeNGramTokenizer, IcuFoldingFilter, KeywordTokenizer, LengthFilter, LowercaseFilter,
- NGramTokenizer, PatternTokenizer, ShingleFilter, StemmerFilter, StandardTokenizer,
+ NGramTokenizer, PatternTokenizer, ShingleFilter, StandardTokenizer, StemmerFilter,
StopwordsFilter, SynonymFilter, ThaiTokenizer, Token, TokenFilter, Tokenizer,
WhitespaceTokenizer,
};
pub use bm25::{Bm25Scorer, FieldStats, QueryExplanation, ScoreBreakdown, DEFAULT_B, DEFAULT_K1};
-
pub use index::{FieldIndexConfig, FtsIndexReader, FtsIndexWriter};
pub use postings::{
@@ -80,6 +79,5 @@ pub use postings::{
};
pub use search::{
- search_segments, BoolQuery, FtsSearcher, PhraseQuery, PrefixQuery, Query, ScoredHit,
- TermQuery,
+ search_segments, BoolQuery, FtsSearcher, PhraseQuery, PrefixQuery, Query, ScoredHit, TermQuery,
};
diff --git a/engine/crates/xerj-fts/src/postings.rs b/engine/crates/xerj-fts/src/postings.rs
index 0f321b0f0..b3e67253b 100644
--- a/engine/crates/xerj-fts/src/postings.rs
+++ b/engine/crates/xerj-fts/src/postings.rs
@@ -55,7 +55,8 @@ impl TermPostings {
let mut buf = [0u8; 20];
let mut c = Cursor::new(buf.as_mut_slice());
c.write_u32::<LittleEndian>(self.doc_frequency).unwrap();
- c.write_u64::<LittleEndian>(self.total_term_frequency).unwrap();
+ c.write_u64::<LittleEndian>(self.total_term_frequency)
+ .unwrap();
c.write_u64::<LittleEndian>(self.postings_offset).unwrap();
// last 4 bytes: length — split from offset to stay 20 B
// rewrite cleanly:
@@ -71,7 +72,10 @@ impl TermPostings {
/// FST value (offset only fits in 64 bits); length is stored here.
pub fn decode(buf: &[u8]) -> io::Result<Self> {
if buf.len() < 20 {
- return Err(io::Error::new(io::ErrorKind::InvalidData, "buffer too short for TermPostings"));
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ "buffer too short for TermPostings",
+ ));
}
let mut c = Cursor::new(buf);
let doc_frequency = c.read_u32::<LittleEndian>()?;
@@ -244,7 +248,7 @@ impl PostingsWriter {
// Full block
let block = &postings[i..i + BLOCK_SIZE];
- if block_count % SKIP_INTERVAL == 0 {
+ if block_count.is_multiple_of(SKIP_INTERVAL) {
skip_table.push(SkipEntry {
max_doc_id: block[BLOCK_SIZE - 1].doc_id,
byte_offset: (output.len() - start_offset as usize) as u32,
@@ -269,7 +273,12 @@ impl PostingsWriter {
// block's final doc_id as well.
if i < n {
let residual = &postings[i..];
- encode_residual(residual, output, prev_block_last_doc_id, self.store_positions);
+ encode_residual(
+ residual,
+ output,
+ prev_block_last_doc_id,
+ self.store_positions,
+ );
}
Some((start_offset, skip_table))
@@ -297,11 +306,15 @@ impl Default for PostingsWriter {
fn pack_u32_block(values: &[u32; BLOCK_SIZE], output: &mut Vec<u8>) {
use bitpacking::BitPacker;
let max_val = *values.iter().max().unwrap_or(&0);
- let num_bits = if max_val == 0 { 1u8 } else { (32 - max_val.leading_zeros()) as u8 };
+ let num_bits = if max_val == 0 {
+ 1u8
+ } else {
+ (32 - max_val.leading_zeros()) as u8
+ };
let num_bits = num_bits.min(32);
// bitpacking::BitPacker4x::BLOCK_LEN == 128
- let byte_len = (BLOCK_SIZE * num_bits as usize + 7) / 8;
+ let byte_len = (BLOCK_SIZE * num_bits as usize).div_ceil(8);
let mut compressed = vec![0u8; byte_len];
let packer = bitpacking::BitPacker4x::new();
@@ -314,14 +327,21 @@ fn pack_u32_block(values: &[u32; BLOCK_SIZE], output: &mut Vec<u8>) {
/// Decode a 128-element block previously written by `pack_u32_block`.
/// Reads cursor forward past the data and fills `out`.
-fn unpack_u32_block(data: &[u8], cursor: &mut Cursor<&[u8]>, out: &mut [u32; BLOCK_SIZE]) -> io::Result<()> {
+fn unpack_u32_block(
+ data: &[u8],
+ cursor: &mut Cursor<&[u8]>,
+ out: &mut [u32; BLOCK_SIZE],
+) -> io::Result<()> {
use bitpacking::BitPacker;
let num_bits = cursor.read_u8()?;
let byte_len = cursor.read_u32::<LittleEndian>()? as usize;
let start = cursor.position() as usize;
let end = start + byte_len;
if end > data.len() {
- return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "postings data truncated"));
+ return Err(io::Error::new(
+ io::ErrorKind::UnexpectedEof,
+ "postings data truncated",
+ ));
}
let compressed = &data[start..end];
bitpacking::BitPacker4x::new().decompress(compressed, out, num_bits);
@@ -366,7 +386,9 @@ fn encode_block_positions(block: &[RawPosting], output: &mut Vec<u8>) {
prev = pos;
}
}
- output.write_u32::<LittleEndian>(pos_buf.len() as u32).unwrap();
+ output
+ .write_u32::<LittleEndian>(pos_buf.len() as u32)
+ .unwrap();
output.extend_from_slice(&pos_buf);
}
@@ -511,6 +533,11 @@ impl<'a> PostingsReader<'a> {
}
/// Advance to the next posting. Returns `None` when exhausted.
+ ///
+ /// An inherent cursor method rather than an `Iterator` impl: decoding
+ /// borrows the reader's internal buffers, which doesn't fit `Iterator`'s
+ /// `Item` lifetime, so `next` stays inherent by design.
+ #[allow(clippy::should_implement_trait)]
pub fn next(&mut self) -> Option<DecodedPosting> {
if self.docs_read >= self.doc_frequency {
return None;
@@ -543,7 +570,11 @@ impl<'a> PostingsReader<'a> {
self.block_idx += 1;
self.docs_read += 1;
- Some(DecodedPosting { doc_id, term_freq, positions })
+ Some(DecodedPosting {
+ doc_id,
+ term_freq,
+ positions,
+ })
}
fn decode_next_full_block(&mut self) -> io::Result<()> {
@@ -625,7 +656,11 @@ impl<'a> PostingsReader<'a> {
(1u32, Vec::new())
};
- result.push(DecodedPosting { doc_id, term_freq, positions });
+ result.push(DecodedPosting {
+ doc_id,
+ term_freq,
+ positions,
+ });
}
self.residual = result;
@@ -667,11 +702,8 @@ mod tests {
#[test]
fn roundtrip_small_posting_list() {
- let postings: Vec<(u32, u32, &[u32])> = vec![
- (1, 2, &[0, 5]),
- (3, 1, &[2]),
- (7, 3, &[0, 1, 2]),
- ];
+ let postings: Vec<(u32, u32, &[u32])> =
+ vec![(1, 2, &[0, 5]), (3, 1, &[2]), (7, 3, &[0, 1, 2])];
let (data, doc_freq) = build_postings(&postings);
let mut reader = PostingsReader::new(&data, doc_freq);
@@ -704,8 +736,10 @@ mod tests {
let mut last_doc = u32::MAX;
let mut count = 0u32;
while let Some(p) = reader.next() {
- assert!(p.doc_id < last_doc || last_doc == u32::MAX || p.doc_id > last_doc,
- "doc IDs must be monotonically increasing");
+ assert!(
+ p.doc_id < last_doc || last_doc == u32::MAX || p.doc_id > last_doc,
+ "doc IDs must be monotonically increasing"
+ );
if last_doc != u32::MAX {
assert!(p.doc_id > last_doc);
}
diff --git a/engine/crates/xerj-fts/src/search.rs b/engine/crates/xerj-fts/src/search.rs
index 7c0aad250..234ab161a 100644
--- a/engine/crates/xerj-fts/src/search.rs
+++ b/engine/crates/xerj-fts/src/search.rs
@@ -201,7 +201,11 @@ pub struct DisMaxQuery {
impl DisMaxQuery {
pub fn new(queries: Vec<Query>) -> Self {
- Self { queries, tie_breaker: 0.0, boost: 1.0 }
+ Self {
+ queries,
+ tie_breaker: 0.0,
+ boost: 1.0,
+ }
}
pub fn tie_breaker(mut self, t: f32) -> Self {
@@ -276,9 +280,8 @@ impl FtsSearcher {
};
let has_positions = self.reader.field_has_positions(&tq.field);
- let mut reader = PostingsReader::new_with_positions(
- post_data, tp.doc_frequency, has_positions,
- );
+ let mut reader =
+ PostingsReader::new_with_positions(post_data, tp.doc_frequency, has_positions);
let mut hits = Vec::with_capacity(tp.doc_frequency as usize);
while let Some(posting) = reader.next() {
@@ -298,15 +301,16 @@ impl FtsSearcher {
let boosted_bd = ScoreBreakdown { score: s, ..bd };
(s, Some(QueryExplanation::new(vec![boosted_bd])))
} else {
- let s = scorer.score_term(
- tp.doc_frequency as u64,
- posting.term_freq,
- doc_len,
- ) * tq.boost;
+ let s = scorer.score_term(tp.doc_frequency as u64, posting.term_freq, doc_len)
+ * tq.boost;
(s, None)
};
- hits.push(ScoredHit { doc_id: posting.doc_id, score, explanation });
+ hits.push(ScoredHit {
+ doc_id: posting.doc_id,
+ score,
+ explanation,
+ });
}
Ok(hits)
@@ -343,9 +347,8 @@ impl FtsSearcher {
if !has_positions {
return Ok(Vec::new());
}
- let mut reader = PostingsReader::new_with_positions(
- post_data, tp.doc_frequency, has_positions,
- );
+ let mut reader =
+ PostingsReader::new_with_positions(post_data, tp.doc_frequency, has_positions);
let mut postings: Vec<DecodedPosting> = Vec::new();
while let Some(p) = reader.next() {
postings.push(p);
@@ -380,7 +383,9 @@ impl FtsSearcher {
} else {
0 // will adjust relative to anchor below
};
- let adjusted: Vec<u32> = p.positions.iter()
+ let adjusted: Vec<u32> = p
+ .positions
+ .iter()
.filter_map(|&pos| pos.checked_sub(offset))
.collect();
all_positions.push(adjusted);
@@ -424,7 +429,11 @@ impl FtsSearcher {
None
};
- hits.push(ScoredHit { doc_id, score: total_score, explanation });
+ hits.push(ScoredHit {
+ doc_id,
+ score: total_score,
+ explanation,
+ });
}
Ok(hits)
@@ -465,7 +474,11 @@ impl FtsSearcher {
Ok(score_map
.into_iter()
- .map(|(doc_id, (score, explanation))| ScoredHit { doc_id, score, explanation })
+ .map(|(doc_id, (score, explanation))| ScoredHit {
+ doc_id,
+ score,
+ explanation,
+ })
.collect())
}
@@ -484,7 +497,11 @@ impl FtsSearcher {
fn execute_bool(&self, bq: &BoolQuery, explain: bool) -> Result<Vec<ScoredHit>> {
// Determine effective min_should_match
let min_should = bq.min_should_match.unwrap_or(if bq.must.is_empty() {
- if bq.should.is_empty() { 0 } else { 1 }
+ if bq.should.is_empty() {
+ 0
+ } else {
+ 1
+ }
} else {
0
});
@@ -566,7 +583,11 @@ impl FtsSearcher {
})
.map(|doc_id| {
let score = score_map.get(&doc_id).copied().unwrap_or(0.0) * bq.boost;
- ScoredHit { doc_id, score, explanation: None }
+ ScoredHit {
+ doc_id,
+ score,
+ explanation: None,
+ }
})
.collect();
@@ -580,8 +601,7 @@ impl FtsSearcher {
/// all multiplied by the query boost.
fn execute_dis_max(&self, dq: &DisMaxQuery, explain: bool) -> Result<Vec<ScoredHit>> {
// Per-doc (max, sum) accumulator over sub-query scores.
- let mut acc: std::collections::HashMap<u32, (f32, f32)> =
- std::collections::HashMap::new();
+ let mut acc: std::collections::HashMap<u32, (f32, f32)> = std::collections::HashMap::new();
for sub in &dq.queries {
for hit in self.execute(sub, explain)? {
let e = acc.entry(hit.doc_id).or_insert((f32::NEG_INFINITY, 0.0));
@@ -595,7 +615,11 @@ impl FtsSearcher {
.into_iter()
.map(|(doc_id, (max, sum))| {
let score = (max + dq.tie_breaker * (sum - max)) * dq.boost;
- ScoredHit { doc_id, score, explanation: None }
+ ScoredHit {
+ doc_id,
+ score,
+ explanation: None,
+ }
})
.collect())
}
@@ -663,16 +687,17 @@ fn phrase_matches(all_positions: &[Vec<u32>], slop: u32) -> bool {
let mut current_pos = start_pos;
for positions in &all_positions[1..] {
// Find the first position in `positions` that is within [current_pos, current_pos + slop + 1]
- let found = positions.iter().any(|&p| {
- p >= current_pos && p <= current_pos + slop + 1
- });
+ let found = positions
+ .iter()
+ .any(|&p| p >= current_pos && p <= current_pos + slop + 1);
if !found {
continue 'outer;
}
// Advance current_pos to the matched position
- if let Some(&next_pos) = positions.iter().find(|&&p| {
- p >= current_pos && p <= current_pos + slop + 1
- }) {
+ if let Some(&next_pos) = positions
+ .iter()
+ .find(|&&p| p >= current_pos && p <= current_pos + slop + 1)
+ {
current_pos = next_pos;
}
}
@@ -733,11 +758,13 @@ mod tests {
let mut writer = FtsIndexWriter::new(dir, "seg0", Arc::clone(&registry));
// Use whitespace analyzer to avoid stemming surprises in tests
- let mut cfg = FieldIndexConfig::default();
- cfg.analyzer = "whitespace".to_owned();
+ let cfg = FieldIndexConfig {
+ analyzer: "whitespace".to_owned(),
+ ..Default::default()
+ };
writer.configure_field("body", cfg);
- let docs = vec![
+ let docs = [
"the quick brown fox jumps over the lazy dog",
"quick brown fox",
"the lazy dog slept",
@@ -746,8 +773,9 @@ mod tests {
];
for (i, text) in docs.iter().enumerate() {
- let fields: HashMap<String, String> =
- [("body".to_owned(), text.to_string())].into_iter().collect();
+ let fields: HashMap<String, String> = [("body".to_owned(), text.to_string())]
+ .into_iter()
+ .collect();
writer.add_document(i as u32, &fields);
}
writer.finish().unwrap();
@@ -765,7 +793,11 @@ mod tests {
let hits = searcher.search(&q, 10, false).unwrap();
// "fox" appears in docs 0, 1, 4
- assert!(hits.len() >= 2, "expected at least 2 hits for 'fox', got {}", hits.len());
+ assert!(
+ hits.len() >= 2,
+ "expected at least 2 hits for 'fox', got {}",
+ hits.len()
+ );
let doc_ids: Vec<u32> = hits.iter().map(|h| h.doc_id).collect();
assert!(doc_ids.contains(&1), "doc 1 should match");
assert!(doc_ids.contains(&4), "doc 4 should match (triple fox)");
@@ -783,7 +815,12 @@ mod tests {
let doc4_score = hits.iter().find(|h| h.doc_id == 4).map(|h| h.score);
let doc1_score = hits.iter().find(|h| h.doc_id == 1).map(|h| h.score);
if let (Some(s4), Some(s1)) = (doc4_score, doc1_score) {
- assert!(s4 > s1, "doc4 (TF=4) should outscore doc1 (TF=1): {} vs {}", s4, s1);
+ assert!(
+ s4 > s1,
+ "doc4 (TF=4) should outscore doc1 (TF=1): {} vs {}",
+ s4,
+ s1
+ );
}
}
@@ -796,7 +833,10 @@ mod tests {
let hits = searcher.search(&q, 10, true).unwrap();
for hit in &hits {
- assert!(hit.explanation.is_some(), "explain mode should populate explanation");
+ assert!(
+ hit.explanation.is_some(),
+ "explain mode should populate explanation"
+ );
let exp = hit.explanation.as_ref().unwrap();
assert!(!exp.term_breakdowns.is_empty());
assert!(exp.total_score > 0.0);
@@ -813,7 +853,10 @@ mod tests {
let hits = searcher.search(&q, 10, false).unwrap();
// "lazy" appears in docs 0 and 2
- assert!(!hits.is_empty(), "prefix 'la' should match docs containing 'lazy'");
+ assert!(
+ !hits.is_empty(),
+ "prefix 'la' should match docs containing 'lazy'"
+ );
}
#[test]
@@ -831,8 +874,14 @@ mod tests {
// Docs 0 and 1 contain both "fox" and "quick"
let doc_ids: Vec<u32> = hits.iter().map(|h| h.doc_id).collect();
- assert!(doc_ids.contains(&0), "doc0 should match must(fox AND quick)");
- assert!(doc_ids.contains(&1), "doc1 should match must(fox AND quick)");
+ assert!(
+ doc_ids.contains(&0),
+ "doc0 should match must(fox AND quick)"
+ );
+ assert!(
+ doc_ids.contains(&1),
+ "doc1 should match must(fox AND quick)"
+ );
// Doc 4 has fox but not quick
assert!(!doc_ids.contains(&4), "doc4 should NOT match (no 'quick')");
}
@@ -878,8 +927,10 @@ mod tests {
let dir = TempDir::new().unwrap();
let registry = Arc::new(AnalyzerRegistry::default());
let mut writer = FtsIndexWriter::new(dir.path(), "seg0", Arc::clone(&registry));
- let mut cfg = FieldIndexConfig::default();
- cfg.analyzer = "whitespace".to_owned();
+ let cfg = FieldIndexConfig {
+ analyzer: "whitespace".to_owned(),
+ ..Default::default()
+ };
writer.configure_field("title", cfg.clone());
writer.configure_field("body", cfg);
@@ -915,11 +966,21 @@ mod tests {
let dm_hits = searcher.search(&dm, 10, false).unwrap();
// Union of matching docs: all three.
- assert_eq!(dm_hits.len(), 3, "dis_max must return the union of sub-query docs");
+ assert_eq!(
+ dm_hits.len(),
+ 3,
+ "dis_max must return the union of sub-query docs"
+ );
for hit in &dm_hits {
- let ts = title_hits.iter().find(|h| h.doc_id == hit.doc_id).map(|h| h.score);
- let bs = body_hits.iter().find(|h| h.doc_id == hit.doc_id).map(|h| h.score);
+ let ts = title_hits
+ .iter()
+ .find(|h| h.doc_id == hit.doc_id)
+ .map(|h| h.score);
+ let bs = body_hits
+ .iter()
+ .find(|h| h.doc_id == hit.doc_id)
+ .map(|h| h.score);
let expected = ts.unwrap_or(0.0).max(bs.unwrap_or(0.0));
assert!(hit.score > 0.0, "dis_max hit must have positive score");
assert!(
diff --git a/engine/crates/xerj-logs/src/columnar.rs b/engine/crates/xerj-logs/src/columnar.rs
index 4c8953db6..34cb0d0bd 100644
--- a/engine/crates/xerj-logs/src/columnar.rs
+++ b/engine/crates/xerj-logs/src/columnar.rs
@@ -7,7 +7,7 @@
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use serde::{Deserialize, Serialize};
-use std::io::{Cursor, Read, Write};
+use std::io::{Cursor, Read};
use xerj_common::XerjError;
use xerj_compress::{
codec::{get_codec, CompressionLevel},
@@ -163,7 +163,10 @@ pub struct ColumnWriter {
impl ColumnWriter {
pub fn new(column: Column, compression: CompressionLevel) -> Self {
- Self { column, compression }
+ Self {
+ column,
+ compression,
+ }
}
/// Serialize and compress the column, returning the encoded bytes.
@@ -179,10 +182,15 @@ impl ColumnWriter {
}
fn encode_timestamps(&self) -> Result<Vec<u8>> {
- let vals: Vec<i64> = self.column.values.iter().map(|v| match v {
- ColumnValue::Timestamp(t) => *t,
- _ => 0,
- }).collect();
+ let vals: Vec<i64> = self
+ .column
+ .values
+ .iter()
+ .map(|v| match v {
+ ColumnValue::Timestamp(t) => *t,
+ _ => 0,
+ })
+ .collect();
let first = vals.first().copied().unwrap_or(0);
@@ -212,10 +220,15 @@ impl ColumnWriter {
}
fn encode_i64s(&self) -> Result<Vec<u8>> {
- let vals: Vec<i64> = self.column.values.iter().map(|v| match v {
- ColumnValue::I64(n) => *n,
- _ => 0,
- }).collect();
+ let vals: Vec<i64> = self
+ .column
+ .values
+ .iter()
+ .map(|v| match v {
+ ColumnValue::I64(n) => *n,
+ _ => 0,
+ })
+ .collect();
let mut buf = Vec::with_capacity(vals.len() * 8 + 4);
buf.write_u32::<LittleEndian>(vals.len() as u32)
@@ -233,10 +246,15 @@ impl ColumnWriter {
}
fn encode_f64s(&self) -> Result<Vec<u8>> {
- let vals: Vec<f64> = self.column.values.iter().map(|v| match v {
- ColumnValue::F64(f) => *f,
- _ => 0.0,
- }).collect();
+ let vals: Vec<f64> = self
+ .column
+ .values
+ .iter()
+ .map(|v| match v {
+ ColumnValue::F64(f) => *f,
+ _ => 0.0,
+ })
+ .collect();
let mut buf = Vec::with_capacity(vals.len() * 8 + 4);
buf.write_u32::<LittleEndian>(vals.len() as u32)
@@ -250,10 +268,15 @@ impl ColumnWriter {
}
fn encode_strings(&self) -> Result<Vec<u8>> {
- let strs: Vec<&str> = self.column.values.iter().map(|v| match v {
- ColumnValue::String(s) => s.as_str(),
- _ => "",
- }).collect();
+ let strs: Vec<&str> = self
+ .column
+ .values
+ .iter()
+ .map(|v| match v {
+ ColumnValue::String(s) => s.as_str(),
+ _ => "",
+ })
+ .collect();
// Try dictionary encoding if cardinality is low
let distinct: std::collections::HashSet<&str> = strs.iter().copied().collect();
@@ -293,13 +316,18 @@ impl ColumnWriter {
}
fn encode_bools(&self) -> Result<Vec<u8>> {
- let vals: Vec<bool> = self.column.values.iter().map(|v| match v {
- ColumnValue::Bool(b) => *b,
- _ => false,
- }).collect();
+ let vals: Vec<bool> = self
+ .column
+ .values
+ .iter()
+ .map(|v| match v {
+ ColumnValue::Bool(b) => *b,
+ _ => false,
+ })
+ .collect();
// Pack 8 bools per byte
- let byte_count = (vals.len() + 7) / 8;
+ let byte_count = vals.len().div_ceil(8);
let mut packed = vec![0u8; byte_count];
for (i, &b) in vals.iter().enumerate() {
if b {
@@ -376,9 +404,11 @@ impl ColumnReader {
fn decode_timestamps(&self, raw: &[u8]) -> Result<Vec<ColumnValue>> {
let mut cur = Cursor::new(raw);
- let first = cur.read_u64::<LittleEndian>()
+ let first = cur
+ .read_u64::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))? as i64;
- let count = cur.read_u32::<LittleEndian>()
+ let count = cur
+ .read_u32::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))? as usize;
let mut result = Vec::with_capacity(count);
@@ -387,7 +417,8 @@ impl ColumnReader {
result.push(ColumnValue::Timestamp(first));
for _ in 1..count {
- let dod = cur.read_i64::<LittleEndian>()
+ let dod = cur
+ .read_i64::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))?;
let delta = prev_delta + dod;
let v = prev + delta;
@@ -400,12 +431,14 @@ impl ColumnReader {
fn decode_i64s(&self, raw: &[u8]) -> Result<Vec<ColumnValue>> {
let mut cur = Cursor::new(raw);
- let count = cur.read_u32::<LittleEndian>()
+ let count = cur
+ .read_u32::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))? as usize;
let mut result = Vec::with_capacity(count);
let mut prev = 0i64;
for _ in 0..count {
- let delta = cur.read_i64::<LittleEndian>()
+ let delta = cur
+ .read_i64::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))?;
let v = prev + delta;
result.push(ColumnValue::I64(v));
@@ -416,11 +449,13 @@ impl ColumnReader {
fn decode_f64s(&self, raw: &[u8]) -> Result<Vec<ColumnValue>> {
let mut cur = Cursor::new(raw);
- let count = cur.read_u32::<LittleEndian>()
+ let count = cur
+ .read_u32::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))? as usize;
let mut result = Vec::with_capacity(count);
for _ in 0..count {
- let v = cur.read_f64::<LittleEndian>()
+ let v = cur
+ .read_f64::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))?;
result.push(ColumnValue::F64(v));
}
@@ -432,30 +467,36 @@ impl ColumnReader {
let mut cur = Cursor::new(&raw[1..]);
if use_dict {
- let dict_len = cur.read_u32::<LittleEndian>()
- .map_err(|e| XerjError::internal(e.to_string()))? as usize;
+ let dict_len =
+ cur.read_u32::<LittleEndian>()
+ .map_err(|e| XerjError::internal(e.to_string()))? as usize;
let mut dict_bytes = vec![0u8; dict_len];
cur.read_exact(&mut dict_bytes)
.map_err(|e| XerjError::internal(e.to_string()))?;
let decoder = DictionaryDecoder::deserialize(&dict_bytes)?;
- let count = cur.read_u32::<LittleEndian>()
+ let count = cur
+ .read_u32::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))? as usize;
let mut result = Vec::with_capacity(count);
for _ in 0..count {
- let id = cur.read_u16::<LittleEndian>()
+ let id = cur
+ .read_u16::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))?;
let s = decoder.decode(id).unwrap_or("").to_owned();
result.push(ColumnValue::String(s));
}
Ok(result)
} else {
- let count = cur.read_u32::<LittleEndian>()
+ let count = cur
+ .read_u32::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))? as usize;
let mut result = Vec::with_capacity(count);
for _ in 0..count {
- let slen = cur.read_u32::<LittleEndian>()
- .map_err(|e| XerjError::internal(e.to_string()))? as usize;
+ let slen = cur
+ .read_u32::<LittleEndian>()
+ .map_err(|e| XerjError::internal(e.to_string()))?
+ as usize;
let mut sbytes = vec![0u8; slen];
cur.read_exact(&mut sbytes)
.map_err(|e| XerjError::internal(e.to_string()))?;
@@ -469,9 +510,10 @@ impl ColumnReader {
fn decode_bools(&self, raw: &[u8]) -> Result<Vec<ColumnValue>> {
let mut cur = Cursor::new(raw);
- let count = cur.read_u32::<LittleEndian>()
+ let count = cur
+ .read_u32::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))? as usize;
- let byte_count = (count + 7) / 8;
+ let byte_count = count.div_ceil(8);
let mut packed = vec![0u8; byte_count];
cur.read_exact(&mut packed)
.map_err(|e| XerjError::internal(e.to_string()))?;
@@ -486,11 +528,13 @@ impl ColumnReader {
fn decode_binary(&self, raw: &[u8]) -> Result<Vec<ColumnValue>> {
let mut cur = Cursor::new(raw);
- let count = cur.read_u32::<LittleEndian>()
+ let count = cur
+ .read_u32::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))? as usize;
let mut result = Vec::with_capacity(count);
for _ in 0..count {
- let blen = cur.read_u32::<LittleEndian>()
+ let blen = cur
+ .read_u32::<LittleEndian>()
.map_err(|e| XerjError::internal(e.to_string()))? as usize;
let mut bytes = vec![0u8; blen];
cur.read_exact(&mut bytes)
@@ -511,7 +555,9 @@ mod tests {
fn roundtrip(column: Column) -> Vec<ColumnValue> {
let compression = CompressionLevel::Fast;
- let encoded = ColumnWriter::new(column.clone(), compression).encode().unwrap();
+ let encoded = ColumnWriter::new(column.clone(), compression)
+ .encode()
+ .unwrap();
let reader = ColumnReader::new(&column.name, column.column_type, encoded, 65536);
reader.decode(compression).unwrap()
}
diff --git a/engine/crates/xerj-logs/src/ingest.rs b/engine/crates/xerj-logs/src/ingest.rs
index ba5587757..8aa78cfc8 100644
--- a/engine/crates/xerj-logs/src/ingest.rs
+++ b/engine/crates/xerj-logs/src/ingest.rs
@@ -11,8 +11,6 @@ use std::sync::{Arc, Mutex};
use tracing::{debug, warn};
use xerj_common::XerjError;
-use crate::columnar::{Column, ColumnType, ColumnValue};
-
/// Result alias.
pub type Result<T> = std::result::Result<T, XerjError>;
@@ -37,16 +35,14 @@ pub struct LogRecord {
impl LogRecord {
/// Parse from a JSON byte slice.
pub fn from_json(data: &[u8]) -> Result<Self> {
- serde_json::from_slice(data).map_err(|e| {
- XerjError::serialization(format!("log record parse: {e}"))
- })
+ serde_json::from_slice(data)
+ .map_err(|e| XerjError::serialization(format!("log record parse: {e}")))
}
/// Parse from a JSON string.
pub fn from_json_str(s: &str) -> Result<Self> {
- serde_json::from_str(s).map_err(|e| {
- XerjError::serialization(format!("log record parse: {e}"))
- })
+ serde_json::from_str(s)
+ .map_err(|e| XerjError::serialization(format!("log record parse: {e}")))
}
/// Create a simple record with the current time.
@@ -163,6 +159,8 @@ fn looks_like_variable(token: &str) -> bool {
/// A mutable in-memory partition for one time bucket.
struct Memtable {
+ // Retained for provenance/debugging; not read on the hot path.
+ #[allow(dead_code)]
bucket: HourBucket,
/// Core columns always present.
timestamps: Vec<i64>,
@@ -268,7 +266,9 @@ impl LogIngester {
pub fn ingest(&self, record: LogRecord) -> Result<()> {
let bucket = record.hour_bucket();
let mut tables = self.memtables.lock().unwrap();
- let table = tables.entry(bucket).or_insert_with(|| Memtable::new(bucket));
+ let table = tables
+ .entry(bucket)
+ .or_insert_with(|| Memtable::new(bucket));
table.push(&record);
*self.total_ingested.lock().unwrap() += 1;
@@ -415,7 +415,8 @@ mod tests {
#[test]
fn ndjson_skips_bad_lines() {
let ingester = LogIngester::new();
- let data = "not-json\n{\"timestamp\":1700000000000000000,\"level\":\"INFO\",\"message\":\"ok\"}\n";
+ let data =
+ "not-json\n{\"timestamp\":1700000000000000000,\"level\":\"INFO\",\"message\":\"ok\"}\n";
let count = ingester.ingest_ndjson(data).unwrap();
assert_eq!(count, 1);
}
diff --git a/engine/crates/xerj-logs/src/lib.rs b/engine/crates/xerj-logs/src/lib.rs
index 460ab3cc5..6cc9ed7f6 100644
--- a/engine/crates/xerj-logs/src/lib.rs
+++ b/engine/crates/xerj-logs/src/lib.rs
@@ -16,6 +16,6 @@ pub mod retention;
pub use columnar::{Column, ColumnReader, ColumnType, ColumnWriter};
pub use ingest::{LogIngester, LogRecord};
pub use query::{Aggregation, LogQuery, LogQueryExecutor, QueryResult};
-pub use retention::{RetentionPolicy};
+pub use retention::RetentionPolicy;
pub use xerj_common::Result;
diff --git a/engine/crates/xerj-logs/src/query.rs b/engine/crates/xerj-logs/src/query.rs
index fc3db30f8..db3eecf14 100644
--- a/engine/crates/xerj-logs/src/query.rs
+++ b/engine/crates/xerj-logs/src/query.rs
@@ -39,29 +39,21 @@ pub enum Filter {
impl Filter {
fn matches(&self, record: &LogRecord) -> bool {
match self {
- Filter::Level { value } => {
- record.level.as_deref().map_or(false, |l| {
- l.eq_ignore_ascii_case(value)
- })
- }
- Filter::FieldEquals { field, value } => {
- record.fields.get(field).map_or(false, |v| v == value)
- }
- Filter::FieldContains { field, substring } => {
- record.fields.get(field).and_then(|v| v.as_str()).map_or(
- false,
- |s| s.contains(substring.as_str()),
- )
- }
- Filter::FieldRange { field, min, max } => {
- record
- .fields
- .get(field)
- .and_then(|v| v.as_f64())
- .map_or(false, |n| {
- min.map_or(true, |m| n >= m) && max.map_or(true, |m| n <= m)
- })
- }
+ Filter::Level { value } => record
+ .level
+ .as_deref()
+ .is_some_and(|l| l.eq_ignore_ascii_case(value)),
+ Filter::FieldEquals { field, value } => record.fields.get(field) == Some(value),
+ Filter::FieldContains { field, substring } => record
+ .fields
+ .get(field)
+ .and_then(|v| v.as_str())
+ .is_some_and(|s| s.contains(substring.as_str())),
+ Filter::FieldRange { field, min, max } => record
+ .fields
+ .get(field)
+ .and_then(|v| v.as_f64())
+ .is_some_and(|n| min.is_none_or(|m| n >= m) && max.is_none_or(|m| n <= m)),
}
}
}
@@ -119,7 +111,9 @@ impl LogQuery {
}
pub fn with_level(mut self, level: impl Into<String>) -> Self {
- self.filters.push(Filter::Level { value: level.into() });
+ self.filters.push(Filter::Level {
+ value: level.into(),
+ });
self
}
@@ -274,7 +268,10 @@ impl<'a> LogQueryExecutor<'a> {
} else {
Some(vals.iter().sum::<f64>() / vals.len() as f64)
};
- Ok(AggregationResult { value: avg, buckets: vec![] })
+ Ok(AggregationResult {
+ value: avg,
+ buckets: vec![],
+ })
}
Aggregation::Min { field } => {
@@ -315,7 +312,10 @@ impl<'a> LogQueryExecutor<'a> {
})
.collect();
buckets.sort_by_key(|b| b.key.parse::<i64>().unwrap_or(0));
- Ok(AggregationResult { value: None, buckets })
+ Ok(AggregationResult {
+ value: None,
+ buckets,
+ })
}
Aggregation::Terms { field, size } => {
@@ -345,7 +345,10 @@ impl<'a> LogQueryExecutor<'a> {
.collect();
buckets.sort_by(|a, b| b.doc_count.cmp(&a.doc_count));
buckets.truncate(*size);
- Ok(AggregationResult { value: None, buckets })
+ Ok(AggregationResult {
+ value: None,
+ buckets,
+ })
}
}
}
@@ -367,18 +370,38 @@ mod tests {
fn setup_ingester() -> LogIngester {
let ingester = LogIngester::new();
let records = vec![
- LogRecord { timestamp: ts(0), level: Some("INFO".into()), message: Some("start".into()), fields: HashMap::new() },
- LogRecord { timestamp: ts(10), level: Some("WARN".into()), message: Some("slow".into()), fields: {
- let mut m = HashMap::new();
- m.insert("latency_ms".into(), Value::from(150.0));
- m
- }},
- LogRecord { timestamp: ts(20), level: Some("ERROR".into()), message: Some("fail".into()), fields: {
- let mut m = HashMap::new();
- m.insert("latency_ms".into(), Value::from(5000.0));
- m
- }},
- LogRecord { timestamp: ts(30), level: Some("INFO".into()), message: Some("done".into()), fields: HashMap::new() },
+ LogRecord {
+ timestamp: ts(0),
+ level: Some("INFO".into()),
+ message: Some("start".into()),
+ fields: HashMap::new(),
+ },
+ LogRecord {
+ timestamp: ts(10),
+ level: Some("WARN".into()),
+ message: Some("slow".into()),
+ fields: {
+ let mut m = HashMap::new();
+ m.insert("latency_ms".into(), Value::from(150.0));
+ m
+ },
+ },
+ LogRecord {
+ timestamp: ts(20),
+ level: Some("ERROR".into()),
+ message: Some("fail".into()),
+ fields: {
+ let mut m = HashMap::new();
+ m.insert("latency_ms".into(), Value::from(5000.0));
+ m
+ },
+ },
+ LogRecord {
+ timestamp: ts(30),
+ level: Some("INFO".into()),
+ message: Some("done".into()),
+ fields: HashMap::new(),
+ },
];
for r in records {
ingester.ingest(r).unwrap();
@@ -408,8 +431,9 @@ mod tests {
fn sum_aggregation() {
let ingester = setup_ingester();
let exec = LogQueryExecutor::new(&ingester);
- let q = LogQuery::new(ts(-1), ts(60))
- .with_aggregation(Aggregation::Sum { field: "latency_ms".into() });
+ let q = LogQuery::new(ts(-1), ts(60)).with_aggregation(Aggregation::Sum {
+ field: "latency_ms".into(),
+ });
let result = exec.execute(&q).unwrap();
assert_eq!(result.aggregations["result"].value, Some(5150.0));
}
@@ -418,8 +442,9 @@ mod tests {
fn max_aggregation() {
let ingester = setup_ingester();
let exec = LogQueryExecutor::new(&ingester);
- let q = LogQuery::new(ts(-1), ts(60))
- .with_aggregation(Aggregation::Max { field: "latency_ms".into() });
+ let q = LogQuery::new(ts(-1), ts(60)).with_aggregation(Aggregation::Max {
+ field: "latency_ms".into(),
+ });
let result = exec.execute(&q).unwrap();
assert_eq!(result.aggregations["result"].value, Some(5000.0));
}
@@ -428,8 +453,10 @@ mod tests {
fn terms_aggregation() {
let ingester = setup_ingester();
let exec = LogQueryExecutor::new(&ingester);
- let q = LogQuery::new(ts(-1), ts(60))
- .with_aggregation(Aggregation::Terms { field: "level".into(), size: 5 });
+ let q = LogQuery::new(ts(-1), ts(60)).with_aggregation(Aggregation::Terms {
+ field: "level".into(),
+ size: 5,
+ });
let result = exec.execute(&q).unwrap();
let buckets = &result.aggregations["result"].buckets;
// INFO appears twice, should be first
diff --git a/engine/crates/xerj-logs/src/retention.rs b/engine/crates/xerj-logs/src/retention.rs
index a2cbbc952..a2dbadf0e 100644
--- a/engine/crates/xerj-logs/src/retention.rs
+++ b/engine/crates/xerj-logs/src/retention.rs
@@ -184,10 +184,10 @@ mod tests {
fn expired_buckets_returns_correct_subset() {
let policy = RetentionPolicy::retain_days(30);
let buckets = vec![
- old_bucket(5), // recent
- old_bucket(40), // expired
- old_bucket(60), // expired
- old_bucket(10), // recent
+ old_bucket(5), // recent
+ old_bucket(40), // expired
+ old_bucket(60), // expired
+ old_bucket(10), // recent
];
let expired = policy.expired_buckets(&buckets);
assert_eq!(expired.len(), 2);
diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rs
index f20f314f4..70605f68d 100644
--- a/engine/crates/xerj-query/src/ast.rs
+++ b/engine/crates/xerj-query/src/ast.rs
@@ -76,19 +76,15 @@ impl Default for FusionStrategy {
/// Controls how many edit operations are allowed in a fuzzy query.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
+#[derive(Default)]
pub enum Fuzziness {
/// AUTO fuzziness: 0 edits for 1-2 chars, 1 edit for 3-5 chars, 2 edits for 6+ chars.
+ #[default]
Auto,
/// Fixed maximum number of edit operations.
Fixed(u32),
}
-impl Default for Fuzziness {
- fn default() -> Self {
- Self::Auto
- }
-}
-
/// `minimum_should_match` — either an absolute count or a percentage.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
@@ -232,7 +228,6 @@ pub struct DistanceFeature {
#[serde(rename_all = "snake_case", tag = "type")]
pub enum QueryNode {
// ── Structural ────────────────────────────────────────────────────────────
-
/// Matches every document in the index.
MatchAll,
@@ -259,19 +254,12 @@ pub enum QueryNode {
},
/// Wraps a query, replacing its score with a fixed constant.
- Constant {
- score: f32,
- query: Box<QueryNode>,
- },
+ Constant { score: f32, query: Box<QueryNode> },
/// Multiplies the wrapped query's score by `boost`.
- Boosted {
- boost: f32,
- query: Box<QueryNode>,
- },
+ Boosted { boost: f32, query: Box<QueryNode> },
// ── Term-level ────────────────────────────────────────────────────────────
-
/// Exact-value match on a single field.
Term {
field: String,
@@ -327,7 +315,6 @@ pub enum QueryNode {
Ids { values: Vec<String> },
// ── Full-text ─────────────────────────────────────────────────────────────
-
/// Analysed text query against a single field.
Match {
field: String,
@@ -380,7 +367,6 @@ pub enum QueryNode {
},
// ── AI-native extensions ──────────────────────────────────────────────────
-
/// Dense vector k-NN search (exact or approximate).
Knn {
field: String,
@@ -449,7 +435,6 @@ pub enum QueryNode {
},
// ── Extended query types ───────────────────────────────────────────────────
-
/// Fuzzy match — allows typo tolerance via Levenshtein edit distance.
Fuzzy {
field: String,
@@ -459,10 +444,7 @@ pub enum QueryNode {
},
/// Regular expression match against a field value.
- Regexp {
- field: String,
- pattern: String,
- },
+ Regexp { field: String, pattern: String },
/// Position-aware text match (ES `intervals` query). The `rule` is the
/// raw rule JSON (one of match/all_of/any_of/prefix/wildcard/fuzzy,
@@ -512,7 +494,6 @@ pub enum QueryNode {
},
// ── Nested / join / specialised queries ───────────────────────────────────
-
/// Nested query — runs an inner query against each element of a nested array.
///
/// If any element matches the inner query, the document matches.
@@ -559,12 +540,8 @@ pub enum QueryNode {
},
// ── Span queries ──────────────────────────────────────────────────────────
-
/// Span term — exact value match within span context.
- SpanTerm {
- field: String,
- value: String,
- },
+ SpanTerm { field: String, value: String },
/// Span near — span terms that appear within `slop` positions of each other.
SpanNear {
@@ -574,9 +551,7 @@ pub enum QueryNode {
},
/// Span or — matches if any of the span clauses match.
- SpanOr {
- clauses: Vec<QueryNode>,
- },
+ SpanOr { clauses: Vec<QueryNode> },
/// Span not — matches the include span but not the exclude span.
SpanNot {
@@ -591,7 +566,6 @@ pub enum QueryNode {
},
// ── Join queries ──────────────────────────────────────────────────────────
-
/// Has-child — matches parent documents that have child documents matching the query.
HasChild {
child_type: String,
@@ -607,7 +581,6 @@ pub enum QueryNode {
},
// ── Geo shape queries ─────────────────────────────────────────────────────
-
/// Geo polygon — matches documents whose geo_point falls within the polygon.
GeoPolygon {
field: String,
@@ -615,10 +588,7 @@ pub enum QueryNode {
},
/// Geo shape — matches documents whose geo field matches the given shape.
- GeoShape {
- field: String,
- shape: GeoShapeType,
- },
+ GeoShape { field: String, shape: GeoShapeType },
}
/// Shape type for geo_shape queries.
@@ -628,14 +598,14 @@ pub enum GeoShapeType {
/// A single point (lat, lon).
Point { lat: f64, lon: f64 },
/// An envelope (bounding box) given by top-left and bottom-right corners (lat, lon).
- Envelope { top_left: (f64, f64), bottom_right: (f64, f64) },
+ Envelope {
+ top_left: (f64, f64),
+ bottom_right: (f64, f64),
+ },
/// A polygon defined by a list of (lat, lon) points.
Polygon { points: Vec<(f64, f64)> },
/// A circle defined by a center point and a radius in kilometres.
- Circle {
- center: (f64, f64),
- radius_km: f64,
- },
+ Circle { center: (f64, f64), radius_km: f64 },
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -740,9 +710,13 @@ impl QueryNode {
/// (no scoring contribution from sub-queries).
pub fn is_filter_only(&self) -> bool {
match self {
- QueryNode::Bool { must, should, filter, must_not, .. } => {
- must.is_empty() && should.is_empty() && !filter.is_empty() || !must_not.is_empty()
- }
+ QueryNode::Bool {
+ must,
+ should,
+ filter,
+ must_not,
+ ..
+ } => must.is_empty() && should.is_empty() && !filter.is_empty() || !must_not.is_empty(),
QueryNode::Exists { .. }
| QueryNode::Term { .. }
| QueryNode::Terms { .. }
@@ -755,7 +729,13 @@ impl QueryNode {
/// Rough structural depth — used by the planner for cost estimation.
pub fn depth(&self) -> usize {
match self {
- QueryNode::Bool { must, should, must_not, filter, .. } => {
+ QueryNode::Bool {
+ must,
+ should,
+ must_not,
+ filter,
+ ..
+ } => {
let max_child = must
.iter()
.chain(should.iter())
@@ -766,10 +746,10 @@ impl QueryNode {
.unwrap_or(0);
1 + max_child
}
- QueryNode::Constant { query, .. }
- | QueryNode::Boosted { query, .. } => 1 + query.depth(),
- QueryNode::Knn { filter, .. }
- | QueryNode::SemanticSearch { filter, .. } => {
+ QueryNode::Constant { query, .. } | QueryNode::Boosted { query, .. } => {
+ 1 + query.depth()
+ }
+ QueryNode::Knn { filter, .. } | QueryNode::SemanticSearch { filter, .. } => {
1 + filter.as_ref().map(|f| f.depth()).unwrap_or(0)
}
QueryNode::Hybrid { queries, .. } => {
@@ -777,9 +757,9 @@ impl QueryNode {
1 + max_child
}
QueryNode::FunctionScore { query, .. } => 1 + query.depth(),
- QueryNode::Boosting { positive, negative, .. } => {
- 1 + positive.depth().max(negative.depth())
- }
+ QueryNode::Boosting {
+ positive, negative, ..
+ } => 1 + positive.depth().max(negative.depth()),
QueryNode::DisMax { queries, .. } => {
let max_child = queries.iter().map(|q| q.depth()).max().unwrap_or(0);
1 + max_child
@@ -810,9 +790,11 @@ impl QueryNode {
/// - `Limit(N)` — stop counting after N docs; return `Gte` relation.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
+#[derive(Default)]
pub enum TrackTotalHits {
/// Count all matching documents (default).
#[serde(rename = "true")]
+ #[default]
True,
/// Do not count beyond `size` hits.
#[serde(rename = "false")]
@@ -821,12 +803,6 @@ pub enum TrackTotalHits {
Limit(u64),
}
-impl Default for TrackTotalHits {
- fn default() -> Self {
- TrackTotalHits::True
- }
-}
-
/// The fully-parsed search request.
///
/// This is what the REST layer hands to the query engine after deserialising
diff --git a/engine/crates/xerj-query/src/executor.rs b/engine/crates/xerj-query/src/executor.rs
index 843ab606f..b0b4452c0 100644
--- a/engine/crates/xerj-query/src/executor.rs
+++ b/engine/crates/xerj-query/src/executor.rs
@@ -84,15 +84,19 @@ pub struct Explanation {
impl Explanation {
pub fn leaf(value: f32, description: impl Into<String>) -> Self {
- Self { value, description: description.into(), details: vec![] }
+ Self {
+ value,
+ description: description.into(),
+ details: vec![],
+ }
}
- pub fn compound(
- value: f32,
- description: impl Into<String>,
- details: Vec<Explanation>,
- ) -> Self {
- Self { value, description: description.into(), details }
+ pub fn compound(value: f32, description: impl Into<String>, details: Vec<Explanation>) -> Self {
+ Self {
+ value,
+ description: description.into(),
+ details,
+ }
}
}
@@ -125,7 +129,10 @@ impl SearchResult {
pub fn empty(took_ms: u64) -> Self {
Self {
hits: vec![],
- total: TotalHits { value: 0, relation: TotalHitsRelation::Eq },
+ total: TotalHits {
+ value: 0,
+ relation: TotalHitsRelation::Eq,
+ },
took_ms,
aggs: None,
timed_out: false,
@@ -218,8 +225,7 @@ pub fn merge_hits(
});
} else {
all.sort_unstable_by(|a, b| {
- compare_sort_keys(&a.sort, &b.sort, sort_fields)
- .then_with(|| a.id.cmp(&b.id))
+ compare_sort_keys(&a.sort, &b.sort, sort_fields).then_with(|| a.id.cmp(&b.id))
});
}
@@ -238,7 +244,10 @@ pub fn merge_totals(segment_totals: &[(u64, TotalHitsRelation)]) -> TotalHits {
relation = TotalHitsRelation::Gte;
}
}
- TotalHits { value: total, relation }
+ TotalHits {
+ value: total,
+ relation,
+ }
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -286,7 +295,11 @@ impl TopKHeap {
/// Push a hit, evicting the lowest scorer if over capacity.
pub fn push(&mut self, hit: Hit) {
- let sh = ScoredHit { score: hit.score, id: hit.id.clone(), hit };
+ let sh = ScoredHit {
+ score: hit.score,
+ id: hit.id.clone(),
+ hit,
+ };
self.inner.push(Reverse(sh));
if self.inner.len() > self.capacity {
self.inner.pop(); // evict the lowest scorer
@@ -295,8 +308,7 @@ impl TopKHeap {
/// Drain the heap in score-descending order.
pub fn into_sorted_hits(self) -> Vec<Hit> {
- let mut hits: Vec<Hit> =
- self.inner.into_iter().map(|Reverse(sh)| sh.hit).collect();
+ let mut hits: Vec<Hit> = self.inner.into_iter().map(|Reverse(sh)| sh.hit).collect();
hits.sort_unstable_by(|a, b| {
b.score
.partial_cmp(&a.score)
@@ -310,6 +322,11 @@ impl TopKHeap {
self.inner.len()
}
+ /// Whether the heap currently holds no hits.
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
/// Minimum score currently in the heap (used for early termination).
pub fn min_score(&self) -> Option<f32> {
self.inner.peek().map(|Reverse(sh)| sh.score)
@@ -383,12 +400,7 @@ mod tests {
#[test]
fn test_merge_hits_from_offset() {
- let seg = vec![
- hit("a", 0.9),
- hit("b", 0.8),
- hit("c", 0.7),
- hit("d", 0.6),
- ];
+ let seg = vec![hit("a", 0.9), hit("b", 0.8), hit("c", 0.7), hit("d", 0.6)];
let merged = merge_hits(vec![seg], 2, 2, &[]);
assert_eq!(merged.len(), 2);
assert_eq!(merged[0].id, "c");
diff --git a/engine/crates/xerj-query/src/lib.rs b/engine/crates/xerj-query/src/lib.rs
index a03a0422c..daf98790f 100644
--- a/engine/crates/xerj-query/src/lib.rs
+++ b/engine/crates/xerj-query/src/lib.rs
@@ -80,8 +80,8 @@ pub use planner::{plan_query, ExecutionPlan};
// Executor
pub use executor::{
- merge_hits, merge_totals, Explanation, Hit, SearchResult, SegmentExecutor, TotalHits,
- TotalHitsRelation, TopKHeap,
+ merge_hits, merge_totals, Explanation, Hit, SearchResult, SegmentExecutor, TopKHeap, TotalHits,
+ TotalHitsRelation,
};
// Sort
diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rs
index 87d181c24..9a35616cc 100644
--- a/engine/crates/xerj-query/src/parser.rs
+++ b/engine/crates/xerj-query/src/parser.rs
@@ -19,7 +19,7 @@ use std::cell::Cell;
use tracing::trace;
use crate::ast::{
- BoostMode, BoolOperator, FieldValueFactor, Fuzziness, FusionStrategy, GeoShapeType,
+ BoolOperator, BoostMode, FieldValueFactor, FusionStrategy, Fuzziness, GeoShapeType,
MinShouldMatch, Modifier, MultiMatchType, QueryNode, RandomScore, ScoreFunction, ScoreMode,
SearchRequest, SourceFilter, TrackTotalHits, WeightedQuery,
};
@@ -86,7 +86,10 @@ fn invalid<T>(msg: impl Into<String>) -> Result<T> {
#[inline(always)]
fn maybe_named(node: QueryNode, name: Option<String>) -> QueryNode {
match name {
- Some(n) => QueryNode::Named { name: n, query: Box::new(node) },
+ Some(n) => QueryNode::Named {
+ name: n,
+ query: Box::new(node),
+ },
None => node,
}
}
@@ -114,11 +117,11 @@ fn unknown_type<T>(name: impl Into<String>) -> Result<T> {
/// }
/// ```
pub fn parse_request(body: &Value) -> Result<SearchRequest> {
- let obj = body
- .as_object()
- .ok_or_else(|| QueryError::Parse(ParseError::Invalid(
+ let obj = body.as_object().ok_or_else(|| {
+ QueryError::Parse(ParseError::Invalid(
"request body must be a JSON object".into(),
- )))?;
+ ))
+ })?;
let query = match obj.get("query") {
Some(q) => parse_query(q)?,
@@ -126,16 +129,20 @@ pub fn parse_request(body: &Value) -> Result<SearchRequest> {
};
let from = match obj.get("from") {
- Some(v) => v.as_u64().ok_or_else(|| QueryError::Parse(ParseError::Invalid(
- "`from` must be a non-negative integer".into(),
- )))? as usize,
+ Some(v) => v.as_u64().ok_or_else(|| {
+ QueryError::Parse(ParseError::Invalid(
+ "`from` must be a non-negative integer".into(),
+ ))
+ })? as usize,
None => 0,
};
let size = match obj.get("size") {
- Some(v) => v.as_u64().ok_or_else(|| QueryError::Parse(ParseError::Invalid(
- "`size` must be a non-negative integer".into(),
- )))? as usize,
+ Some(v) => v.as_u64().ok_or_else(|| {
+ QueryError::Parse(ParseError::Invalid(
+ "`size` must be a non-negative integer".into(),
+ ))
+ })? as usize,
None => 10,
};
@@ -161,10 +168,7 @@ pub fn parse_request(body: &Value) -> Result<SearchRequest> {
None => None,
};
- let aggs = obj
- .get("aggs")
- .or_else(|| obj.get("aggregations"))
- .cloned();
+ let aggs = obj.get("aggs").or_else(|| obj.get("aggregations")).cloned();
let explain = obj
.get("explain")
@@ -176,12 +180,12 @@ pub fn parse_request(body: &Value) -> Result<SearchRequest> {
None => SourceFilter::default(),
};
- let timeout_ms = obj.get("timeout").and_then(|v| parse_timeout(v));
+ let timeout_ms = obj.get("timeout").and_then(parse_timeout);
// Parse optional highlight configuration.
- let highlight = obj.get("highlight").and_then(|v| {
- serde_json::from_value(v.clone()).ok()
- });
+ let highlight = obj
+ .get("highlight")
+ .and_then(|v| serde_json::from_value(v.clone()).ok());
// Parse track_total_hits: true | false | integer
let track_total_hits = match obj.get("track_total_hits") {
@@ -213,10 +217,7 @@ pub fn parse_request(body: &Value) -> Result<SearchRequest> {
};
// profile — include timing breakdown
- let profile = obj
- .get("profile")
- .and_then(Value::as_bool)
- .unwrap_or(false);
+ let profile = obj.get("profile").and_then(Value::as_bool).unwrap_or(false);
Ok(SearchRequest {
query,
@@ -248,11 +249,9 @@ pub fn parse_query(json: &Value) -> Result<QueryNode> {
// an error return here does not leak the increment to a sibling call.
let _depth = DepthGuard::enter()?;
- let obj = json
- .as_object()
- .ok_or_else(|| QueryError::Parse(ParseError::Invalid(
- "query must be a JSON object".into(),
- )))?;
+ let obj = json.as_object().ok_or_else(|| {
+ QueryError::Parse(ParseError::Invalid("query must be a JSON object".into()))
+ })?;
if obj.len() != 1 {
if obj.is_empty() {
@@ -373,17 +372,33 @@ fn parse_match(params: &Value) -> Result<QueryNode> {
.ok_or_else(|| qerr("`match` field value must be a scalar or object"))?;
// Inside the object form, `query` can also be a number / bool.
- let query = vobj.get("query")
+ let query = vobj
+ .get("query")
.and_then(scalar_to_string)
.ok_or_else(|| qerr("`match.query` must be a non-empty scalar"))?;
let operator = parse_bool_operator(vobj.get("operator"))?;
- let analyzer = vobj.get("analyzer").and_then(|v| v.as_str()).map(str::to_string);
+ let analyzer = vobj
+ .get("analyzer")
+ .and_then(|v| v.as_str())
+ .map(str::to_string);
let boost = vobj.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
- let minimum_should_match =
- vobj.get("minimum_should_match").map(parse_min_should_match).transpose()?;
- let name = vobj.get("_name").and_then(|v| v.as_str()).map(str::to_string);
+ let minimum_should_match = vobj
+ .get("minimum_should_match")
+ .map(parse_min_should_match)
+ .transpose()?;
+ let name = vobj
+ .get("_name")
+ .and_then(|v| v.as_str())
+ .map(str::to_string);
- let node = QueryNode::Match { field, query, operator, analyzer, boost, minimum_should_match };
+ let node = QueryNode::Match {
+ field,
+ query,
+ operator,
+ analyzer,
+ boost,
+ minimum_should_match,
+ };
Ok(maybe_named(node, name))
}
@@ -425,11 +440,23 @@ fn parse_match_phrase(params: &Value) -> Result<QueryNode> {
let query = string_field(vobj, "query")?;
let slop = vobj.get("slop").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
- let analyzer = vobj.get("analyzer").and_then(|v| v.as_str()).map(str::to_string);
+ let analyzer = vobj
+ .get("analyzer")
+ .and_then(|v| v.as_str())
+ .map(str::to_string);
let boost = vobj.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
- let name = vobj.get("_name").and_then(|v| v.as_str()).map(str::to_string);
+ let name = vobj
+ .get("_name")
+ .and_then(|v| v.as_str())
+ .map(str::to_string);
- let node = QueryNode::MatchPhrase { field, query, slop, analyzer, boost };
+ let node = QueryNode::MatchPhrase {
+ field,
+ query,
+ slop,
+ analyzer,
+ boost,
+ };
Ok(maybe_named(node, name))
}
@@ -473,20 +500,29 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
return invalid("`multi_match.fields` must not be empty");
}
- let type_str = obj.get("type").and_then(|v| v.as_str()).unwrap_or("best_fields");
+ let type_str = obj
+ .get("type")
+ .and_then(|v| v.as_str())
+ .unwrap_or("best_fields");
// bool_prefix: rewrite to a Bool::should over per-field match_bool_prefix
// clauses. Tokens except the last become Match queries, the last becomes
// a Prefix query — matching ES `match_bool_prefix` semantics across all
// supplied fields.
if type_str == "bool_prefix" {
- let analyzer_opt = obj.get("analyzer").and_then(|v| v.as_str()).map(str::to_string);
- let analyzer_lowercases = match analyzer_opt.as_deref() {
- Some("whitespace") | Some("keyword") => false,
- _ => true,
- };
+ let analyzer_opt = obj
+ .get("analyzer")
+ .and_then(|v| v.as_str())
+ .map(str::to_string);
+ let analyzer_lowercases = !matches!(
+ analyzer_opt.as_deref(),
+ Some("whitespace") | Some("keyword")
+ );
let operator_and = matches!(
- obj.get("operator").and_then(Value::as_str).map(|s| s.to_ascii_lowercase()).as_deref(),
+ obj.get("operator")
+ .and_then(Value::as_str)
+ .map(|s| s.to_ascii_lowercase())
+ .as_deref(),
Some("and")
);
let mm_ms = obj.get("minimum_should_match").and_then(|v| match v {
@@ -502,7 +538,11 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
});
let fuzziness_opt = obj.get("fuzziness").map(|v| match v {
Value::String(s) if s.eq_ignore_ascii_case("auto") => crate::ast::Fuzziness::Auto,
- Value::String(s) => s.parse::<u32>().ok().map(crate::ast::Fuzziness::Fixed).unwrap_or(crate::ast::Fuzziness::Auto),
+ Value::String(s) => s
+ .parse::<u32>()
+ .ok()
+ .map(crate::ast::Fuzziness::Fixed)
+ .unwrap_or(crate::ast::Fuzziness::Auto),
Value::Number(n) => crate::ast::Fuzziness::Fixed(n.as_u64().unwrap_or(0) as u32),
_ => crate::ast::Fuzziness::Auto,
});
@@ -511,9 +551,16 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
} else {
query.split_whitespace().map(str::to_string).collect()
};
- let tokens: Vec<String> = raw_tokens.into_iter().map(|t| {
- if analyzer_lowercases { t.to_lowercase() } else { t }
- }).collect();
+ let tokens: Vec<String> = raw_tokens
+ .into_iter()
+ .map(|t| {
+ if analyzer_lowercases {
+ t.to_lowercase()
+ } else {
+ t
+ }
+ })
+ .collect();
if tokens.is_empty() {
return Ok(QueryNode::MatchAll);
}
@@ -522,7 +569,7 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
for raw_field in &fields {
// Strip any ^boost suffix (e.g. "title^2").
let (field, fb): (&str, Option<f32>) = match raw_field.rfind('^') {
- Some(idx) => (&raw_field[..idx], raw_field[idx+1..].parse::<f32>().ok()),
+ Some(idx) => (&raw_field[..idx], raw_field[idx + 1..].parse::<f32>().ok()),
None => (raw_field.as_str(), None),
};
if tokens.len() == 1 {
@@ -536,8 +583,12 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
}
let last = tokens.len() - 1;
let build_leaf = |tok: &str| -> QueryNode {
- if let Some(fz) = fuzziness_opt.clone() {
- QueryNode::Fuzzy { field: field.to_string(), value: tok.to_string(), fuzziness: fz }
+ if let Some(fz) = fuzziness_opt {
+ QueryNode::Fuzzy {
+ field: field.to_string(),
+ value: tok.to_string(),
+ fuzziness: fz,
+ }
} else {
QueryNode::Match {
field: field.to_string(),
@@ -549,7 +600,8 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
}
}
};
- let mut inner_clauses: Vec<QueryNode> = tokens[..last].iter().map(|t| build_leaf(t)).collect();
+ let mut inner_clauses: Vec<QueryNode> =
+ tokens[..last].iter().map(|t| build_leaf(t)).collect();
inner_clauses.push(QueryNode::Prefix {
field: field.to_string(),
value: tokens[last].clone(),
@@ -558,7 +610,7 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
let (im, is, imm) = if operator_and {
(inner_clauses, vec![], None)
} else {
- (vec![], inner_clauses, mm_ms.clone())
+ (vec![], inner_clauses, mm_ms)
};
let inner_bool = QueryNode::Bool {
must: im,
@@ -568,8 +620,13 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
minimum_should_match: imm,
};
let field_clause = if let Some(b) = fb {
- QueryNode::Boosted { boost: b, query: Box::new(inner_bool) }
- } else { inner_bool };
+ QueryNode::Boosted {
+ boost: b,
+ query: Box::new(inner_bool),
+ }
+ } else {
+ inner_bool
+ };
should.push(field_clause);
}
return Ok(QueryNode::Bool {
@@ -578,8 +635,12 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
filter: vec![],
must_not: vec![],
minimum_should_match: None,
- }).map(|n| match boost {
- Some(b) => QueryNode::Boosted { query: Box::new(n), boost: b },
+ })
+ .map(|n| match boost {
+ Some(b) => QueryNode::Boosted {
+ query: Box::new(n),
+ boost: b,
+ },
None => n,
});
}
@@ -594,10 +655,20 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
};
let operator = parse_bool_operator(obj.get("operator")).ok();
- let analyzer = obj.get("analyzer").and_then(|v| v.as_str()).map(str::to_string);
+ let analyzer = obj
+ .get("analyzer")
+ .and_then(|v| v.as_str())
+ .map(str::to_string);
let boost = obj.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
- Ok(QueryNode::MultiMatch { fields, query, match_type, operator, analyzer, boost })
+ Ok(QueryNode::MultiMatch {
+ fields,
+ query,
+ match_type,
+ operator,
+ analyzer,
+ boost,
+ })
}
fn parse_term(params: &Value) -> Result<QueryNode> {
@@ -613,8 +684,14 @@ fn parse_term(params: &Value) -> Result<QueryNode> {
if let Some(inner) = raw.as_object() {
if let Some(value) = inner.get("value") {
- let boost = inner.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
- let name = inner.get("_name").and_then(|v| v.as_str()).map(str::to_string);
+ let boost = inner
+ .get("boost")
+ .and_then(|v| v.as_f64())
+ .map(|b| b as f32);
+ let name = inner
+ .get("_name")
+ .and_then(|v| v.as_str())
+ .map(str::to_string);
let case_insensitive = inner
.get("case_insensitive")
.and_then(Value::as_bool)
@@ -630,15 +707,27 @@ fn parse_term(params: &Value) -> Result<QueryNode> {
Value::String(s) => s.clone(),
other => other.to_string().trim_matches('"').to_string(),
};
- let node = QueryNode::Wildcard { field, value: val_str, boost };
+ let node = QueryNode::Wildcard {
+ field,
+ value: val_str,
+ boost,
+ };
return Ok(maybe_named(node, name));
}
- let node = QueryNode::Term { field, value: value.clone(), boost };
+ let node = QueryNode::Term {
+ field,
+ value: value.clone(),
+ boost,
+ };
return Ok(maybe_named(node, name));
}
}
- Ok(QueryNode::Term { field, value: raw.clone(), boost: None })
+ Ok(QueryNode::Term {
+ field,
+ value: raw.clone(),
+ boost: None,
+ })
}
fn parse_terms(params: &Value) -> Result<QueryNode> {
@@ -646,10 +735,7 @@ fn parse_terms(params: &Value) -> Result<QueryNode> {
.as_object()
.ok_or_else(|| qerr("`terms` must be an object"))?;
- let field_entries: Vec<_> = obj
- .iter()
- .filter(|(k, _)| k.as_str() != "boost")
- .collect();
+ let field_entries: Vec<_> = obj.iter().filter(|(k, _)| k.as_str() != "boost").collect();
if field_entries.len() != 1 {
return invalid("`terms` query must have exactly one field");
@@ -675,7 +761,11 @@ fn parse_terms(params: &Value) -> Result<QueryNode> {
let boost = obj.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
- Ok(QueryNode::Terms { field, values, boost })
+ Ok(QueryNode::Terms {
+ field,
+ values,
+ boost,
+ })
}
/// Resolve an ES date-math expression like `now`, `now-24h`, `now+1d/d`
@@ -705,14 +795,14 @@ fn resolve_now_expr(expr: &str) -> Option<String> {
let n: f64 = num_part.parse().ok()?;
let secs = match unit_part.as_str() {
"ms" => n / 1000.0,
- "s" => n,
- "m" => n * 60.0,
- "h" => n * 3600.0,
- "d" => n * 86_400.0,
- "w" => n * 7.0 * 86_400.0,
- "M" => n * 30.0 * 86_400.0,
- "y" => n * 365.0 * 86_400.0,
- _ => return None,
+ "s" => n,
+ "m" => n * 60.0,
+ "h" => n * 3600.0,
+ "d" => n * 86_400.0,
+ "w" => n * 7.0 * 86_400.0,
+ "M" => n * 30.0 * 86_400.0,
+ "y" => n * 365.0 * 86_400.0,
+ _ => return None,
};
delta_secs = (sign as f64 * secs) as i64;
}
@@ -739,7 +829,10 @@ fn parse_range(params: &Value) -> Result<QueryNode> {
let mut gt = inner.get("gt").cloned();
let mut lte = inner.get("lte").cloned();
let mut lt = inner.get("lt").cloned();
- let boost = inner.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
+ let boost = inner
+ .get("boost")
+ .and_then(|v| v.as_f64())
+ .map(|b| b as f32);
// ES date math: `gte: "now-24h"` etc. resolve at query-time against
// wall clock. Without this, downstream `doc_matches_query` would
@@ -757,10 +850,18 @@ fn parse_range(params: &Value) -> Result<QueryNode> {
_ => v,
}
};
- if let Some(v) = gte.take() { gte = Some(resolve_now(v)); }
- if let Some(v) = gt.take() { gt = Some(resolve_now(v)); }
- if let Some(v) = lte.take() { lte = Some(resolve_now(v)); }
- if let Some(v) = lt.take() { lt = Some(resolve_now(v)); }
+ if let Some(v) = gte.take() {
+ gte = Some(resolve_now(v));
+ }
+ if let Some(v) = gt.take() {
+ gt = Some(resolve_now(v));
+ }
+ if let Some(v) = lte.take() {
+ lte = Some(resolve_now(v));
+ }
+ if let Some(v) = lt.take() {
+ lt = Some(resolve_now(v));
+ }
// `format: uuuu` (year-only): ES resolves the bound as year-first-day
// (YEAR-01-01T00:00:00) and applies range rounding at day granularity —
@@ -775,7 +876,9 @@ fn parse_range(params: &Value) -> Result<QueryNode> {
_ => None,
};
let Some(y) = year else { return b };
- if !matches!(fmt, "uuuu" | "yyyy" | "year") { return b; }
+ if !matches!(fmt, "uuuu" | "yyyy" | "year") {
+ return b;
+ }
let iso = if upper {
format!("{:04}-01-01T23:59:59.999Z", y)
} else {
@@ -783,15 +886,26 @@ fn parse_range(params: &Value) -> Result<QueryNode> {
};
Value::String(iso)
};
- if let Some(v) = gte.take() { gte = Some(rewrite(v, false)); }
- if let Some(v) = lte.take() { lte = Some(rewrite(v, true)); }
+ if let Some(v) = gte.take() {
+ gte = Some(rewrite(v, false));
+ }
+ if let Some(v) = lte.take() {
+ lte = Some(rewrite(v, true));
+ }
}
if gte.is_none() && gt.is_none() && lte.is_none() && lt.is_none() {
return invalid("`range` query must have at least one bound (gte, gt, lte, lt)");
}
- Ok(QueryNode::Range { field, gte, gt, lte, lt, boost })
+ Ok(QueryNode::Range {
+ field,
+ gte,
+ gt,
+ lte,
+ lt,
+ boost,
+ })
}
fn parse_prefix(params: &Value) -> Result<QueryNode> {
@@ -806,7 +920,11 @@ fn parse_prefix(params: &Value) -> Result<QueryNode> {
let field = field.clone();
if let Some(value) = raw.as_str() {
- return Ok(QueryNode::Prefix { field, value: value.to_string(), boost: None });
+ return Ok(QueryNode::Prefix {
+ field,
+ value: value.to_string(),
+ boost: None,
+ });
}
let inner = raw
@@ -814,9 +932,16 @@ fn parse_prefix(params: &Value) -> Result<QueryNode> {
.ok_or_else(|| qerr("`prefix` field value must be a string or object"))?;
let value = string_field(inner, "value")?;
- let boost = inner.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
+ let boost = inner
+ .get("boost")
+ .and_then(|v| v.as_f64())
+ .map(|b| b as f32);
- Ok(QueryNode::Prefix { field, value, boost })
+ Ok(QueryNode::Prefix {
+ field,
+ value,
+ boost,
+ })
}
fn parse_wildcard(params: &Value) -> Result<QueryNode> {
@@ -831,7 +956,11 @@ fn parse_wildcard(params: &Value) -> Result<QueryNode> {
let field = field.clone();
if let Some(value) = raw.as_str() {
- return Ok(QueryNode::Wildcard { field, value: value.to_string(), boost: None });
+ return Ok(QueryNode::Wildcard {
+ field,
+ value: value.to_string(),
+ boost: None,
+ });
}
let inner = raw
@@ -839,9 +968,16 @@ fn parse_wildcard(params: &Value) -> Result<QueryNode> {
.ok_or_else(|| qerr("`wildcard` field value must be a string or object"))?;
let value = string_field(inner, "value")?;
- let boost = inner.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
+ let boost = inner
+ .get("boost")
+ .and_then(|v| v.as_f64())
+ .map(|b| b as f32);
- Ok(QueryNode::Wildcard { field, value, boost })
+ Ok(QueryNode::Wildcard {
+ field,
+ value,
+ boost,
+ })
}
fn parse_exists(params: &Value) -> Result<QueryNode> {
@@ -891,14 +1027,24 @@ fn parse_query_string(params: &Value) -> Result<QueryNode> {
// QueryString node if lowering isn't possible (the FTS path still
// tokenizes it) — but malformed range syntax is a hard parse error,
// never a silent term-match fallback.
- if let Some(lowered) = try_lower_query_string(&query, default_field.as_deref(), default_operator)? {
+ if let Some(lowered) =
+ try_lower_query_string(&query, default_field.as_deref(), default_operator)?
+ {
return Ok(match boost {
- Some(b) if b != 1.0 => QueryNode::Boosted { boost: b, query: Box::new(lowered) },
+ Some(b) if b != 1.0 => QueryNode::Boosted {
+ boost: b,
+ query: Box::new(lowered),
+ },
_ => lowered,
});
}
- Ok(QueryNode::QueryString { query, default_field, default_operator, boost })
+ Ok(QueryNode::QueryString {
+ query,
+ default_field,
+ default_operator,
+ boost,
+ })
}
/// Lower a Lucene-style query string into a QueryNode::Bool tree.
@@ -923,8 +1069,12 @@ fn try_lower_query_string(
default_field: Option<&str>,
default_op: Option<BoolOperator>,
) -> Result<Option<QueryNode>> {
- let Some(tokens) = tokenize_query_string(q)? else { return Ok(None) };
- if tokens.is_empty() { return Ok(None); }
+ let Some(tokens) = tokenize_query_string(q)? else {
+ return Ok(None);
+ };
+ if tokens.is_empty() {
+ return Ok(None);
+ }
// Range clauses must target a concrete field: resolve unqualified
// ranges against default_field up-front so `>10` with no usable
// default errors instead of degrading to a term match.
@@ -932,8 +1082,7 @@ fn try_lower_query_string(
for t in &tokens {
if let QsTok::Range { field, .. } = t {
has_range = true;
- if field.is_empty()
- && !matches!(default_field, Some(df) if !df.is_empty() && df != "*")
+ if field.is_empty() && !matches!(default_field, Some(df) if !df.is_empty() && df != "*")
{
return Err(qerr(
"query_string range requires an explicit field (e.g. `price:>10`) or a non-wildcard default_field",
@@ -954,9 +1103,10 @@ fn try_lower_query_string(
#[derive(Debug, Clone)]
enum QsTok {
- Term(String, String), // (field, value) — field empty if unqualified
- Phrase(String, String), // (field, value)
- Range { // field empty if unqualified
+ Term(String, String), // (field, value) — field empty if unqualified
+ Phrase(String, String), // (field, value)
+ Range {
+ // field empty if unqualified
field: String,
gt: Option<Value>,
gte: Option<Value>,
@@ -1032,9 +1182,26 @@ fn qs_parse_bracket_range(q: &str, field: &str, open: usize) -> Result<(QsTok, u
));
}
let (mut gt, mut gte, mut lt, mut lte) = (None, None, None, None);
- if inclusive_lo { gte = lo; } else { gt = lo; }
- if inclusive_hi { lte = hi; } else { lt = hi; }
- Ok((QsTok::Range { field: field.to_string(), gt, gte, lt, lte }, j + 1))
+ if inclusive_lo {
+ gte = lo;
+ } else {
+ gt = lo;
+ }
+ if inclusive_hi {
+ lte = hi;
+ } else {
+ lt = hi;
+ }
+ Ok((
+ QsTok::Range {
+ field: field.to_string(),
+ gt,
+ gte,
+ lt,
+ lte,
+ },
+ j + 1,
+ ))
}
/// Parse a comparison range (`>N`, `>=N`, `<N`, `<=N`). `rest` is the text
@@ -1074,7 +1241,16 @@ fn qs_parse_cmp_range(q: &str, field: &str, rest: &str, mut i: usize) -> Result<
"<" => lt = Some(bound),
_ => lte = Some(bound),
}
- Ok((QsTok::Range { field: field.to_string(), gt, gte, lt, lte }, i))
+ Ok((
+ QsTok::Range {
+ field: field.to_string(),
+ gt,
+ gte,
+ lt,
+ lte,
+ },
+ i,
+ ))
}
/// Tokenize a Lucene query string. `Ok(None)` means "can't tokenize — fall
@@ -1086,17 +1262,40 @@ fn tokenize_query_string(q: &str) -> Result<Option<Vec<QsTok>>> {
let mut i = 0;
while i < bytes.len() {
let c = bytes[i] as char;
- if c.is_whitespace() { i += 1; continue; }
- if c == '(' { out.push(QsTok::LParen); i += 1; continue; }
- if c == ')' { out.push(QsTok::RParen); i += 1; continue; }
- if c == '+' { out.push(QsTok::Must); i += 1; continue; }
- if c == '-' { out.push(QsTok::Not); i += 1; continue; }
+ if c.is_whitespace() {
+ i += 1;
+ continue;
+ }
+ if c == '(' {
+ out.push(QsTok::LParen);
+ i += 1;
+ continue;
+ }
+ if c == ')' {
+ out.push(QsTok::RParen);
+ i += 1;
+ continue;
+ }
+ if c == '+' {
+ out.push(QsTok::Must);
+ i += 1;
+ continue;
+ }
+ if c == '-' {
+ out.push(QsTok::Not);
+ i += 1;
+ continue;
+ }
// Quoted phrase — optional `field:"value"` prefix handled below.
if c == '"' {
let start = i + 1;
let mut j = start;
- while j < bytes.len() && bytes[j] as char != '"' { j += 1; }
- if j >= bytes.len() { return Ok(None); }
+ while j < bytes.len() && bytes[j] as char != '"' {
+ j += 1;
+ }
+ if j >= bytes.len() {
+ return Ok(None);
+ }
let val = q[start..j].to_string();
out.push(QsTok::Phrase(String::new(), val));
i = j + 1;
@@ -1122,15 +1321,26 @@ fn tokenize_query_string(q: &str) -> Result<Option<Vec<QsTok>>> {
let start = i;
while i < bytes.len() {
let ch = bytes[i] as char;
- if ch.is_whitespace() || ch == '(' || ch == ')' { break; }
+ if ch.is_whitespace() || ch == '(' || ch == ')' {
+ break;
+ }
i += 1;
}
let tok = &q[start..i];
// Keywords OR / AND / NOT (case-insensitive, must stand alone).
match tok {
- "OR" | "or" | "||" => { out.push(QsTok::Or); continue; }
- "AND" | "and" | "&&" => { out.push(QsTok::And); continue; }
- "NOT" | "not" | "!" => { out.push(QsTok::Not); continue; }
+ "OR" | "or" | "||" => {
+ out.push(QsTok::Or);
+ continue;
+ }
+ "AND" | "and" | "&&" => {
+ out.push(QsTok::And);
+ continue;
+ }
+ "NOT" | "not" | "!" => {
+ out.push(QsTok::Not);
+ continue;
+ }
_ => {}
}
// Lucene query_string unescapes `\X` → `X` for any char X;
@@ -1142,7 +1352,9 @@ fn tokenize_query_string(q: &str) -> Result<Option<Vec<QsTok>>> {
let mut chars = s.chars();
while let Some(c) = chars.next() {
if c == '\\' {
- if let Some(nxt) = chars.next() { out.push(nxt); }
+ if let Some(nxt) = chars.next() {
+ out.push(nxt);
+ }
} else {
out.push(c);
}
@@ -1153,17 +1365,21 @@ fn tokenize_query_string(q: &str) -> Result<Option<Vec<QsTok>>> {
if let Some(colon) = tok.find(':') {
let field = &tok[..colon];
let rest = &tok[colon + 1..];
- if rest.starts_with('"') {
+ if let Some(after_quote) = rest.strip_prefix('"') {
// Consume continuation until matching quote (may have been split).
- let mut buf = rest[1..].to_string();
+ let mut buf = after_quote.to_string();
if buf.ends_with('"') {
buf.pop();
out.push(QsTok::Phrase(field.to_string(), buf));
} else {
// Rejoin from original string.
let mut j = i;
- while j < bytes.len() && bytes[j] as char != '"' { j += 1; }
- if j >= bytes.len() { return Ok(None); }
+ while j < bytes.len() && bytes[j] as char != '"' {
+ j += 1;
+ }
+ if j >= bytes.len() {
+ return Ok(None);
+ }
let whole = format!("{} {}", buf, &q[i..j]);
out.push(QsTok::Phrase(field.to_string(), whole));
i = j + 1;
@@ -1206,7 +1422,9 @@ fn parse_qs_or(
*pos += 1;
let right = parse_qs_and(toks, pos, default_field, default_op)?;
left = QueryNode::Bool {
- must: vec![], must_not: vec![], filter: vec![],
+ must: vec![],
+ must_not: vec![],
+ filter: vec![],
should: vec![left, right],
minimum_should_match: Some(MinShouldMatch::Fixed(1)),
};
@@ -1220,7 +1438,9 @@ fn parse_qs_or(
while *pos < toks.len() && !matches!(toks[*pos], QsTok::RParen | QsTok::Or | QsTok::And) {
let right = parse_qs_and(toks, pos, default_field, default_op)?;
left = QueryNode::Bool {
- must: vec![], must_not: vec![], filter: vec![],
+ must: vec![],
+ must_not: vec![],
+ filter: vec![],
should: vec![left, right],
minimum_should_match: Some(MinShouldMatch::Fixed(1)),
};
@@ -1240,26 +1460,34 @@ fn parse_qs_and(
let mut explicit_and = false;
loop {
// Optional + / - / NOT prefix for each clause.
- let mut force_must = false;
let mut force_not = false;
while *pos < toks.len() {
match &toks[*pos] {
- QsTok::Must => { force_must = true; *pos += 1; }
- QsTok::Not => { force_not = true; *pos += 1; }
+ QsTok::Must => {
+ *pos += 1;
+ }
+ QsTok::Not => {
+ force_not = true;
+ *pos += 1;
+ }
_ => break,
}
}
let node = parse_qs_unary(toks, pos, default_field)?;
if force_not {
not_clauses.push(node);
- } else if force_must {
- clauses.push(node);
} else {
clauses.push(node);
}
- if *pos >= toks.len() { break; }
+ if *pos >= toks.len() {
+ break;
+ }
match &toks[*pos] {
- QsTok::And => { explicit_and = true; *pos += 1; continue; }
+ QsTok::And => {
+ explicit_and = true;
+ *pos += 1;
+ continue;
+ }
QsTok::Or | QsTok::RParen => break,
_ => {
// Juxtaposition — treat as OR unless default_op is AND.
@@ -1273,20 +1501,22 @@ fn parse_qs_and(
if clauses.len() == 1 && not_clauses.is_empty() {
return Some(clauses.pop().unwrap());
}
- let should_mode = !explicit_and
- && !clauses.is_empty()
- && !matches!(default_op, Some(BoolOperator::And));
+ let should_mode =
+ !explicit_and && !clauses.is_empty() && !matches!(default_op, Some(BoolOperator::And));
let node = if should_mode {
QueryNode::Bool {
- must: vec![], filter: vec![],
+ must: vec![],
+ filter: vec![],
must_not: not_clauses,
should: clauses,
minimum_should_match: Some(MinShouldMatch::Fixed(1)),
}
} else {
QueryNode::Bool {
- must: clauses, filter: vec![],
- must_not: not_clauses, should: vec![],
+ must: clauses,
+ filter: vec![],
+ must_not: not_clauses,
+ should: vec![],
minimum_should_match: None,
}
};
@@ -1298,44 +1528,83 @@ fn parse_qs_unary(
pos: &mut usize,
default_field: Option<&str>,
) -> Option<QueryNode> {
- if *pos >= toks.len() { return None; }
+ if *pos >= toks.len() {
+ return None;
+ }
match toks[*pos].clone() {
QsTok::LParen => {
*pos += 1;
let n = parse_qs_or(toks, pos, default_field, None)?;
- if *pos >= toks.len() || !matches!(toks[*pos], QsTok::RParen) { return None; }
+ if *pos >= toks.len() || !matches!(toks[*pos], QsTok::RParen) {
+ return None;
+ }
*pos += 1;
Some(n)
}
QsTok::Term(field, value) => {
*pos += 1;
- let f = if field.is_empty() { default_field.unwrap_or("*").to_string() } else { field };
+ let f = if field.is_empty() {
+ default_field.unwrap_or("*").to_string()
+ } else {
+ field
+ };
// A bare term containing `*` or `?` is a Lucene wildcard —
// emit a Wildcard query so `q=shor*` / `q=te?t` match text
// tokens with the expected substitution semantics.
if value.contains('*') || value.contains('?') {
return Some(QueryNode::Wildcard {
- field: f, value, boost: None,
+ field: f,
+ value,
+ boost: None,
});
}
Some(QueryNode::Match {
- field: f, query: value, operator: BoolOperator::Or,
- analyzer: None, boost: None, minimum_should_match: None,
+ field: f,
+ query: value,
+ operator: BoolOperator::Or,
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
})
}
QsTok::Phrase(field, value) => {
*pos += 1;
- let f = if field.is_empty() { default_field.unwrap_or("*").to_string() } else { field };
+ let f = if field.is_empty() {
+ default_field.unwrap_or("*").to_string()
+ } else {
+ field
+ };
Some(QueryNode::MatchPhrase {
- field: f, query: value, slop: 0, analyzer: None, boost: None,
+ field: f,
+ query: value,
+ slop: 0,
+ analyzer: None,
+ boost: None,
})
}
- QsTok::Range { field, gt, gte, lt, lte } => {
+ QsTok::Range {
+ field,
+ gt,
+ gte,
+ lt,
+ lte,
+ } => {
*pos += 1;
// Unqualified ranges with no usable default_field were already
// rejected in try_lower_query_string.
- let f = if field.is_empty() { default_field.unwrap_or("*").to_string() } else { field };
- Some(QueryNode::Range { field: f, gte, gt, lte, lt, boost: None })
+ let f = if field.is_empty() {
+ default_field.unwrap_or("*").to_string()
+ } else {
+ field
+ };
+ Some(QueryNode::Range {
+ field: f,
+ gte,
+ gt,
+ lte,
+ lt,
+ boost: None,
+ })
}
_ => None,
}
@@ -1370,19 +1639,28 @@ fn parse_fuzzy(params: &Value) -> Result<QueryNode> {
Some(Value::String(s)) if s.to_uppercase() == "AUTO" => Fuzziness::Auto,
Some(Value::String(s)) => {
let n = s.parse::<u32>().map_err(|_| {
- qerr(format!("`fuzzy.fuzziness` must be 'AUTO' or an integer, got '{s}'"))
+ qerr(format!(
+ "`fuzzy.fuzziness` must be 'AUTO' or an integer, got '{s}'"
+ ))
})?;
Fuzziness::Fixed(n)
}
Some(Value::Number(n)) => {
- let n = n.as_u64().ok_or_else(|| qerr("`fuzzy.fuzziness` must be a non-negative integer"))? as u32;
+ let n = n
+ .as_u64()
+ .ok_or_else(|| qerr("`fuzzy.fuzziness` must be a non-negative integer"))?
+ as u32;
Fuzziness::Fixed(n)
}
None => Fuzziness::Auto,
_ => return invalid("`fuzzy.fuzziness` must be 'AUTO' or an integer"),
};
- Ok(QueryNode::Fuzzy { field, value, fuzziness })
+ Ok(QueryNode::Fuzzy {
+ field,
+ value,
+ fuzziness,
+ })
}
fn parse_regexp(params: &Value) -> Result<QueryNode> {
@@ -1397,7 +1675,10 @@ fn parse_regexp(params: &Value) -> Result<QueryNode> {
let field = field.clone();
if let Some(pattern) = raw.as_str() {
- return Ok(QueryNode::Regexp { field, pattern: pattern.to_string() });
+ return Ok(QueryNode::Regexp {
+ field,
+ pattern: pattern.to_string(),
+ });
}
let inner = raw
@@ -1437,7 +1718,11 @@ fn parse_match_phrase_prefix(params: &Value) -> Result<QueryNode> {
.and_then(Value::as_u64)
.unwrap_or(50) as u32;
- Ok(QueryNode::MatchPhrasePrefix { field, query, max_expansions })
+ Ok(QueryNode::MatchPhrasePrefix {
+ field,
+ query,
+ max_expansions,
+ })
}
fn parse_simple_query_string(params: &Value) -> Result<QueryNode> {
@@ -1494,7 +1779,9 @@ fn parse_simple_query_string(params: &Value) -> Result<QueryNode> {
} else {
(' ', tok)
};
- if term_text.is_empty() { continue; }
+ if term_text.is_empty() {
+ continue;
+ }
let node = make_simple_query_node(term_text, &fields);
match sign {
'+' => must.push(node),
@@ -1601,7 +1888,12 @@ fn parse_geo_distance(params: &Value) -> Result<QueryNode> {
let (lat, lon) = parse_lat_lon(location)?;
- Ok(QueryNode::GeoDistance { field, lat, lon, distance_km })
+ Ok(QueryNode::GeoDistance {
+ field,
+ lat,
+ lon,
+ distance_km,
+ })
}
/// Parse a `geo_bounding_box` query.
@@ -1646,14 +1938,20 @@ fn parse_geo_bounding_box(params: &Value) -> Result<QueryNode> {
let top_left = parse_lat_lon(tl_val)?;
let bottom_right = parse_lat_lon(br_val)?;
- Ok(QueryNode::GeoBoundingBox { field, top_left, bottom_right })
+ Ok(QueryNode::GeoBoundingBox {
+ field,
+ top_left,
+ bottom_right,
+ })
}
/// Parse a distance string like "10km", "5mi", "100m" into kilometres.
fn parse_distance_km(s: &str) -> Result<f64> {
let s = s.trim();
if let Some(km) = s.strip_suffix("km") {
- km.trim().parse::<f64>().map_err(|_| qerr(format!("invalid distance: `{s}`")))
+ km.trim()
+ .parse::<f64>()
+ .map_err(|_| qerr(format!("invalid distance: `{s}`")))
} else if let Some(mi) = s.strip_suffix("mi") {
mi.trim()
.parse::<f64>()
@@ -1666,7 +1964,8 @@ fn parse_distance_km(s: &str) -> Result<f64> {
.map_err(|_| qerr(format!("invalid distance: `{s}`")))
} else {
// Bare number interpreted as kilometres.
- s.parse::<f64>().map_err(|_| qerr(format!("invalid distance: `{s}`")))
+ s.parse::<f64>()
+ .map_err(|_| qerr(format!("invalid distance: `{s}`")))
}
}
@@ -1689,8 +1988,12 @@ fn parse_lat_lon(value: &Value) -> Result<(f64, f64)> {
}
Value::Array(arr) if arr.len() == 2 => {
// GeoJSON order: [lon, lat]
- let lon = arr[0].as_f64().ok_or_else(|| qerr("geo_point array element must be a number"))?;
- let lat = arr[1].as_f64().ok_or_else(|| qerr("geo_point array element must be a number"))?;
+ let lon = arr[0]
+ .as_f64()
+ .ok_or_else(|| qerr("geo_point array element must be a number"))?;
+ let lat = arr[1]
+ .as_f64()
+ .ok_or_else(|| qerr("geo_point array element must be a number"))?;
Ok((lat, lon))
}
Value::String(s) => {
@@ -1699,8 +2002,14 @@ fn parse_lat_lon(value: &Value) -> Result<(f64, f64)> {
if parts.len() != 2 {
return invalid(format!("geo_point string must be `lat,lon`, got `{s}`"));
}
- let lat = parts[0].trim().parse::<f64>().map_err(|_| qerr(format!("invalid lat in `{s}`")))?;
- let lon = parts[1].trim().parse::<f64>().map_err(|_| qerr(format!("invalid lon in `{s}`")))?;
+ let lat = parts[0]
+ .trim()
+ .parse::<f64>()
+ .map_err(|_| qerr(format!("invalid lat in `{s}`")))?;
+ let lon = parts[1]
+ .trim()
+ .parse::<f64>()
+ .map_err(|_| qerr(format!("invalid lon in `{s}`")))?;
Ok((lat, lon))
}
_ => invalid("geo_point must be an object {lat,lon}, array [lon,lat], or 'lat,lon' string"),
@@ -1730,7 +2039,13 @@ fn parse_bool(params: &Value) -> Result<QueryNode> {
.map(parse_min_should_match)
.transpose()?;
- Ok(QueryNode::Bool { must, should, must_not, filter, minimum_should_match })
+ Ok(QueryNode::Bool {
+ must,
+ should,
+ must_not,
+ filter,
+ minimum_should_match,
+ })
}
fn parse_constant_score(params: &Value) -> Result<QueryNode> {
@@ -1809,7 +2124,10 @@ fn parse_dis_max(params: &Value) -> Result<QueryNode> {
.and_then(|v| v.as_f64())
.unwrap_or(0.0) as f32;
- Ok(QueryNode::DisMax { queries, tie_breaker })
+ Ok(QueryNode::DisMax {
+ queries,
+ tie_breaker,
+ })
}
fn parse_knn(params: &Value) -> Result<QueryNode> {
@@ -1845,7 +2163,13 @@ fn parse_knn(params: &Value) -> Result<QueryNode> {
let boost = obj.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
- Ok(QueryNode::Knn { field, vector, k, filter, boost })
+ Ok(QueryNode::Knn {
+ field,
+ vector,
+ k,
+ filter,
+ boost,
+ })
}
fn parse_semantic(params: &Value) -> Result<QueryNode> {
@@ -1865,7 +2189,13 @@ fn parse_semantic(params: &Value) -> Result<QueryNode> {
let boost = obj.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
- Ok(QueryNode::SemanticSearch { field, text, k, filter, boost })
+ Ok(QueryNode::SemanticSearch {
+ field,
+ text,
+ k,
+ filter,
+ boost,
+ })
}
fn parse_hybrid(params: &Value) -> Result<QueryNode> {
@@ -1882,17 +2212,14 @@ fn parse_hybrid(params: &Value) -> Result<QueryNode> {
.iter()
.enumerate()
.map(|(i, v)| {
- let entry = v.as_object().ok_or_else(|| {
- qerr(format!("`hybrid.queries[{i}]` must be an object"))
- })?;
- let q_val = entry.get("query").ok_or_else(|| {
- qerr(format!("`hybrid.queries[{i}]` missing `query`"))
- })?;
+ let entry = v
+ .as_object()
+ .ok_or_else(|| qerr(format!("`hybrid.queries[{i}]` must be an object")))?;
+ let q_val = entry
+ .get("query")
+ .ok_or_else(|| qerr(format!("`hybrid.queries[{i}]` missing `query`")))?;
let query = parse_query(q_val)?;
- let weight = entry
- .get("weight")
- .and_then(|v| v.as_f64())
- .unwrap_or(1.0) as f32;
+ let weight = entry.get("weight").and_then(|v| v.as_f64()).unwrap_or(1.0) as f32;
Ok(WeightedQuery { query, weight })
})
.collect::<Result<Vec<_>>>()?;
@@ -1949,7 +2276,10 @@ fn parse_function_score(params: &Value) -> Result<QueryNode> {
.as_object()
.ok_or_else(|| qerr("`function_score` must be an object"))?;
- let name = obj.get("_name").and_then(|v| v.as_str()).map(str::to_string);
+ let name = obj
+ .get("_name")
+ .and_then(|v| v.as_str())
+ .map(str::to_string);
// Inner query (defaults to match_all).
let query = match obj.get("query") {
@@ -1975,7 +2305,11 @@ fn parse_function_score(params: &Value) -> Result<QueryNode> {
}
};
- let score_mode = match obj.get("score_mode").and_then(Value::as_str).unwrap_or("multiply") {
+ let score_mode = match obj
+ .get("score_mode")
+ .and_then(Value::as_str)
+ .unwrap_or("multiply")
+ {
"multiply" => ScoreMode::Multiply,
"sum" => ScoreMode::Sum,
"avg" => ScoreMode::Avg,
@@ -1985,7 +2319,11 @@ fn parse_function_score(params: &Value) -> Result<QueryNode> {
other => return invalid(format!("unknown score_mode `{other}`")),
};
- let boost_mode = match obj.get("boost_mode").and_then(Value::as_str).unwrap_or("multiply") {
+ let boost_mode = match obj
+ .get("boost_mode")
+ .and_then(Value::as_str)
+ .unwrap_or("multiply")
+ {
"multiply" => BoostMode::Multiply,
"replace" => BoostMode::Replace,
"sum" => BoostMode::Sum,
@@ -1995,7 +2333,10 @@ fn parse_function_score(params: &Value) -> Result<QueryNode> {
other => return invalid(format!("unknown boost_mode `{other}`")),
};
- let max_boost = obj.get("max_boost").and_then(|v| v.as_f64()).map(|b| b as f32);
+ let max_boost = obj
+ .get("max_boost")
+ .and_then(|v| v.as_f64())
+ .map(|b| b as f32);
let node = QueryNode::FunctionScore {
query: Box::new(query),
@@ -2017,7 +2358,11 @@ fn parse_score_function(v: &Value) -> Result<ScoreFunction> {
/// Parse score function fields from an existing object map.
fn parse_score_function_inline(obj: &serde_json::Map<String, Value>) -> Result<ScoreFunction> {
- let filter = obj.get("filter").map(parse_query).transpose()?.map(Box::new);
+ let filter = obj
+ .get("filter")
+ .map(parse_query)
+ .transpose()?
+ .map(Box::new);
let weight = obj.get("weight").and_then(|v| v.as_f64()).map(|w| w as f32);
let field_value_factor = match obj.get("field_value_factor") {
@@ -2031,7 +2376,11 @@ fn parse_score_function_inline(obj: &serde_json::Map<String, Value>) -> Result<S
.ok_or_else(|| qerr("`field_value_factor.field` is required"))?
.to_string();
let factor = fvf.get("factor").and_then(|v| v.as_f64()).unwrap_or(1.0) as f32;
- let modifier = match fvf.get("modifier").and_then(Value::as_str).unwrap_or("none") {
+ let modifier = match fvf
+ .get("modifier")
+ .and_then(Value::as_str)
+ .unwrap_or("none")
+ {
"none" => Modifier::None,
"log" => Modifier::Log,
"log1p" => Modifier::Log1p,
@@ -2045,7 +2394,12 @@ fn parse_score_function_inline(obj: &serde_json::Map<String, Value>) -> Result<S
other => return invalid(format!("unknown modifier `{other}`")),
};
let missing = fvf.get("missing").and_then(|v| v.as_f64());
- Some(FieldValueFactor { field, factor, modifier, missing })
+ Some(FieldValueFactor {
+ field,
+ factor,
+ modifier,
+ missing,
+ })
}
None => None,
};
@@ -2065,15 +2419,12 @@ fn parse_score_function_inline(obj: &serde_json::Map<String, Value>) -> Result<S
// script_score within a function: literal numeric source goes into
// the fast-path `script_score` field; richer Painless source goes
// into `script_source` and is evaluated at score time.
- let script_obj = obj
- .get("script_score")
- .and_then(|ss| ss.get("script"));
+ let script_obj = obj.get("script_score").and_then(|ss| ss.get("script"));
let script_source_str = script_obj
.and_then(|s| s.get("source"))
.and_then(Value::as_str)
.map(String::from);
- let script_params = script_obj
- .and_then(|s| s.get("params").cloned());
+ let script_params = script_obj.and_then(|s| s.get("params").cloned());
let literal_score = script_source_str
.as_ref()
.and_then(|s| s.trim().parse::<f32>().ok());
@@ -2187,12 +2538,15 @@ fn parse_sort_field_spec(field: &str, spec: &Value) -> Result<SortField> {
Some(other) => SortMissing::Value(Value::String(other.to_string())),
};
- let format = obj
- .get("format")
- .and_then(|v| v.as_str())
- .map(String::from);
+ let format = obj.get("format").and_then(|v| v.as_str()).map(String::from);
- Ok(SortField { field: field.to_string(), order, mode, missing, format })
+ Ok(SortField {
+ field: field.to_string(),
+ order,
+ mode,
+ missing,
+ format,
+ })
}
fn parse_sort_order(s: &str) -> Result<SortOrder> {
@@ -2263,9 +2617,9 @@ fn qerr(msg: impl Into<String>) -> QueryError {
fn parse_min_should_match(v: &Value) -> Result<MinShouldMatch> {
match v {
Value::Number(n) => {
- let i = n.as_u64().ok_or_else(|| {
- qerr("`minimum_should_match` must be non-negative")
- })?;
+ let i = n
+ .as_u64()
+ .ok_or_else(|| qerr("`minimum_should_match` must be non-negative"))?;
Ok(MinShouldMatch::Fixed(i as u32))
}
Value::String(s) => {
@@ -2274,12 +2628,16 @@ fn parse_min_should_match(v: &Value) -> Result<MinShouldMatch> {
let pct: i64 = pct.trim().parse().map_err(|_| {
qerr(format!("invalid `minimum_should_match` percentage: `{s}`"))
})?;
- let pct = if pct < 0 { (100 + pct).max(0) as u32 } else { pct as u32 };
+ let pct = if pct < 0 {
+ (100 + pct).max(0) as u32
+ } else {
+ pct as u32
+ };
Ok(MinShouldMatch::Percentage(pct))
} else {
- let n: u32 = s.parse().map_err(|_| {
- qerr(format!("invalid `minimum_should_match` value: `{s}`"))
- })?;
+ let n: u32 = s
+ .parse()
+ .map_err(|_| qerr(format!("invalid `minimum_should_match` value: `{s}`")))?;
Ok(MinShouldMatch::Fixed(n))
}
}
@@ -2302,10 +2660,7 @@ fn parse_bool_operator(v: Option<&Value>) -> Result<BoolOperator> {
}
/// Parse a bool clause list. ES allows a single object or an array.
-fn parse_clause_list(
- obj: &serde_json::Map<String, Value>,
- key: &str,
-) -> Result<Vec<QueryNode>> {
+fn parse_clause_list(obj: &serde_json::Map<String, Value>, key: &str) -> Result<Vec<QueryNode>> {
match obj.get(key) {
None => Ok(vec![]),
Some(Value::Array(arr)) => arr.iter().map(parse_query).collect(),
@@ -2413,7 +2768,11 @@ fn parse_more_like_this(params: &Value) -> Result<QueryNode> {
.values()
.filter_map(|v| v.as_str().map(str::to_string))
.collect();
- if text.is_empty() { None } else { Some(text.join(" ")) }
+ if text.is_empty() {
+ None
+ } else {
+ Some(text.join(" "))
+ }
} else {
None
}
@@ -2568,9 +2927,16 @@ fn parse_span_near(params: &Value) -> Result<QueryNode> {
}
let slop = obj.get("slop").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
- let in_order = obj.get("in_order").and_then(|v| v.as_bool()).unwrap_or(false);
+ let in_order = obj
+ .get("in_order")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
- Ok(QueryNode::SpanNear { clauses, slop, in_order })
+ Ok(QueryNode::SpanNear {
+ clauses,
+ slop,
+ in_order,
+ })
}
/// Parse a `span_or` query.
@@ -2611,7 +2977,10 @@ fn parse_span_not(params: &Value) -> Result<QueryNode> {
.ok_or_else(|| qerr("`span_not` requires an `exclude` clause"))
.and_then(parse_query)?;
- Ok(QueryNode::SpanNot { include: Box::new(include), exclude: Box::new(exclude) })
+ Ok(QueryNode::SpanNot {
+ include: Box::new(include),
+ exclude: Box::new(exclude),
+ })
}
/// Parse a `span_first` query.
@@ -2628,7 +2997,10 @@ fn parse_span_first(params: &Value) -> Result<QueryNode> {
.and_then(parse_query)?;
let end = obj.get("end").and_then(|v| v.as_u64()).unwrap_or(1) as u32;
- Ok(QueryNode::SpanFirst { match_query: Box::new(match_query), end })
+ Ok(QueryNode::SpanFirst {
+ match_query: Box::new(match_query),
+ end,
+ })
}
/// Parse span_containing / span_within — use the `big` clause only.
@@ -2662,9 +3034,16 @@ fn parse_has_child(params: &Value) -> Result<QueryNode> {
.ok_or_else(|| qerr("`has_child` requires a `query`"))
.and_then(parse_query)?;
- let score_mode = obj.get("score_mode").and_then(|v| v.as_str()).map(str::to_string);
+ let score_mode = obj
+ .get("score_mode")
+ .and_then(|v| v.as_str())
+ .map(str::to_string);
- Ok(QueryNode::HasChild { child_type, query: Box::new(query), score_mode })
+ Ok(QueryNode::HasChild {
+ child_type,
+ query: Box::new(query),
+ score_mode,
+ })
}
/// Parse a `has_parent` query.
@@ -2688,7 +3067,11 @@ fn parse_has_parent(params: &Value) -> Result<QueryNode> {
let score = obj.get("score").and_then(|v| v.as_bool()).unwrap_or(false);
- Ok(QueryNode::HasParent { parent_type, query: Box::new(query), score })
+ Ok(QueryNode::HasParent {
+ parent_type,
+ query: Box::new(query),
+ score,
+ })
}
/// Parse a `geo_polygon` query.
@@ -2714,7 +3097,7 @@ fn parse_geo_polygon(params: &Value) -> Result<QueryNode> {
let points = points_arr
.iter()
- .map(|p| parse_lat_lon_obj(p))
+ .map(parse_lat_lon_obj)
.collect::<Result<Vec<_>>>()?;
Ok(QueryNode::GeoPolygon { field, points })
@@ -2722,10 +3105,16 @@ fn parse_geo_polygon(params: &Value) -> Result<QueryNode> {
/// Parse a geo point from a JSON object like `{"lat": 40, "lon": -74}`.
fn parse_lat_lon_obj(v: &Value) -> Result<(f64, f64)> {
- let obj = v.as_object().ok_or_else(|| qerr("geo point must be an object {lat, lon}"))?;
- let lat = obj.get("lat").and_then(|x| x.as_f64())
+ let obj = v
+ .as_object()
+ .ok_or_else(|| qerr("geo point must be an object {lat, lon}"))?;
+ let lat = obj
+ .get("lat")
+ .and_then(|x| x.as_f64())
.ok_or_else(|| qerr("geo point missing `lat`"))?;
- let lon = obj.get("lon").and_then(|x| x.as_f64())
+ let lon = obj
+ .get("lon")
+ .and_then(|x| x.as_f64())
.ok_or_else(|| qerr("geo point missing `lon`"))?;
Ok((lat, lon))
}
@@ -2740,7 +3129,9 @@ fn parse_geo_shape(params: &Value) -> Result<QueryNode> {
let (field, field_params) = obj
.iter()
- .find(|(k, _)| k.as_str() != "boost" && k.as_str() != "_name" && k.as_str() != "ignore_unmapped")
+ .find(|(k, _)| {
+ k.as_str() != "boost" && k.as_str() != "_name" && k.as_str() != "ignore_unmapped"
+ })
.ok_or_else(|| qerr("`geo_shape` must specify a field"))?;
let field = field.clone();
@@ -2759,39 +3150,52 @@ fn parse_geo_shape(params: &Value) -> Result<QueryNode> {
let shape = match shape_type.as_str() {
"point" => {
// coordinates: [lon, lat]
- let coords = shape_val.get("coordinates")
+ let coords = shape_val
+ .get("coordinates")
.and_then(|v| v.as_array())
.ok_or_else(|| qerr("`geo_shape` point needs `coordinates`"))?;
- let lon = coords.get(0).and_then(|v| v.as_f64()).unwrap_or(0.0);
+ let lon = coords.first().and_then(|v| v.as_f64()).unwrap_or(0.0);
let lat = coords.get(1).and_then(|v| v.as_f64()).unwrap_or(0.0);
GeoShapeType::Point { lat, lon }
}
"envelope" => {
// coordinates: [[top_left_lon, top_left_lat], [bottom_right_lon, bottom_right_lat]]
- let coords = shape_val.get("coordinates")
+ let coords = shape_val
+ .get("coordinates")
.and_then(|v| v.as_array())
.ok_or_else(|| qerr("`geo_shape` envelope needs `coordinates`"))?;
- let tl = coords.get(0).and_then(|v| v.as_array())
+ let tl = coords
+ .first()
+ .and_then(|v| v.as_array())
.ok_or_else(|| qerr("`geo_shape` envelope top-left missing"))?;
- let br = coords.get(1).and_then(|v| v.as_array())
+ let br = coords
+ .get(1)
+ .and_then(|v| v.as_array())
.ok_or_else(|| qerr("`geo_shape` envelope bottom-right missing"))?;
- let tl_lon = tl.get(0).and_then(|v| v.as_f64()).unwrap_or(0.0);
+ let tl_lon = tl.first().and_then(|v| v.as_f64()).unwrap_or(0.0);
let tl_lat = tl.get(1).and_then(|v| v.as_f64()).unwrap_or(0.0);
- let br_lon = br.get(0).and_then(|v| v.as_f64()).unwrap_or(0.0);
+ let br_lon = br.first().and_then(|v| v.as_f64()).unwrap_or(0.0);
let br_lat = br.get(1).and_then(|v| v.as_f64()).unwrap_or(0.0);
- GeoShapeType::Envelope { top_left: (tl_lat, tl_lon), bottom_right: (br_lat, br_lon) }
+ GeoShapeType::Envelope {
+ top_left: (tl_lat, tl_lon),
+ bottom_right: (br_lat, br_lon),
+ }
}
"polygon" => {
// coordinates: [[[lon, lat], ...]] (outer ring only)
- let rings = shape_val.get("coordinates")
+ let rings = shape_val
+ .get("coordinates")
.and_then(|v| v.as_array())
.ok_or_else(|| qerr("`geo_shape` polygon needs `coordinates`"))?;
- let outer = rings.get(0).and_then(|v| v.as_array())
+ let outer = rings
+ .first()
+ .and_then(|v| v.as_array())
.ok_or_else(|| qerr("`geo_shape` polygon outer ring missing"))?;
- let points = outer.iter()
+ let points = outer
+ .iter()
.filter_map(|p| {
let arr = p.as_array()?;
- let lon = arr.get(0)?.as_f64()?;
+ let lon = arr.first()?.as_f64()?;
let lat = arr.get(1)?.as_f64()?;
Some((lat, lon))
})
@@ -2800,14 +3204,21 @@ fn parse_geo_shape(params: &Value) -> Result<QueryNode> {
}
"circle" => {
// coordinates: [lon, lat], radius: "10km" or number
- let coords = shape_val.get("coordinates")
+ let coords = shape_val
+ .get("coordinates")
.and_then(|v| v.as_array())
.ok_or_else(|| qerr("`geo_shape` circle needs `coordinates`"))?;
- let lon = coords.get(0).and_then(|v| v.as_f64()).unwrap_or(0.0);
+ let lon = coords.first().and_then(|v| v.as_f64()).unwrap_or(0.0);
let lat = coords.get(1).and_then(|v| v.as_f64()).unwrap_or(0.0);
- let radius_km = parse_distance_to_km(shape_val.get("radius")
- .ok_or_else(|| qerr("`geo_shape` circle needs `radius`"))?)?;
- GeoShapeType::Circle { center: (lat, lon), radius_km }
+ let radius_km = parse_distance_to_km(
+ shape_val
+ .get("radius")
+ .ok_or_else(|| qerr("`geo_shape` circle needs `radius`"))?,
+ )?;
+ GeoShapeType::Circle {
+ center: (lat, lon),
+ radius_km,
+ }
}
other => return invalid(format!("unsupported geo_shape type `{}`", other)),
};
@@ -2823,14 +3234,23 @@ fn parse_distance_to_km(v: &Value) -> Result<f64> {
if let Some(s) = v.as_str() {
let s = s.trim().to_lowercase();
if let Some(stripped) = s.strip_suffix("km") {
- return stripped.trim().parse::<f64>().map_err(|_| qerr("invalid distance km"));
+ return stripped
+ .trim()
+ .parse::<f64>()
+ .map_err(|_| qerr("invalid distance km"));
}
if let Some(stripped) = s.strip_suffix("mi") {
- let miles: f64 = stripped.trim().parse().map_err(|_| qerr("invalid distance mi"))?;
+ let miles: f64 = stripped
+ .trim()
+ .parse()
+ .map_err(|_| qerr("invalid distance mi"))?;
return Ok(miles * 1.60934);
}
if let Some(stripped) = s.strip_suffix('m') {
- let metres: f64 = stripped.trim().parse().map_err(|_| qerr("invalid distance m"))?;
+ let metres: f64 = stripped
+ .trim()
+ .parse()
+ .map_err(|_| qerr("invalid distance m"))?;
return Ok(metres / 1000.0);
}
if let Ok(n) = s.parse::<f64>() {
@@ -2864,7 +3284,9 @@ fn parse_match_bool_prefix(params: &Value) -> Result<QueryNode> {
let (query_str, mm, operator_and, analyzer, fuzziness) = if let Some(s) = raw.as_str() {
(s.to_string(), None, false, None, None)
} else {
- let inner = raw.as_object().ok_or_else(|| qerr("`match_bool_prefix` field value must be a string or object"))?;
+ let inner = raw
+ .as_object()
+ .ok_or_else(|| qerr("`match_bool_prefix` field value must be a string or object"))?;
let q = string_field(inner, "query")?;
let mm_val = inner.get("minimum_should_match").and_then(|v| match v {
Value::Number(n) => n.as_u64().map(|i| MinShouldMatch::Fixed(i as u32)),
@@ -2878,13 +3300,24 @@ fn parse_match_bool_prefix(params: &Value) -> Result<QueryNode> {
_ => None,
});
let op_and = matches!(
- inner.get("operator").and_then(Value::as_str).map(|s| s.to_ascii_lowercase()).as_deref(),
+ inner
+ .get("operator")
+ .and_then(Value::as_str)
+ .map(|s| s.to_ascii_lowercase())
+ .as_deref(),
Some("and")
);
- let analyzer = inner.get("analyzer").and_then(Value::as_str).map(str::to_string);
+ let analyzer = inner
+ .get("analyzer")
+ .and_then(Value::as_str)
+ .map(str::to_string);
let fuzziness = inner.get("fuzziness").map(|v| match v {
Value::String(s) if s.eq_ignore_ascii_case("auto") => crate::ast::Fuzziness::Auto,
- Value::String(s) => s.parse::<u32>().ok().map(crate::ast::Fuzziness::Fixed).unwrap_or(crate::ast::Fuzziness::Auto),
+ Value::String(s) => s
+ .parse::<u32>()
+ .ok()
+ .map(crate::ast::Fuzziness::Fixed)
+ .unwrap_or(crate::ast::Fuzziness::Auto),
Value::Number(n) => crate::ast::Fuzziness::Fixed(n.as_u64().unwrap_or(0) as u32),
_ => crate::ast::Fuzziness::Auto,
});
@@ -2894,12 +3327,13 @@ fn parse_match_bool_prefix(params: &Value) -> Result<QueryNode> {
// Analyzer semantics: "whitespace" preserves case, "keyword" treats
// input as single token, default/standard lowercases. We approximate
// by case-folding unless the requested analyzer is case-preserving.
- let analyzer_lowercases = match analyzer.as_deref() {
- Some("whitespace") | Some("keyword") => false,
- _ => true,
- };
+ let analyzer_lowercases = !matches!(analyzer.as_deref(), Some("whitespace") | Some("keyword"));
let fold = |t: &str| -> String {
- if analyzer_lowercases { t.to_lowercase() } else { t.to_string() }
+ if analyzer_lowercases {
+ t.to_lowercase()
+ } else {
+ t.to_string()
+ }
};
let raw_tokens: Vec<String> = if analyzer.as_deref() == Some("keyword") {
@@ -2912,14 +3346,22 @@ fn parse_match_bool_prefix(params: &Value) -> Result<QueryNode> {
}
if raw_tokens.len() == 1 {
- return Ok(QueryNode::Prefix { field, value: fold(&raw_tokens[0]), boost: None });
+ return Ok(QueryNode::Prefix {
+ field,
+ value: fold(&raw_tokens[0]),
+ boost: None,
+ });
}
let last_idx = raw_tokens.len() - 1;
let build_leaf = |tok: &str| -> QueryNode {
let folded = fold(tok);
- if let Some(fz) = fuzziness.clone() {
- QueryNode::Fuzzy { field: field.clone(), value: folded, fuzziness: fz }
+ if let Some(fz) = fuzziness {
+ QueryNode::Fuzzy {
+ field: field.clone(),
+ value: folded,
+ fuzziness: fz,
+ }
} else {
QueryNode::Match {
field: field.clone(),
@@ -2931,7 +3373,10 @@ fn parse_match_bool_prefix(params: &Value) -> Result<QueryNode> {
}
}
};
- let mut clauses: Vec<QueryNode> = raw_tokens[..last_idx].iter().map(|t| build_leaf(t)).collect();
+ let mut clauses: Vec<QueryNode> = raw_tokens[..last_idx]
+ .iter()
+ .map(|t| build_leaf(t))
+ .collect();
clauses.push(QueryNode::Prefix {
field: field.clone(),
value: fold(&raw_tokens[last_idx]),
@@ -2963,10 +3408,7 @@ fn parse_terms_set(params: &Value) -> Result<QueryNode> {
.as_object()
.ok_or_else(|| qerr("`terms_set` must be an object"))?;
- let field_entries: Vec<_> = obj
- .iter()
- .filter(|(k, _)| k.as_str() != "boost")
- .collect();
+ let field_entries: Vec<_> = obj.iter().filter(|(k, _)| k.as_str() != "boost").collect();
if field_entries.len() != 1 {
return invalid("`terms_set` query must have exactly one field entry");
@@ -2993,7 +3435,10 @@ fn parse_terms_set(params: &Value) -> Result<QueryNode> {
crate::ast::MinShouldMatch::Fixed(1)
})
.or_else(|| {
- inner.get("minimum_should_match").map(parse_min_should_match).and_then(|r| r.ok())
+ inner
+ .get("minimum_should_match")
+ .map(parse_min_should_match)
+ .and_then(|r| r.ok())
});
let should: Vec<QueryNode> = terms
@@ -3044,72 +3489,6 @@ fn parse_intervals(params: &Value) -> Result<QueryNode> {
})
}
-fn parse_intervals_rule(field: &str, rule: &Value) -> Result<QueryNode> {
- let field = field.to_string();
- // `match` sub-rule.
- if let Some(m) = rule.get("match") {
- let q = m.get("query").and_then(Value::as_str).unwrap_or("").to_string();
- let ordered = m.get("ordered").and_then(Value::as_bool).unwrap_or(false);
- let max_gaps = m.get("max_gaps").and_then(Value::as_i64);
- if q.is_empty() {
- return Ok(QueryNode::Exists { field });
- }
- if ordered {
- // Positive max_gaps maps to slop; -1 / absent → large slop.
- let slop = match max_gaps {
- Some(n) if n >= 0 => n as u32,
- _ => 50,
- };
- return Ok(QueryNode::MatchPhrase { field, query: q, slop, analyzer: None, boost: None });
- }
- // Unordered: all tokens must be present.
- return Ok(QueryNode::Match {
- field, query: q, operator: BoolOperator::And,
- analyzer: None, boost: None, minimum_should_match: None,
- });
- }
- if let Some(p) = rule.get("prefix").and_then(Value::as_str) {
- return Ok(QueryNode::Prefix { field, value: p.to_string(), boost: None });
- }
- if let Some(p) = rule.get("prefix").and_then(|v| v.get("prefix")).and_then(Value::as_str) {
- return Ok(QueryNode::Prefix { field, value: p.to_string(), boost: None });
- }
- if let Some(w) = rule.get("wildcard").and_then(|v| v.get("pattern")).and_then(Value::as_str) {
- return Ok(QueryNode::Wildcard { field, value: w.to_string(), boost: None });
- }
- if let Some(fz) = rule.get("fuzzy").and_then(|v| v.get("term")).and_then(Value::as_str) {
- return Ok(QueryNode::Fuzzy { field, value: fz.to_string(), fuzziness: Fuzziness::Auto });
- }
- // all_of → bool.must of sub-intervals on the same field.
- if let Some(ao) = rule.get("all_of") {
- let subs = ao.get("intervals").and_then(Value::as_array);
- if let Some(arr) = subs {
- let must: Vec<QueryNode> = arr.iter()
- .filter_map(|sub| parse_intervals_rule(&field, sub).ok())
- .collect();
- return Ok(QueryNode::Bool {
- must, should: vec![], must_not: vec![], filter: vec![],
- minimum_should_match: None,
- });
- }
- }
- // any_of → bool.should of sub-intervals (minimum_should_match = 1).
- if let Some(ao) = rule.get("any_of") {
- let subs = ao.get("intervals").and_then(Value::as_array);
- if let Some(arr) = subs {
- let should: Vec<QueryNode> = arr.iter()
- .filter_map(|sub| parse_intervals_rule(&field, sub).ok())
- .collect();
- return Ok(QueryNode::Bool {
- must: vec![], should, must_not: vec![], filter: vec![],
- minimum_should_match: Some(MinShouldMatch::Fixed(1)),
- });
- }
- }
- // Unknown rule: accept without error (existence check).
- Ok(QueryNode::Exists { field })
-}
-
/// `script_score` — wrap the inner query and apply a Painless script
/// to compute each matched doc's score. The script's returned value
/// REPLACES the BM25 score (boost_mode: Replace).
@@ -3141,7 +3520,11 @@ fn parse_script_score(params: &Value) -> Result<QueryNode> {
field_value_factor: None,
random_score: None,
script_score: literal_score,
- script_source: if literal_score.is_some() { None } else { source },
+ script_source: if literal_score.is_some() {
+ None
+ } else {
+ source
+ },
script_params: s_params,
name: None,
distance_feature: None,
@@ -3194,7 +3577,9 @@ fn parse_distance_feature(params: &Value) -> Result<QueryNode> {
weight: boost,
field_value_factor: None,
random_score: None,
- script_score: None, script_source: None, script_params: None,
+ script_score: None,
+ script_source: None,
+ script_params: None,
name: None,
distance_feature: Some(df),
}],
@@ -3233,7 +3618,9 @@ fn parse_rank_feature(params: &Value) -> Result<QueryNode> {
weight: boost,
field_value_factor: Some(fvf),
random_score: None,
- script_score: None, script_source: None, script_params: None,
+ script_score: None,
+ script_source: None,
+ script_params: None,
name: None,
distance_feature: None,
}],
@@ -3278,8 +3665,10 @@ mod tests {
#[test]
fn test_match_shorthand() {
let node = q(json!({"match": {"title": "hello world"}}));
- assert!(matches!(node, QueryNode::Match { ref field, ref query, operator: BoolOperator::Or, .. }
- if field == "title" && query == "hello world"));
+ assert!(
+ matches!(node, QueryNode::Match { ref field, ref query, operator: BoolOperator::Or, .. }
+ if field == "title" && query == "hello world")
+ );
}
#[test]
@@ -3293,7 +3682,14 @@ mod tests {
}
}
}));
- if let QueryNode::Match { field, query, operator, analyzer, .. } = node {
+ if let QueryNode::Match {
+ field,
+ query,
+ operator,
+ analyzer,
+ ..
+ } = node
+ {
assert_eq!(field, "title");
assert_eq!(query, "hello world");
assert_eq!(operator, BoolOperator::And);
@@ -3335,8 +3731,10 @@ mod tests {
#[test]
fn test_match_phrase_shorthand() {
let node = q(json!({"match_phrase": {"body": "quick brown fox"}}));
- assert!(matches!(node, QueryNode::MatchPhrase { ref field, slop: 0, .. }
- if field == "body"));
+ assert!(
+ matches!(node, QueryNode::MatchPhrase { ref field, slop: 0, .. }
+ if field == "body")
+ );
}
#[test]
@@ -3355,7 +3753,13 @@ mod tests {
"fields": ["title", "body^2"]
}
}));
- if let QueryNode::MultiMatch { fields, query, match_type, .. } = node {
+ if let QueryNode::MultiMatch {
+ fields,
+ query,
+ match_type,
+ ..
+ } = node
+ {
assert_eq!(query, "hello");
assert_eq!(fields, vec!["title", "body^2"]);
assert_eq!(match_type, MultiMatchType::BestFields);
@@ -3374,7 +3778,13 @@ mod tests {
"operator": "AND"
}
}));
- assert!(matches!(node, QueryNode::MultiMatch { match_type: MultiMatchType::CrossFields, .. }));
+ assert!(matches!(
+ node,
+ QueryNode::MultiMatch {
+ match_type: MultiMatchType::CrossFields,
+ ..
+ }
+ ));
}
// ── term ──────────────────────────────────────────────────────────────────
@@ -3417,7 +3827,15 @@ mod tests {
#[test]
fn test_range_numeric() {
let node = q(json!({"range": {"age": {"gte": 18, "lt": 65}}}));
- if let QueryNode::Range { field, gte, lt, gt, lte, .. } = node {
+ if let QueryNode::Range {
+ field,
+ gte,
+ lt,
+ gt,
+ lte,
+ ..
+ } = node
+ {
assert_eq!(field, "age");
assert_eq!(gte, Some(json!(18)));
assert_eq!(lt, Some(json!(65)));
@@ -3444,8 +3862,10 @@ mod tests {
#[test]
fn test_prefix_shorthand() {
let node = q(json!({"prefix": {"name": "Joh"}}));
- assert!(matches!(node, QueryNode::Prefix { ref field, ref value, .. }
- if field == "name" && value == "Joh"));
+ assert!(
+ matches!(node, QueryNode::Prefix { ref field, ref value, .. }
+ if field == "name" && value == "Joh")
+ );
}
#[test]
@@ -3459,8 +3879,10 @@ mod tests {
#[test]
fn test_wildcard() {
let node = q(json!({"wildcard": {"name": "Jo*n"}}));
- assert!(matches!(node, QueryNode::Wildcard { ref field, ref value, .. }
- if field == "name" && value == "Jo*n"));
+ assert!(
+ matches!(node, QueryNode::Wildcard { ref field, ref value, .. }
+ if field == "name" && value == "Jo*n")
+ );
}
// ── exists ────────────────────────────────────────────────────────────────
@@ -3494,7 +3916,14 @@ mod tests {
"filter": [{"range": {"age": {"gte": 18}}}]
}
}));
- if let QueryNode::Bool { must, must_not, filter, should, .. } = node {
+ if let QueryNode::Bool {
+ must,
+ must_not,
+ filter,
+ should,
+ ..
+ } = node
+ {
assert_eq!(must.len(), 1);
assert_eq!(must_not.len(), 1);
assert_eq!(filter.len(), 1);
@@ -3534,7 +3963,11 @@ mod tests {
"minimum_should_match": 2
}
}));
- if let QueryNode::Bool { minimum_should_match, .. } = node {
+ if let QueryNode::Bool {
+ minimum_should_match,
+ ..
+ } = node
+ {
assert_eq!(minimum_should_match, Some(MinShouldMatch::Fixed(2)));
}
}
@@ -3547,7 +3980,11 @@ mod tests {
"minimum_should_match": "75%"
}
}));
- if let QueryNode::Bool { minimum_should_match, .. } = node {
+ if let QueryNode::Bool {
+ minimum_should_match,
+ ..
+ } = node
+ {
assert_eq!(minimum_should_match, Some(MinShouldMatch::Percentage(75)));
}
}
@@ -3590,9 +4027,24 @@ mod tests {
q(json!({"query_string": {"query": query}}))
}
- fn expect_range(node: QueryNode) -> (String, Option<Value>, Option<Value>, Option<Value>, Option<Value>) {
+ fn expect_range(
+ node: QueryNode,
+ ) -> (
+ String,
+ Option<Value>,
+ Option<Value>,
+ Option<Value>,
+ Option<Value>,
+ ) {
match node {
- QueryNode::Range { field, gt, gte, lt, lte, .. } => (field, gt, gte, lt, lte),
+ QueryNode::Range {
+ field,
+ gt,
+ gte,
+ lt,
+ lte,
+ ..
+ } => (field, gt, gte, lt, lte),
other => panic!("expected Range, got {other:?}"),
}
}
@@ -3694,7 +4146,13 @@ mod tests {
fn test_query_string_range_combined_and() {
// "msg:foo AND n:>1" → Bool.must = [Match(msg), Range(n, gt 1)]
let node = qs("msg:foo AND n:>1");
- let QueryNode::Bool { must, must_not, should, .. } = node else {
+ let QueryNode::Bool {
+ must,
+ must_not,
+ should,
+ ..
+ } = node
+ else {
panic!("expected Bool");
};
assert!(must_not.is_empty() && should.is_empty());
@@ -3708,7 +4166,9 @@ mod tests {
#[test]
fn test_query_string_range_combined_bracket_and() {
let node = qs("msg:foo AND n:[2 TO *]");
- let QueryNode::Bool { must, .. } = node else { panic!("expected Bool") };
+ let QueryNode::Bool { must, .. } = node else {
+ panic!("expected Bool")
+ };
assert_eq!(must.len(), 2);
let (field, _, gte, ..) = expect_range(must[1].clone());
assert_eq!(field, "n");
@@ -3731,14 +4191,17 @@ mod tests {
// Malformed / unrepresentable ranges must be parse errors,
// never a silent term match.
for bad in [
- "n:[2 TO", // unterminated bracket
- "n:[2 5]", // missing TO
- "n:>", // missing value
- "n:[* TO *]", // no usable bound
- ">10", // no field and no default_field
+ "n:[2 TO", // unterminated bracket
+ "n:[2 5]", // missing TO
+ "n:>", // missing value
+ "n:[* TO *]", // no usable bound
+ ">10", // no field and no default_field
] {
let res = parse_query(&json!({"query_string": {"query": bad}}));
- assert!(res.is_err(), "expected parse error for {bad:?}, got {res:?}");
+ assert!(
+ res.is_err(),
+ "expected parse error for {bad:?}, got {res:?}"
+ );
}
}
@@ -3753,7 +4216,14 @@ mod tests {
"k": 5
}
}));
- if let QueryNode::Knn { field, vector, k, filter, .. } = node {
+ if let QueryNode::Knn {
+ field,
+ vector,
+ k,
+ filter,
+ ..
+ } = node
+ {
assert_eq!(field, "embedding");
assert_eq!(k, 5);
assert_eq!(vector.len(), 3);
@@ -3773,7 +4243,13 @@ mod tests {
"filter": {"term": {"status": "active"}}
}
}));
- assert!(matches!(node, QueryNode::Knn { filter: Some(_), .. }));
+ assert!(matches!(
+ node,
+ QueryNode::Knn {
+ filter: Some(_),
+ ..
+ }
+ ));
}
// ── full request ──────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rs
index 1de9482e9..ddca7dec7 100644
--- a/engine/crates/xerj-query/src/planner.rs
+++ b/engine/crates/xerj-query/src/planner.rs
@@ -178,45 +178,72 @@ pub fn plan_query(query: QueryNode, schema: &Schema) -> Result<ExecutionPlan> {
Ok(plan)
}
+// `schema` is threaded through only so recursive sub-plans can consult it;
+// the current arms don't read it directly, so silence the recursion lint.
+#[allow(clippy::only_used_in_recursion)]
fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
match query {
QueryNode::MatchAll => ExecutionPlan::MatchAll,
QueryNode::MatchNone => ExecutionPlan::MatchNone,
- QueryNode::Term { field, value, .. } => {
- ExecutionPlan::FtsScan {
- cost: 1.0,
- terms: vec![value],
- field,
- }
- }
+ QueryNode::Term { field, value, .. } => ExecutionPlan::FtsScan {
+ cost: 1.0,
+ terms: vec![value],
+ field,
+ },
QueryNode::Terms { field, values, .. } => {
let cost = values.len() as f64;
- ExecutionPlan::FtsScan { field, terms: values, cost }
+ ExecutionPlan::FtsScan {
+ field,
+ terms: values,
+ cost,
+ }
}
QueryNode::Ids { values } => ExecutionPlan::IdLookup { ids: values },
QueryNode::Exists { field } => ExecutionPlan::ExistsScan { field },
- QueryNode::Range { field, gte, gt, lte, lt, .. } => {
- ExecutionPlan::RangeScan { field, gte, gt, lte, lt, cost: 3.0 }
- }
-
- QueryNode::Prefix { field, value, .. } => {
- ExecutionPlan::PrefixScan { field, prefix: value, wildcard: false, cost: 5.0 }
- }
-
- QueryNode::Wildcard { field, value, .. } => {
- ExecutionPlan::PrefixScan { field, prefix: value, wildcard: true, cost: 5.0 }
- }
-
- QueryNode::Match { field, query, operator, .. } => {
+ QueryNode::Range {
+ field,
+ gte,
+ gt,
+ lte,
+ lt,
+ ..
+ } => ExecutionPlan::RangeScan {
+ field,
+ gte,
+ gt,
+ lte,
+ lt,
+ cost: 3.0,
+ },
+
+ QueryNode::Prefix { field, value, .. } => ExecutionPlan::PrefixScan {
+ field,
+ prefix: value,
+ wildcard: false,
+ cost: 5.0,
+ },
+
+ QueryNode::Wildcard { field, value, .. } => ExecutionPlan::PrefixScan {
+ field,
+ prefix: value,
+ wildcard: true,
+ cost: 5.0,
+ },
+
+ QueryNode::Match {
+ field,
+ query,
+ operator,
+ ..
+ } => {
// Simple whitespace tokenisation for cost estimation.
// Actual tokenisation happens in the FTS engine.
- let tokens: Vec<String> =
- query.split_whitespace().map(str::to_string).collect();
+ let tokens: Vec<String> = query.split_whitespace().map(str::to_string).collect();
let n = tokens.len().max(1);
let require_all = matches!(operator, crate::ast::BoolOperator::And);
ExecutionPlan::FtsSearch {
@@ -229,9 +256,10 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
}
}
- QueryNode::MatchPhrase { field, query, slop, .. } => {
- let tokens: Vec<String> =
- query.split_whitespace().map(str::to_string).collect();
+ QueryNode::MatchPhrase {
+ field, query, slop, ..
+ } => {
+ let tokens: Vec<String> = query.split_whitespace().map(str::to_string).collect();
ExecutionPlan::FtsSearch {
cost: 8.0,
field,
@@ -242,13 +270,20 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
}
}
- QueryNode::MultiMatch { fields, query, match_type, .. } => {
+ QueryNode::MultiMatch {
+ fields,
+ query,
+ match_type,
+ ..
+ } => {
// Expand to a Bool(should=[ Match(field, query) for field in fields ])
// and plan that.
- let tokens: Vec<String> =
- query.split_whitespace().map(str::to_string).collect();
+ let tokens: Vec<String> = query.split_whitespace().map(str::to_string).collect();
let n = tokens.len().max(1);
- let phrase = matches!(match_type, MultiMatchType::Phrase | MultiMatchType::PhrasePrefix);
+ let phrase = matches!(
+ match_type,
+ MultiMatchType::Phrase | MultiMatchType::PhrasePrefix
+ );
let cost = 5.0 * fields.len() as f64 * n as f64;
let sub_plans: Vec<ExecutionPlan> = fields
@@ -286,7 +321,13 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
}
}
- QueryNode::Bool { must, should, must_not, filter, minimum_should_match } => {
+ QueryNode::Bool {
+ must,
+ should,
+ must_not,
+ filter,
+ minimum_should_match,
+ } => {
// Plan each clause, then sort must/should by cost (cheapest first).
let plan_list = |clauses: Vec<QueryNode>| {
let mut plans: Vec<ExecutionPlan> =
@@ -317,25 +358,57 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
QueryNode::Constant { score, query } => {
let inner = plan_node(*query, schema);
let cost = inner.cost();
- ExecutionPlan::ConstantScore { score, inner: Box::new(inner), cost }
+ ExecutionPlan::ConstantScore {
+ score,
+ inner: Box::new(inner),
+ cost,
+ }
}
QueryNode::Boosted { boost, query } => {
let inner = plan_node(*query, schema);
let cost = inner.cost();
- ExecutionPlan::Boost { boost, inner: Box::new(inner), cost }
+ ExecutionPlan::Boost {
+ boost,
+ inner: Box::new(inner),
+ cost,
+ }
}
- QueryNode::Knn { field, vector, k, filter, .. } => {
+ QueryNode::Knn {
+ field,
+ vector,
+ k,
+ filter,
+ ..
+ } => {
let filter_plan = filter.map(|f| Box::new(plan_node(*f, schema)));
let cost = 50.0 * k as f64;
- ExecutionPlan::VectorScan { field, vector, k, filter: filter_plan, cost }
+ ExecutionPlan::VectorScan {
+ field,
+ vector,
+ k,
+ filter: filter_plan,
+ cost,
+ }
}
- QueryNode::SemanticSearch { field, text, k, filter, .. } => {
+ QueryNode::SemanticSearch {
+ field,
+ text,
+ k,
+ filter,
+ ..
+ } => {
let filter_plan = filter.map(|f| Box::new(plan_node(*f, schema)));
let cost = 50.0 * k as f64 + 20.0; // +20 for embedding inference
- ExecutionPlan::SemanticScan { field, text, k, filter: filter_plan, cost }
+ ExecutionPlan::SemanticScan {
+ field,
+ text,
+ k,
+ filter: filter_plan,
+ cost,
+ }
}
QueryNode::Hybrid { queries, fusion } => {
@@ -348,11 +421,19 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
(p, wq.weight)
})
.collect();
- ExecutionPlan::HybridMerge { plans, fusion, cost: total_cost }
+ ExecutionPlan::HybridMerge {
+ plans,
+ fusion,
+ cost: total_cost,
+ }
}
// Boosting: plan the positive sub-query; negative is evaluated at score time.
- QueryNode::Boosting { positive, negative, negative_boost } => {
+ QueryNode::Boosting {
+ positive,
+ negative,
+ negative_boost,
+ } => {
let pos_plan = plan_node(*positive, schema);
let neg_plan = plan_node(*negative, schema);
let cost = pos_plan.cost() + neg_plan.cost();
@@ -394,29 +475,23 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
}
// Fuzzy: treat as a term scan with higher cost due to edit-distance computation.
- QueryNode::Fuzzy { field, value, .. } => {
- ExecutionPlan::FtsScan {
- field,
- terms: vec![serde_json::Value::String(value)],
- cost: 10.0,
- }
- }
+ QueryNode::Fuzzy { field, value, .. } => ExecutionPlan::FtsScan {
+ field,
+ terms: vec![serde_json::Value::String(value)],
+ cost: 10.0,
+ },
// Regexp: full scan cost since we must evaluate each doc.
- QueryNode::Regexp { field, pattern } => {
- ExecutionPlan::FtsScan {
- field,
- terms: vec![serde_json::Value::String(pattern)],
- cost: 20.0,
- }
- }
+ QueryNode::Regexp { field, pattern } => ExecutionPlan::FtsScan {
+ field,
+ terms: vec![serde_json::Value::String(pattern)],
+ cost: 20.0,
+ },
// Intervals: position-aware text match. Plan as an exists-scan
// on the field — the doc-scan executor evaluates the rule
// directly against tokens (see engine::doc_matches_query).
- QueryNode::Intervals { field, .. } => {
- ExecutionPlan::ExistsScan { field }
- }
+ QueryNode::Intervals { field, .. } => ExecutionPlan::ExistsScan { field },
// MatchPhrasePrefix: treat as a phrase search on the given field.
QueryNode::MatchPhrasePrefix { field, query, .. } => {
@@ -432,46 +507,47 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
}
// SimpleQueryString: resolved to Bool at parse time; fallback to full scan.
- QueryNode::SimpleQueryString { query, .. } => {
- ExecutionPlan::FtsSearch {
- field: "_all".to_string(),
- tokens: vec![query],
- require_all: false,
- phrase: false,
- slop: 0,
- cost: 15.0,
- }
- }
+ QueryNode::SimpleQueryString { query, .. } => ExecutionPlan::FtsSearch {
+ field: "_all".to_string(),
+ tokens: vec![query],
+ require_all: false,
+ phrase: false,
+ slop: 0,
+ cost: 15.0,
+ },
// GeoDistance: full doc-scan with per-doc haversine computation.
- QueryNode::GeoDistance { field, .. } => {
- ExecutionPlan::ExistsScan { field }
- }
+ QueryNode::GeoDistance { field, .. } => ExecutionPlan::ExistsScan { field },
// GeoBoundingBox: full doc-scan with per-doc bounding box check.
- QueryNode::GeoBoundingBox { field, .. } => {
- ExecutionPlan::ExistsScan { field }
- }
+ QueryNode::GeoBoundingBox { field, .. } => ExecutionPlan::ExistsScan { field },
// FunctionScore: plan the inner query; function application happens at score time.
QueryNode::FunctionScore { query, .. } => {
let inner = plan_node(*query, schema);
let cost = inner.cost() + 5.0; // small overhead for function application
- ExecutionPlan::Boost { boost: 1.0, inner: Box::new(inner), cost }
+ ExecutionPlan::Boost {
+ boost: 1.0,
+ inner: Box::new(inner),
+ cost,
+ }
}
// Nested: plan the inner query with a doc-scan cost.
QueryNode::Nested { query, .. } => {
let inner = plan_node(*query, schema);
let cost = inner.cost() + 10.0; // array traversal overhead
- ExecutionPlan::Boost { boost: 1.0, inner: Box::new(inner), cost }
+ ExecutionPlan::Boost {
+ boost: 1.0,
+ inner: Box::new(inner),
+ cost,
+ }
}
// MoreLikeThis: treat as a multi-term FTS search.
QueryNode::MoreLikeThis { like, .. } => {
let all_text = like.join(" ");
- let tokens: Vec<String> =
- all_text.split_whitespace().map(str::to_string).collect();
+ let tokens: Vec<String> = all_text.split_whitespace().map(str::to_string).collect();
let cost = 5.0 * tokens.len().max(1) as f64;
ExecutionPlan::FtsSearch {
field: "_all".to_string(),
@@ -503,13 +579,11 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
QueryNode::Named { query, .. } => plan_node(*query, schema),
// ── Span queries — treat as doc-scans ────────────────────────────────
- QueryNode::SpanTerm { field, value } => {
- ExecutionPlan::FtsScan {
- field,
- terms: vec![serde_json::Value::String(value)],
- cost: 2.0,
- }
- }
+ QueryNode::SpanTerm { field, value } => ExecutionPlan::FtsScan {
+ field,
+ terms: vec![serde_json::Value::String(value)],
+ cost: 2.0,
+ },
QueryNode::SpanNear { clauses, .. } => {
let sub_plans: Vec<ExecutionPlan> =
@@ -559,7 +633,11 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
QueryNode::HasChild { query, .. } | QueryNode::HasParent { query, .. } => {
let inner = plan_node(*query, schema);
let cost = inner.cost() + 10.0;
- ExecutionPlan::Boost { boost: 1.0, inner: Box::new(inner), cost }
+ ExecutionPlan::Boost {
+ boost: 1.0,
+ inner: Box::new(inner),
+ cost,
+ }
}
// ── Geo shape queries — treat as doc-scans ────────────────────────────
@@ -672,7 +750,9 @@ mod tests {
query: Box::new(term_node("status", "active")),
};
let plan = plan_query(q, &empty_schema()).unwrap();
- assert!(matches!(plan, ExecutionPlan::ConstantScore { score, .. } if (score - 1.5).abs() < 0.001));
+ assert!(
+ matches!(plan, ExecutionPlan::ConstantScore { score, .. } if (score - 1.5).abs() < 0.001)
+ );
}
#[test]
diff --git a/engine/crates/xerj-query/src/rewriter.rs b/engine/crates/xerj-query/src/rewriter.rs
index 134bf2688..e55c58e9f 100644
--- a/engine/crates/xerj-query/src/rewriter.rs
+++ b/engine/crates/xerj-query/src/rewriter.rs
@@ -48,7 +48,13 @@ pub fn rewrite(query: QueryNode) -> QueryNode {
/// Single rewrite pass. Returns `(rewritten_query, did_change)`.
fn rewrite_once(query: QueryNode) -> (QueryNode, bool) {
match query {
- QueryNode::Bool { mut must, mut should, mut must_not, mut filter, minimum_should_match } => {
+ QueryNode::Bool {
+ mut must,
+ mut should,
+ mut must_not,
+ mut filter,
+ minimum_should_match,
+ } => {
// ── Recurse into children first ──
let (must2, c1) = rewrite_list(must);
let (should2, c2) = rewrite_list(should);
@@ -154,13 +160,22 @@ fn rewrite_once(query: QueryNode) -> (QueryNode, bool) {
{
let inner = filter.remove(0);
return (
- QueryNode::Constant { score: 0.0, query: Box::new(inner) },
+ QueryNode::Constant {
+ score: 0.0,
+ query: Box::new(inner),
+ },
true,
);
}
(
- QueryNode::Bool { must, should, must_not, filter, minimum_should_match },
+ QueryNode::Bool {
+ must,
+ should,
+ must_not,
+ filter,
+ minimum_should_match,
+ },
changed,
)
}
@@ -168,32 +183,87 @@ fn rewrite_once(query: QueryNode) -> (QueryNode, bool) {
// ── Constant / Boosted — recurse into child ──
QueryNode::Constant { score, query } => {
let (q2, changed) = rewrite_once(*query);
- (QueryNode::Constant { score, query: Box::new(q2) }, changed)
+ (
+ QueryNode::Constant {
+ score,
+ query: Box::new(q2),
+ },
+ changed,
+ )
}
QueryNode::Boosted { boost, query } => {
let (q2, changed) = rewrite_once(*query);
if q2.is_match_none() {
return (QueryNode::MatchNone, true);
}
- (QueryNode::Boosted { boost, query: Box::new(q2) }, changed)
+ (
+ QueryNode::Boosted {
+ boost,
+ query: Box::new(q2),
+ },
+ changed,
+ )
}
// ── Vector queries — recurse into filter ──
- QueryNode::Knn { field, vector, k, filter, boost } => {
- match filter {
- None => (QueryNode::Knn { field, vector, k, filter: None, boost }, false),
- Some(f) => {
- let (f2, changed) = rewrite_once(*f);
- (QueryNode::Knn { field, vector, k, filter: Some(Box::new(f2)), boost }, changed)
- }
+ QueryNode::Knn {
+ field,
+ vector,
+ k,
+ filter,
+ boost,
+ } => match filter {
+ None => (
+ QueryNode::Knn {
+ field,
+ vector,
+ k,
+ filter: None,
+ boost,
+ },
+ false,
+ ),
+ Some(f) => {
+ let (f2, changed) = rewrite_once(*f);
+ (
+ QueryNode::Knn {
+ field,
+ vector,
+ k,
+ filter: Some(Box::new(f2)),
+ boost,
+ },
+ changed,
+ )
}
- }
- QueryNode::SemanticSearch { field, text, k, filter, boost } => match filter {
- None => (QueryNode::SemanticSearch { field, text, k, filter: None, boost }, false),
+ },
+ QueryNode::SemanticSearch {
+ field,
+ text,
+ k,
+ filter,
+ boost,
+ } => match filter {
+ None => (
+ QueryNode::SemanticSearch {
+ field,
+ text,
+ k,
+ filter: None,
+ boost,
+ },
+ false,
+ ),
Some(f) => {
let (f2, changed) = rewrite_once(*f);
(
- QueryNode::SemanticSearch { field, text, k, filter: Some(Box::new(f2)), boost },
+ QueryNode::SemanticSearch {
+ field,
+ text,
+ k,
+ filter: Some(Box::new(f2)),
+ boost,
+ },
changed,
)
}
@@ -207,10 +277,19 @@ fn rewrite_once(query: QueryNode) -> (QueryNode, bool) {
.map(|wq| {
let (q2, c) = rewrite_once(wq.query);
changed |= c;
- crate::ast::WeightedQuery { query: q2, weight: wq.weight }
+ crate::ast::WeightedQuery {
+ query: q2,
+ weight: wq.weight,
+ }
})
.collect();
- (QueryNode::Hybrid { queries: queries2, fusion }, changed)
+ (
+ QueryNode::Hybrid {
+ queries: queries2,
+ fusion,
+ },
+ changed,
+ )
}
// ── Leaf nodes — nothing to rewrite ──
diff --git a/engine/crates/xerj-query/src/sort.rs b/engine/crates/xerj-query/src/sort.rs
index 7baab9166..a595cec13 100644
--- a/engine/crates/xerj-query/src/sort.rs
+++ b/engine/crates/xerj-query/src/sort.rs
@@ -132,9 +132,10 @@ pub enum SortMode {
/// `missing:"_last"` and `missing:"_first"`. That caused
/// `search/630_format_sort_missing_dates.yml` to serve a stale
/// missing-Last result to a subsequent missing-First request.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Default)]
pub enum SortMissing {
/// Documents missing the field sort last (default).
+ #[default]
Last,
/// Documents missing the field sort first.
First,
@@ -163,12 +164,6 @@ impl<'de> Deserialize<'de> for SortMissing {
}
}
-impl Default for SortMissing {
- fn default() -> Self {
- SortMissing::Last
- }
-}
-
// ─────────────────────────────────────────────────────────────────────────────
// Sort key extraction helpers (used by executor)
// ─────────────────────────────────────────────────────────────────────────────
@@ -200,7 +195,11 @@ pub fn compare_sort_keys(
SortMissing::Last => std::cmp::Ordering::Greater,
SortMissing::Value(_) => {
let raw = compare_values(av, bv, &field.missing);
- if field.order == SortOrder::Desc { raw.reverse() } else { raw }
+ if field.order == SortOrder::Desc {
+ raw.reverse()
+ } else {
+ raw
+ }
}
},
(_, serde_json::Value::Null) => match field.missing {
@@ -208,12 +207,20 @@ pub fn compare_sort_keys(
SortMissing::Last => std::cmp::Ordering::Less,
SortMissing::Value(_) => {
let raw = compare_values(av, bv, &field.missing);
- if field.order == SortOrder::Desc { raw.reverse() } else { raw }
+ if field.order == SortOrder::Desc {
+ raw.reverse()
+ } else {
+ raw
+ }
}
},
(_, _) => {
let raw = compare_values(av, bv, &field.missing);
- if field.order == SortOrder::Desc { raw.reverse() } else { raw }
+ if field.order == SortOrder::Desc {
+ raw.reverse()
+ } else {
+ raw
+ }
}
};
if cmp != std::cmp::Ordering::Equal {
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index f6d5539e0..33c6f7873 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -36,16 +36,16 @@ use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use axum::Router;
-use tokio::signal;
use tokio::net::TcpListener;
+use tokio::signal;
use tracing::{error, info, warn};
use tracing_subscriber::EnvFilter;
use xerj_api::{build_es_compat_router, build_native_router, AppState};
-use xerj_cluster::{ClusterNode, ClusterRunner, transport::TcpTransport};
+use xerj_cluster::{transport::TcpTransport, ClusterNode, ClusterRunner};
use xerj_common::{config::Config, metrics::Metrics};
-use xerj_engine::Engine;
use xerj_console_api::{state::ClusterMode, ConsoleState};
+use xerj_engine::Engine;
// ─────────────────────────────────────────────────────────────────────────────
// CLI
@@ -90,7 +90,11 @@ fn parse_args() -> CliArgs {
}
}
- CliArgs { config, data_dir, insecure }
+ CliArgs {
+ config,
+ data_dir,
+ insecure,
+ }
}
fn print_help() {
@@ -122,7 +126,10 @@ fn print_banner(cfg: &Config, startup_ms: u128) {
let tls = if cfg.tls.enabled { "TLS " } else { "plain" };
println!();
println!(" ▀▄▀ ▄▀█ █▄█ ▄▀█ █ █");
- println!(" █ █ █▀█ █ █▀█ █▀█ · ai v{}", env!("CARGO_PKG_VERSION"));
+ println!(
+ " █ █ █▀█ █ █▀█ █▀█ · ai v{}",
+ env!("CARGO_PKG_VERSION")
+ );
println!();
println!(" byte-perfect from design to code");
println!();
@@ -156,7 +163,9 @@ fn print_banner(cfg: &Config, startup_ms: u128) {
if !cfg.auth.enabled {
println!(" │ ⚠ Auth: DISABLED (--insecure) — anyone on the network can write");
} else {
- println!(" │ ✓ Auth: single API-key (no RBAC; per-doc / per-field controls roadmap v0.9)");
+ println!(
+ " │ ✓ Auth: single API-key (no RBAC; per-doc / per-field controls roadmap v0.9)"
+ );
}
println!(" │ ⚠ Audit: request tracing only — tamper-evident WORM audit log v0.9");
println!(" │ ⚠ Encryption-at-rest: not engine-level — use OS FDE or S3 SSE for now");
@@ -176,8 +185,7 @@ fn load_config(args: &CliArgs) -> Result<Config> {
let mut cfg = if let Some(path) = config_path {
info!("loading config from {}", path.display());
- Config::load(&path)
- .with_context(|| format!("load config from {}", path.display()))?
+ Config::load(&path).with_context(|| format!("load config from {}", path.display()))?
} else {
info!("no config file — using defaults");
Config::default()
@@ -410,22 +418,13 @@ fn parse_index_args() -> IndexCmdArgs {
"--index" | "-i" => index = args.next(),
"--file" | "-f" => file = args.next().map(PathBuf::from),
"--batch" | "-b" => {
- batch = args
- .next()
- .and_then(|s| s.parse().ok())
- .unwrap_or(5000);
+ batch = args.next().and_then(|s| s.parse().ok()).unwrap_or(5000);
}
"--workers" | "-w" => {
- workers = args
- .next()
- .and_then(|s| s.parse().ok())
- .unwrap_or(0);
+ workers = args.next().and_then(|s| s.parse().ok()).unwrap_or(0);
}
"--limit" | "-l" => {
- limit = args
- .next()
- .and_then(|s| s.parse().ok())
- .unwrap_or(0);
+ limit = args.next().and_then(|s| s.parse().ok()).unwrap_or(0);
}
"--config" | "-c" => config = args.next().map(PathBuf::from),
"--data-dir" | "-d" => data_dir = args.next(),
@@ -449,7 +448,15 @@ fn parse_index_args() -> IndexCmdArgs {
std::process::exit(2);
});
- IndexCmdArgs { index, file, batch, workers, limit, config, data_dir }
+ IndexCmdArgs {
+ index,
+ file,
+ batch,
+ workers,
+ limit,
+ config,
+ data_dir,
+ }
}
fn print_index_help() {
@@ -536,8 +543,8 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
// pre-M5.14 BufReader::lines() loop that was pinned to one thread
// at ~400 MB/s and became the hot-path bottleneck once xerj was
// doing 900k docs/s peak.
- let file = std::fs::File::open(&cmd.file)
- .with_context(|| format!("open {}", cmd.file.display()))?;
+ let file =
+ std::fs::File::open(&cmd.file).with_context(|| format!("open {}", cmd.file.display()))?;
let file_size = file.metadata().map(|m| m.len()).unwrap_or(0);
let mmap: memmap2::Mmap = unsafe { memmap2::Mmap::map(&file) }
.with_context(|| format!("mmap {}", cmd.file.display()))?;
@@ -629,10 +636,7 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
let scan_result = tokio::task::spawn_blocking(move || {
use rayon::prelude::*;
- let pairs: Vec<(usize, usize)> = boundaries
- .windows(2)
- .map(|w| (w[0], w[1]))
- .collect();
+ let pairs: Vec<(usize, usize)> = boundaries.windows(2).map(|w| (w[0], w[1])).collect();
let seen = std::sync::atomic::AtomicU64::new(0);
@@ -640,16 +644,12 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
.par_iter()
.enumerate()
.for_each(|(scanner_idx, &(start, end))| {
- if limit > 0
- && seen.load(std::sync::atomic::Ordering::Relaxed)
- >= limit as u64
- {
+ if limit > 0 && seen.load(std::sync::atomic::Ordering::Relaxed) >= limit as u64 {
return;
}
let chunk = &data[start..end];
let mut cursor = 0usize;
- let mut current: Vec<(String, StdArc<[u8]>)> =
- Vec::with_capacity(batch_size);
+ let mut current: Vec<(String, StdArc<[u8]>)> = Vec::with_capacity(batch_size);
// Synchronous submit with back-pressure retry. The engine
// returns ResourceExhausted only when memtable > 3× flush
@@ -668,10 +668,7 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
let b = current_batch.take().unwrap();
match idx_for_rayon.index_batch_sync_raw(b) {
Ok(_) => {
- sent_for_rayon.fetch_add(
- n,
- std::sync::atomic::Ordering::Relaxed,
- );
+ sent_for_rayon.fetch_add(n, std::sync::atomic::Ordering::Relaxed);
return;
}
Err(xerj_engine::EngineError::Common(
@@ -682,22 +679,16 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
// 50 ms; add a tiny thread::sleep to avoid
// a tight busy-loop on persistent back-
// pressure (flusher is clearly saturated).
- std::thread::sleep(
- std::time::Duration::from_millis(5),
- );
+ std::thread::sleep(std::time::Duration::from_millis(5));
let reclone: Vec<(String, StdArc<[u8]>)> = retry
.iter()
.map(|(id, b)| (id.clone(), StdArc::clone(b)))
.collect();
- current_batch =
- Some(std::mem::replace(&mut retry, reclone));
+ current_batch = Some(std::mem::replace(&mut retry, reclone));
}
Err(e) => {
error!("batch ingest error: {e}");
- errs_for_rayon.fetch_add(
- n,
- std::sync::atomic::Ordering::Relaxed,
- );
+ errs_for_rayon.fetch_add(n, std::sync::atomic::Ordering::Relaxed);
return;
}
}
@@ -705,8 +696,8 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
};
while cursor < chunk.len() {
- let nl_rel = memchr::memchr(b'\n', &chunk[cursor..])
- .unwrap_or(chunk.len() - cursor);
+ let nl_rel =
+ memchr::memchr(b'\n', &chunk[cursor..]).unwrap_or(chunk.len() - cursor);
let line_end = cursor + nl_rel;
let line = &chunk[cursor..line_end];
cursor = line_end + 1;
@@ -714,8 +705,7 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
continue;
}
- let seq =
- seen.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ let seq = seen.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
if limit > 0 && seq >= limit as u64 {
break;
}
@@ -725,10 +715,7 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
current.push((doc_id, bytes));
if current.len() >= batch_size {
- let batch = std::mem::replace(
- &mut current,
- Vec::with_capacity(batch_size),
- );
+ let batch = std::mem::replace(&mut current, Vec::with_capacity(batch_size));
submit(batch);
}
}
@@ -774,10 +761,16 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
println!(" docs sent : {}", sent);
println!(" errors : {}", errs);
println!(" ingest time : {:.2} s", ingest_elapsed.as_secs_f64());
- println!(" ingest rate : {:.0} docs/s (WAL-durable, in-memtable)", ingest_rate);
+ println!(
+ " ingest rate : {:.0} docs/s (WAL-durable, in-memtable)",
+ ingest_rate
+ );
println!(" final flush : {:.2} s", flush_elapsed.as_secs_f64());
println!(" total elapsed : {:.2} s", elapsed.as_secs_f64());
- println!(" total rate : {:.0} docs/s (fully segment-durable)", total_rate);
+ println!(
+ " total rate : {:.0} docs/s (fully segment-durable)",
+ total_rate
+ );
println!(" workers : {}", workers);
println!(" batch size : {}", batch_size);
println!("═══════════════════════════════════════════════════════════");
@@ -899,7 +892,9 @@ async fn async_main() -> Result<()> {
for peer_str in &cfg.cluster.peers {
if let Some((id, addr_str)) = peer_str.split_once('=') {
match addr_str.parse::<std::net::SocketAddr>() {
- Ok(addr) => { peers.insert(id.to_string(), addr); }
+ Ok(addr) => {
+ peers.insert(id.to_string(), addr);
+ }
Err(e) => warn!("cluster: ignoring invalid peer {peer_str}: {e}"),
}
} else {
@@ -908,16 +903,11 @@ async fn async_main() -> Result<()> {
}
// Derive this node's listen address from the server bind address + cluster port.
- let node_id = format!(
- "{}:{}",
- cfg.server.bind_address, cfg.cluster.port
- );
- let listen_addr: std::net::SocketAddr = format!(
- "{}:{}",
- cfg.server.bind_address, cfg.cluster.port
- )
- .parse()
- .context("parse cluster listen address")?;
+ let node_id = format!("{}:{}", cfg.server.bind_address, cfg.cluster.port);
+ let listen_addr: std::net::SocketAddr =
+ format!("{}:{}", cfg.server.bind_address, cfg.cluster.port)
+ .parse()
+ .context("parse cluster listen address")?;
let tick = std::time::Duration::from_millis(cfg.cluster.tick_ms);
@@ -998,10 +988,8 @@ async fn async_main() -> Result<()> {
// merge would leave xerj-api compiling and serving on its own
// — a property worth preserving.
let xerj_console_router = xerj_console_api::xerj_console_router(xerj_console_state);
- let native_router =
- build_native_router(state.clone()).merge(xerj_console_router.clone());
- let es_router =
- build_es_compat_router(state.clone()).merge(xerj_console_router);
+ let native_router = build_native_router(state.clone()).merge(xerj_console_router.clone());
+ let es_router = build_es_compat_router(state.clone()).merge(xerj_console_router);
// 10. Banner (includes total startup time)
let startup_ms = startup_start.elapsed().as_millis();
diff --git a/engine/crates/xerj-storage/src/cache.rs b/engine/crates/xerj-storage/src/cache.rs
index 4e62ea5fe..e3a72ac1c 100644
--- a/engine/crates/xerj-storage/src/cache.rs
+++ b/engine/crates/xerj-storage/src/cache.rs
@@ -49,7 +49,11 @@ impl SegmentCache {
) -> Self {
let cache_dir = cache_dir.into();
std::fs::create_dir_all(&cache_dir).ok();
- Self { cache_dir, max_size_bytes, backend }
+ Self {
+ cache_dir,
+ max_size_bytes,
+ backend,
+ }
}
/// Return the data for `path`.
@@ -191,7 +195,10 @@ mod tests {
let backend: Arc<dyn StorageBackend> =
Arc::new(LocalFsBackend::new(backend_dir.path()).unwrap());
- backend.write("segments/seg-001.seg", b"segment data here").await.unwrap();
+ backend
+ .write("segments/seg-001.seg", b"segment data here")
+ .await
+ .unwrap();
let cache = SegmentCache::new(cache_dir.path(), 100 * 1024 * 1024, Arc::clone(&backend));
@@ -212,8 +219,7 @@ mod tests {
Arc::new(LocalFsBackend::new(backend_dir.path()).unwrap());
backend.write("seg.seg", b"original").await.unwrap();
- let cache =
- SegmentCache::new(cache_dir.path(), 100 * 1024 * 1024, Arc::clone(&backend));
+ let cache = SegmentCache::new(cache_dir.path(), 100 * 1024 * 1024, Arc::clone(&backend));
// Populate cache.
cache.get("seg.seg").await.unwrap();
@@ -222,7 +228,11 @@ mod tests {
backend.write("seg.seg", b"modified").await.unwrap();
let data = cache.get("seg.seg").await.unwrap();
- assert_eq!(&data[..], b"original", "cache should serve stale-but-local copy");
+ assert_eq!(
+ &data[..],
+ b"original",
+ "cache should serve stale-but-local copy"
+ );
}
#[tokio::test]
@@ -249,7 +259,10 @@ mod tests {
cache.maybe_evict().await.unwrap();
let size_after = cache.cache_size_bytes().await.unwrap();
- assert!(size_after <= 15, "expected <= 15 bytes after eviction, got {size_after}");
+ assert!(
+ size_after <= 15,
+ "expected <= 15 bytes after eviction, got {size_after}"
+ );
}
#[tokio::test]
@@ -261,8 +274,7 @@ mod tests {
Arc::new(LocalFsBackend::new(backend_dir.path()).unwrap());
backend.write("inv.seg", b"data").await.unwrap();
- let cache =
- SegmentCache::new(cache_dir.path(), 100 * 1024 * 1024, Arc::clone(&backend));
+ let cache = SegmentCache::new(cache_dir.path(), 100 * 1024 * 1024, Arc::clone(&backend));
cache.get("inv.seg").await.unwrap();
assert!(cache_dir.path().join("inv.seg").exists());
@@ -281,10 +293,12 @@ mod tests {
let backend: Arc<dyn StorageBackend> =
Arc::new(S3Backend::new(s3_dir.path(), "test-bucket", "xerj/"));
- backend.write("segments/s3-seg.seg", b"s3 segment bytes").await.unwrap();
+ backend
+ .write("segments/s3-seg.seg", b"s3 segment bytes")
+ .await
+ .unwrap();
- let cache =
- SegmentCache::new(cache_dir.path(), 100 * 1024 * 1024, Arc::clone(&backend));
+ let cache = SegmentCache::new(cache_dir.path(), 100 * 1024 * 1024, Arc::clone(&backend));
// Cache miss path through simulated S3.
let data = cache.get("segments/s3-seg.seg").await.unwrap();
diff --git a/engine/crates/xerj-storage/src/doc_values.rs b/engine/crates/xerj-storage/src/doc_values.rs
index 4900e070c..d9eb9a2d6 100644
--- a/engine/crates/xerj-storage/src/doc_values.rs
+++ b/engine/crates/xerj-storage/src/doc_values.rs
@@ -113,6 +113,10 @@ pub struct NumericColumn {
}
impl NumericColumn {
+ // Named `from_iter` deliberately; it is not the FromIterator trait method
+ // (returns Self by value from an Option-yielding iterator). Renaming would
+ // break all callers, so the trait-confusion lint is allowed here.
+ #[allow(clippy::should_implement_trait)]
pub fn from_iter<I: IntoIterator<Item = Option<i64>>>(it: I) -> Self {
let mut null_bitmap = RoaringBitmap::new();
let mut data = Vec::new();
@@ -177,18 +181,15 @@ impl NumericColumn {
if lo_idx >= hi_idx {
return Vec::new();
}
- self.sorted[lo_idx..hi_idx].iter().map(|(_, d)| *d).collect()
+ self.sorted[lo_idx..hi_idx]
+ .iter()
+ .map(|(_, d)| *d)
+ .collect()
}
/// Count-only variant of `range_doc_ids` — returns just the number of
/// matching docs without allocating a Vec.
- pub fn range_count(
- &self,
- min: f64,
- max: f64,
- min_inclusive: bool,
- max_inclusive: bool,
- ) -> u64 {
+ pub fn range_count(&self, min: f64, max: f64, min_inclusive: bool, max_inclusive: bool) -> u64 {
if self.sorted.is_empty() {
return 0;
}
@@ -226,14 +227,13 @@ impl NumericColumn {
/// `doc_count: u32` whose first byte is 0-255 but never matches 'Z'
/// for a real doc count.
pub fn encode(&self) -> Vec<u8> {
- let mut out = Vec::with_capacity(
- 4 + 4 + 4 + self.data.len() * 8,
- );
+ let mut out = Vec::with_capacity(4 + 4 + 4 + self.data.len() * 8);
out.extend_from_slice(b"ZNV1");
out.write_u32::<LittleEndian>(self.doc_count).unwrap();
let mut bitmap_buf = Vec::new();
self.null_bitmap.serialize_into(&mut bitmap_buf).unwrap();
- out.write_u32::<LittleEndian>(bitmap_buf.len() as u32).unwrap();
+ out.write_u32::<LittleEndian>(bitmap_buf.len() as u32)
+ .unwrap();
out.extend_from_slice(&bitmap_buf);
for &v in &self.data {
out.write_i64::<LittleEndian>(v).unwrap();
@@ -246,7 +246,7 @@ impl NumericColumn {
let (mut cur, is_new) = if payload.len() >= 4 && &payload[..4] == b"ZNV1" {
(Cursor::new(&payload[4..]), true)
} else {
- (Cursor::new(&payload[..]), false)
+ (Cursor::new(payload), false)
};
let doc_count = cur.read_u32::<LittleEndian>().map_err(io_to_storage)?;
@@ -358,6 +358,10 @@ pub struct KeywordColumn {
}
impl KeywordColumn {
+ // Named `from_iter` deliberately; it is not the FromIterator trait method
+ // (it is fallible, returning Result<Self>). Renaming would break all
+ // callers, so the trait-confusion lint is allowed here.
+ #[allow(clippy::should_implement_trait)]
pub fn from_iter<I: IntoIterator<Item = Option<String>>>(it: I) -> Result<Self> {
let values: Vec<Option<String>> = it.into_iter().collect();
let doc_count = values.len() as u32;
@@ -371,10 +375,7 @@ impl KeywordColumn {
// `BTreeMap<String, ()>` build, which CLONED every non-null cell
// (hundreds of thousands of String allocs per 31k-doc flush
// segment) — only unique terms are cloned now.
- let mut refs: Vec<&str> = values
- .iter()
- .filter_map(|v| v.as_deref())
- .collect();
+ let mut refs: Vec<&str> = values.iter().filter_map(|v| v.as_deref()).collect();
refs.sort_unstable();
refs.dedup();
let terms: Vec<String> = refs.iter().map(|s| (*s).to_string()).collect();
@@ -452,10 +453,7 @@ impl KeywordColumn {
let Some(ord) = self.ord_for_term(term) else {
return 0;
};
- self.per_ord_count
- .get(ord as usize)
- .copied()
- .unwrap_or(0)
+ self.per_ord_count.get(ord as usize).copied().unwrap_or(0)
}
fn ord_width(num_terms: u32) -> u8 {
@@ -619,7 +617,8 @@ pub fn encode_columns(columns: &BTreeMap<String, Column>) -> Vec<u8> {
out.write_u8(kind | KIND_FLAG_ZSTD).unwrap();
out.write_u32::<LittleEndian>(name.len() as u32).unwrap();
out.extend_from_slice(name.as_bytes());
- out.write_u64::<LittleEndian>(zstd_payload.len() as u64).unwrap();
+ out.write_u64::<LittleEndian>(zstd_payload.len() as u64)
+ .unwrap();
out.extend_from_slice(&zstd_payload);
}
out
@@ -738,8 +737,10 @@ mod tests {
let back = KeywordColumn::decode(&bytes).unwrap();
for i in 0..5 {
assert_eq!(
- col.ord_for(i).map(|o| col.term_for_ord(o).unwrap().to_string()),
- back.ord_for(i).map(|o| back.term_for_ord(o).unwrap().to_string()),
+ col.ord_for(i)
+ .map(|o| col.term_for_ord(o).unwrap().to_string()),
+ back.ord_for(i)
+ .map(|o| back.term_for_ord(o).unwrap().to_string()),
);
}
}
@@ -749,7 +750,11 @@ mod tests {
let mut cols = BTreeMap::new();
cols.insert(
"status".to_string(),
- Column::Numeric(NumericColumn::from_iter(vec![Some(200), Some(404), Some(200)])),
+ Column::Numeric(NumericColumn::from_iter(vec![
+ Some(200),
+ Some(404),
+ Some(200),
+ ])),
);
cols.insert(
"method".to_string(),
diff --git a/engine/crates/xerj-storage/src/lib.rs b/engine/crates/xerj-storage/src/lib.rs
index 4efd9d98a..866098c1f 100644
--- a/engine/crates/xerj-storage/src/lib.rs
+++ b/engine/crates/xerj-storage/src/lib.rs
@@ -66,7 +66,10 @@ pub enum StorageError {
ChecksumMismatch { expected: u32, actual: u32 },
#[error("Invalid magic bytes: expected {expected:?}, got {actual:?}")]
- InvalidMagic { expected: &'static [u8], actual: Vec<u8> },
+ InvalidMagic {
+ expected: &'static [u8],
+ actual: Vec<u8>,
+ },
#[error("Unsupported format version {0}")]
UnsupportedVersion(u16),
@@ -78,7 +81,11 @@ pub enum StorageError {
WalCorrupt(u64, String),
#[error("Version conflict: doc {doc_id} expected seq {expected}, found {actual}")]
- VersionConflict { doc_id: String, expected: SeqNo, actual: SeqNo },
+ VersionConflict {
+ doc_id: String,
+ expected: SeqNo,
+ actual: SeqNo,
+ },
#[error("Backend error: {0}")]
Backend(String),
diff --git a/engine/crates/xerj-storage/src/merge.rs b/engine/crates/xerj-storage/src/merge.rs
index 961b8f762..ce4ed7d9d 100644
--- a/engine/crates/xerj-storage/src/merge.rs
+++ b/engine/crates/xerj-storage/src/merge.rs
@@ -73,7 +73,7 @@ impl Default for SizeTieredMergePolicy {
Self {
min_merge_count: 4,
max_merge_count: 10,
- tier_floor_bytes: 5 * 1024 * 1024, // 5 MiB
+ tier_floor_bytes: 5 * 1024 * 1024, // 5 MiB
max_merged_segment_bytes: 5 * 1024 * 1024 * 1024, // 5 GiB
}
}
@@ -81,7 +81,9 @@ impl Default for SizeTieredMergePolicy {
impl SizeTieredMergePolicy {
fn tier_for(&self, size_bytes: u64) -> u32 {
- if size_bytes == 0 { return 0; }
+ if size_bytes == 0 {
+ return 0;
+ }
let floor = self.tier_floor_bytes.max(1) as f64;
let ratio = (size_bytes as f64 / floor).log2().max(0.0);
ratio as u32
@@ -189,7 +191,10 @@ pub struct MergeConfig {
impl Default for MergeConfig {
fn default() -> Self {
- Self { io_rate_mb_per_sec: 50, check_interval_ms: 5_000 }
+ Self {
+ io_rate_mb_per_sec: 50,
+ check_interval_ms: 5_000,
+ }
}
}
@@ -218,7 +223,9 @@ impl MergeExecutor {
#[instrument(skip(self), fields(count = segment_ids.len()))]
pub fn execute_merge(&self, segment_ids: &[SegmentId]) -> Result<SegmentMeta> {
if segment_ids.len() < 2 {
- return Err(StorageError::MergeAborted("need at least 2 segments to merge".into()));
+ return Err(StorageError::MergeAborted(
+ "need at least 2 segments to merge".into(),
+ ));
}
let snap = self.store.snapshot();
@@ -237,7 +244,9 @@ impl MergeExecutor {
.iter()
.filter(|id| !metas.iter().any(|m| &m.id == *id))
.collect();
- return Err(StorageError::MergeAborted(format!("segments not found: {missing:?}")));
+ return Err(StorageError::MergeAborted(format!(
+ "segments not found: {missing:?}"
+ )));
}
let total_docs: u64 = metas.iter().map(|m| m.doc_count).sum();
@@ -312,17 +321,11 @@ impl MergeExecutor {
self.store.apply_merge(segment_ids, merged_meta.clone())?;
// Update version map for all merged docs · hoist the Arc once.
- let merged_id_arc: std::sync::Arc<str> =
- std::sync::Arc::from(merged_meta.id.as_str());
+ let merged_id_arc: std::sync::Arc<str> = std::sync::Arc::from(merged_meta.id.as_str());
for doc in &merged_docs {
if let Some(doc_id) = doc.get("_id").and_then(|v| v.as_str()) {
if let Some(seq_no) = doc.get("_seq_no").and_then(|v| v.as_u64()) {
- version_map.set(
- doc_id,
- seq_no,
- std::sync::Arc::clone(&merged_id_arc),
- false,
- );
+ version_map.set(doc_id, seq_no, std::sync::Arc::clone(&merged_id_arc), false);
}
}
}
@@ -341,10 +344,7 @@ impl MergeExecutor {
///
/// The thread runs until [`MergeExecutor::shutdown`] is called. Returns a
/// handle that can be joined on shutdown.
- pub fn spawn_background<P>(
- self: Arc<Self>,
- policy: Arc<P>,
- ) -> std::thread::JoinHandle<()>
+ pub fn spawn_background<P>(self: Arc<Self>, policy: Arc<P>) -> std::thread::JoinHandle<()>
where
P: MergePolicy,
{
@@ -358,7 +358,9 @@ impl MergeExecutor {
while !shutdown.load(Ordering::Relaxed) {
std::thread::sleep(interval);
- if shutdown.load(Ordering::Relaxed) { break; }
+ if shutdown.load(Ordering::Relaxed) {
+ break;
+ }
let snap = self.store.snapshot();
let candidates = policy.select_merges(&snap.segments);
@@ -370,7 +372,9 @@ impl MergeExecutor {
}
for batch in candidates {
- if shutdown.load(Ordering::Relaxed) { break; }
+ if shutdown.load(Ordering::Relaxed) {
+ break;
+ }
match self.execute_merge(&batch) {
Ok(meta) => info!(merged_id = meta.id, "background merge completed"),
Err(e) => warn!("background merge failed: {e}"),
@@ -397,10 +401,13 @@ mod tests {
use crate::wal::SyncMode;
fn test_store(dir: &std::path::Path) -> Arc<IndexStore> {
- IndexStore::open(dir, IndexStoreConfig {
- sync_mode: SyncMode::Batched,
- ..Default::default()
- })
+ IndexStore::open(
+ dir,
+ IndexStoreConfig {
+ sync_mode: SyncMode::Batched,
+ ..Default::default()
+ },
+ )
.unwrap()
}
@@ -417,11 +424,19 @@ mod tests {
assert_eq!(store.snapshot().segments.len(), 2);
- let ids: Vec<SegmentId> = store.snapshot().segments.iter().map(|s| s.id.clone()).collect();
+ let ids: Vec<SegmentId> = store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|s| s.id.clone())
+ .collect();
let executor = Arc::new(MergeExecutor::new(
Arc::clone(&store),
- MergeConfig { io_rate_mb_per_sec: 0, ..Default::default() },
+ MergeConfig {
+ io_rate_mb_per_sec: 0,
+ ..Default::default()
+ },
));
let meta = executor.execute_merge(&ids).unwrap();
@@ -445,11 +460,19 @@ mod tests {
store.index("doc-2", serde_json::json!({"v": 2})).unwrap();
store.flush().unwrap();
- let ids: Vec<SegmentId> = store.snapshot().segments.iter().map(|s| s.id.clone()).collect();
+ let ids: Vec<SegmentId> = store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|s| s.id.clone())
+ .collect();
let executor = Arc::new(MergeExecutor::new(
Arc::clone(&store),
- MergeConfig { io_rate_mb_per_sec: 0, ..Default::default() },
+ MergeConfig {
+ io_rate_mb_per_sec: 0,
+ ..Default::default()
+ },
));
let meta = executor.execute_merge(&ids).unwrap();
@@ -459,7 +482,11 @@ mod tests {
#[test]
fn size_tiered_policy_groups_correctly() {
- let policy = SizeTieredMergePolicy { min_merge_count: 2, max_merge_count: 10, ..Default::default() };
+ let policy = SizeTieredMergePolicy {
+ min_merge_count: 2,
+ max_merge_count: 10,
+ ..Default::default()
+ };
let make_meta = |id: &str, size: u64| SegmentMeta {
id: id.to_string(),
@@ -496,11 +523,22 @@ mod tests {
store.index("doc-1", serde_json::json!({})).unwrap();
store.flush().unwrap();
- let ids: Vec<SegmentId> = store.snapshot().segments.iter().map(|s| s.id.clone()).collect();
+ let ids: Vec<SegmentId> = store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|s| s.id.clone())
+ .collect();
let executor = Arc::new(MergeExecutor::new(
Arc::clone(&store),
- MergeConfig { io_rate_mb_per_sec: 0, ..Default::default() },
+ MergeConfig {
+ io_rate_mb_per_sec: 0,
+ ..Default::default()
+ },
+ ));
+ assert!(matches!(
+ executor.execute_merge(&ids),
+ Err(StorageError::MergeAborted(_))
));
- assert!(matches!(executor.execute_merge(&ids), Err(StorageError::MergeAborted(_))));
}
}
diff --git a/engine/crates/xerj-storage/src/segment.rs b/engine/crates/xerj-storage/src/segment.rs
index c2587781c..e6ee1f6b6 100644
--- a/engine/crates/xerj-storage/src/segment.rs
+++ b/engine/crates/xerj-storage/src/segment.rs
@@ -213,7 +213,12 @@ impl SectionEntry {
let length = r.read_u64::<LittleEndian>()?;
let crc32c = r.read_u32::<LittleEndian>()?;
let _reserved = r.read_u32::<LittleEndian>()?;
- Ok(Self { section_type: SectionType::from_u8(t), offset, length, crc32c })
+ Ok(Self {
+ section_type: SectionType::from_u8(t),
+ offset,
+ length,
+ crc32c,
+ })
}
}
@@ -296,7 +301,10 @@ impl SegmentWriter {
min_seq_no: SeqNo,
max_seq_no: SeqNo,
) -> Result<SegmentMeta> {
- let has_tombstones = self.sections.iter().any(|(t, _)| *t == SectionType::Tombstones);
+ let has_tombstones = self
+ .sections
+ .iter()
+ .any(|(t, _)| *t == SectionType::Tombstones);
let flags: u16 = if has_tombstones { 0x0001 } else { 0x0000 };
let header = SegmentHeader {
@@ -384,8 +392,11 @@ impl SegmentWriter {
{
let mut f = BufWriter::new(File::create(&sidx_tmp)?);
// Minimal skip index: one entry pointing to the stored section
- if let Some(stored) = section_entries.iter().find(|e| e.section_type == SectionType::Stored) {
- f.write_u64::<LittleEndian>(0)?; // first doc ordinal
+ if let Some(stored) = section_entries
+ .iter()
+ .find(|e| e.section_type == SectionType::Stored)
+ {
+ f.write_u64::<LittleEndian>(0)?; // first doc ordinal
f.write_u64::<LittleEndian>(stored.offset)?; // file offset
}
f.flush()?;
@@ -457,7 +468,11 @@ impl SegmentReader {
for _ in 0..section_count {
sections.push(SectionEntry::read_from(&mut st_cur)?);
}
- Ok(Self { mmap, header, sections })
+ Ok(Self {
+ mmap,
+ header,
+ sections,
+ })
}
fn from_mmap(mmap: Arc<Mmap>) -> Result<Self> {
@@ -504,11 +519,17 @@ impl SegmentReader {
sections.push(SectionEntry::read_from(&mut st_cur)?);
}
- Ok(Self { mmap, header, sections })
+ Ok(Self {
+ mmap,
+ header,
+ sections,
+ })
}
/// Return the segment header.
- pub fn header(&self) -> &SegmentHeader { &self.header }
+ pub fn header(&self) -> &SegmentHeader {
+ &self.header
+ }
/// Return the raw bytes of a section, or `None` if that section type is
/// not present in this segment.
@@ -547,7 +568,10 @@ impl SegmentReader {
hasher.update(slice);
let crc = hasher.finalize();
if crc != entry.crc32c {
- return Err(StorageError::ChecksumMismatch { expected: entry.crc32c, actual: crc });
+ return Err(StorageError::ChecksumMismatch {
+ expected: entry.crc32c,
+ actual: crc,
+ });
}
Ok(Some(slice))
}
@@ -570,8 +594,12 @@ mod tests {
let mut writer = SegmentWriter::new(dir.path(), 1, 0, 0).unwrap();
let stored_data = b"{\"title\":\"hello xerj\"}";
- writer.add_section(SectionType::Stored, stored_data).unwrap();
- writer.add_section(SectionType::Schema, b"{\"fields\":{}}").unwrap();
+ writer
+ .add_section(SectionType::Stored, stored_data)
+ .unwrap();
+ writer
+ .add_section(SectionType::Schema, b"{\"fields\":{}}")
+ .unwrap();
let meta = writer.finish(1, 10, 10).unwrap();
assert_eq!(meta.doc_count, 1);
@@ -597,7 +625,9 @@ mod tests {
fn segment_with_tombstones_sets_flag() {
let dir = tempfile::tempdir().unwrap();
let mut writer = SegmentWriter::new(dir.path(), 1, 0, 0).unwrap();
- writer.add_section(SectionType::Tombstones, &[0xFFu8; 4]).unwrap();
+ writer
+ .add_section(SectionType::Tombstones, [0xFFu8; 4])
+ .unwrap();
let meta = writer.finish(10, 1, 10).unwrap();
assert!(meta.has_tombstones);
diff --git a/engine/crates/xerj-storage/src/stored_codec.rs b/engine/crates/xerj-storage/src/stored_codec.rs
index f3c38aed8..49f8340db 100644
--- a/engine/crates/xerj-storage/src/stored_codec.rs
+++ b/engine/crates/xerj-storage/src/stored_codec.rs
@@ -55,7 +55,7 @@
use crate::{Result, StorageError};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use std::collections::HashMap;
-use std::io::{Cursor, Read, Write};
+use std::io::{Cursor, Write};
// ── V1: flat LZ4 over JSON (legacy) ───────────────────────────────────────
@@ -236,9 +236,7 @@ pub fn encode_stored_v2_from_values(
/// run that comparison was ~3 s of background CPU per 1M ingested docs).
/// On sub-`V2_MIN_DOCS` inputs the canonical JSON array is built here so
/// the v1 fallback output stays byte-identical to the legacy path.
-pub fn encode_stored_v2_from_values_nojson(
- docs: &[(&str, u64, &serde_json::Value)],
-) -> Vec<u8> {
+pub fn encode_stored_v2_from_values_nojson(docs: &[(&str, u64, &serde_json::Value)]) -> Vec<u8> {
encode_stored_v2_from_values_inner(None, docs)
}
@@ -301,10 +299,8 @@ fn encode_stored_v2_from_values_inner(
// field value straight to its column. The previous two-pass build
// (discover, then `obj.get(cname)` per column per doc) performed
// ~2× the map lookups — measured at ~115 ms per 31k-doc flush.
- let mut columns: Vec<Vec<&serde_json::Value>> = vec![
- Vec::with_capacity(num_docs),
- Vec::with_capacity(num_docs),
- ];
+ let mut columns: Vec<Vec<&serde_json::Value>> =
+ vec![Vec::with_capacity(num_docs), Vec::with_capacity(num_docs)];
for (i, (_, _, src)) in docs.iter().enumerate() {
columns[0].push(&ids[i]);
columns[1].push(&seqs[i]);
@@ -323,8 +319,7 @@ fn encode_stored_v2_from_values_inner(
// `col_order` owns an independent clone.
col_seen.insert(key.as_str(), cix);
col_order.push(key.clone());
- let mut fresh: Vec<&serde_json::Value> =
- Vec::with_capacity(num_docs);
+ let mut fresh: Vec<&serde_json::Value> = Vec::with_capacity(num_docs);
fresh.resize(i, &NULL); // rows before this doc lack the field
columns.push(fresh);
cix
@@ -391,7 +386,9 @@ fn encode_v2_columns(
// Only integer columns may use the lossy i64 cross-dep path;
// float columns fall through to the lossless dict / raw path so
// fractional values (e.g. `0.010127`) survive intact.
- if !col_is_all_integer(col) { return None; }
+ if !col_is_all_integer(col) {
+ return None;
+ }
// Provable upper bound: with a source of ≤S dict entries, the
// mode table yields at most one hit-value per source id, so
// misses ≥ distinct(target) − S and det ≤ 1 − (D − S)/N.
@@ -438,7 +435,8 @@ fn encode_v2_columns(
}
// Fallback: zstd over JSON-array of the column's values.
let col_json = serde_json::to_vec(col).unwrap_or_default();
- let zstd_payload = zstd::encode_all(Cursor::new(&col_json), STORED_ZSTD_LEVEL).unwrap_or_else(|_| col_json.clone());
+ let zstd_payload = zstd::encode_all(Cursor::new(&col_json), STORED_ZSTD_LEVEL)
+ .unwrap_or_else(|_| col_json.clone());
// Choose RAW_JSON vs LZ4_JSON by size.
let lz4_payload = lz4_flex::compress_prepend_size(&col_json);
if lz4_payload.len() + 1 < zstd_payload.len() {
@@ -449,10 +447,17 @@ fn encode_v2_columns(
}
// Assemble the V2 payload.
- let mut out: Vec<u8> = Vec::with_capacity(4 + 8 + col_payloads.iter().map(|(n, _, p)| 2 + n.len() + 5 + p.len()).sum::<usize>());
+ let mut out: Vec<u8> = Vec::with_capacity(
+ 4 + 8
+ + col_payloads
+ .iter()
+ .map(|(n, _, p)| 2 + n.len() + 5 + p.len())
+ .sum::<usize>(),
+ );
out.extend_from_slice(STORED_V2_MAGIC);
out.write_u32::<LittleEndian>(num_docs as u32).unwrap();
- out.write_u32::<LittleEndian>(col_payloads.len() as u32).unwrap();
+ out.write_u32::<LittleEndian>(col_payloads.len() as u32)
+ .unwrap();
for (name, codec_id, payload) in &col_payloads {
out.write_u16::<LittleEndian>(name.len() as u16).unwrap();
out.extend_from_slice(name.as_bytes());
@@ -501,10 +506,14 @@ pub fn decode_stored(bytes: &[u8]) -> Result<Vec<u8>> {
fn decode_stored_v2(body: &[u8]) -> Result<Vec<u8>> {
let mut cur = Cursor::new(body);
- let num_docs = cur.read_u32::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("v2 num_docs: {e}")))? as usize;
- let num_cols = cur.read_u32::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("v2 num_cols: {e}")))? as usize;
+ let num_docs = cur
+ .read_u32::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("v2 num_docs: {e}")))?
+ as usize;
+ let num_cols = cur
+ .read_u32::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("v2 num_cols: {e}")))?
+ as usize;
// First pass: decode each column into Vec<Value>. Store the name
// alongside so we can re-assemble the docs.
@@ -512,8 +521,10 @@ fn decode_stored_v2(body: &[u8]) -> Result<Vec<u8>> {
let mut col_data: Vec<Vec<serde_json::Value>> = Vec::with_capacity(num_cols);
for _ in 0..num_cols {
- let name_len = cur.read_u16::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("v2 name_len: {e}")))? as usize;
+ let name_len = cur
+ .read_u16::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("v2 name_len: {e}")))?
+ as usize;
let pos = cur.position() as usize;
if body.len() < pos + name_len {
return Err(StorageError::Other(anyhow::anyhow!("v2 truncated name")));
@@ -523,12 +534,15 @@ fn decode_stored_v2(body: &[u8]) -> Result<Vec<u8>> {
.to_string();
cur.set_position((pos + name_len) as u64);
- let codec_id = cur.read_u8()
+ let codec_id = cur
+ .read_u8()
.map_err(|e| StorageError::Other(anyhow::anyhow!("v2 codec_id: {e}")))?;
let codec = ColCodec::from_u8(codec_id)
.ok_or_else(|| StorageError::Other(anyhow::anyhow!("v2 unknown codec {}", codec_id)))?;
- let payload_len = cur.read_u32::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("v2 payload_len: {e}")))? as usize;
+ let payload_len = cur
+ .read_u32::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("v2 payload_len: {e}")))?
+ as usize;
let pos = cur.position() as usize;
if body.len() < pos + payload_len {
return Err(StorageError::Other(anyhow::anyhow!("v2 truncated payload")));
@@ -549,13 +563,22 @@ fn decode_stored_v2(body: &[u8]) -> Result<Vec<u8>> {
col_data.push(Vec::new());
// Stash the raw payload in col_data[i] as a single
// "deferred" Value. Second pass recognises and replaces.
- col_data.last_mut().unwrap().push(serde_json::Value::Object({
- let mut m = serde_json::Map::new();
- m.insert("__deferred_cross_dep__".into(), serde_json::Value::Array(
- payload.iter().map(|b| serde_json::Value::Number((*b).into())).collect(),
- ));
- m
- }));
+ col_data
+ .last_mut()
+ .unwrap()
+ .push(serde_json::Value::Object({
+ let mut m = serde_json::Map::new();
+ m.insert(
+ "__deferred_cross_dep__".into(),
+ serde_json::Value::Array(
+ payload
+ .iter()
+ .map(|b| serde_json::Value::Number((*b).into()))
+ .collect(),
+ ),
+ );
+ m
+ }));
continue;
}
};
@@ -578,7 +601,8 @@ fn decode_stored_v2(body: &[u8]) -> Result<Vec<u8>> {
.iter()
.filter_map(|v| v.as_u64().map(|u| u as u8))
.collect();
- let resolved = decode_cross_dep(&payload, num_docs, &col_data, &col_name_to_ix)?;
+ let resolved =
+ decode_cross_dep(&payload, num_docs, &col_data, &col_name_to_ix)?;
col_data[cix] = resolved;
continue;
}
@@ -594,15 +618,32 @@ fn decode_stored_v2(body: &[u8]) -> Result<Vec<u8>> {
for d in 0..num_docs {
let mut source_map = serde_json::Map::new();
for (cix, name) in col_names.iter().enumerate() {
- if cix == id_col_ix || cix == seq_col_ix { continue; }
- let v = col_data[cix].get(d).cloned().unwrap_or(serde_json::Value::Null);
+ if cix == id_col_ix || cix == seq_col_ix {
+ continue;
+ }
+ let v = col_data[cix]
+ .get(d)
+ .cloned()
+ .unwrap_or(serde_json::Value::Null);
if !v.is_null() {
source_map.insert(name.clone(), v);
}
}
let mut doc = serde_json::Map::new();
- doc.insert("_id".into(), col_data[id_col_ix].get(d).cloned().unwrap_or(serde_json::Value::Null));
- doc.insert("_seq_no".into(), col_data[seq_col_ix].get(d).cloned().unwrap_or(serde_json::Value::Null));
+ doc.insert(
+ "_id".into(),
+ col_data[id_col_ix]
+ .get(d)
+ .cloned()
+ .unwrap_or(serde_json::Value::Null),
+ );
+ doc.insert(
+ "_seq_no".into(),
+ col_data[seq_col_ix]
+ .get(d)
+ .cloned()
+ .unwrap_or(serde_json::Value::Null),
+ );
doc.insert("_source".into(), serde_json::Value::Object(source_map));
out_docs.push(serde_json::Value::Object(doc));
}
@@ -627,18 +668,24 @@ fn col_is_all_integer<B: std::borrow::Borrow<serde_json::Value>>(col: &[B]) -> b
let mut saw_num = false;
for v in col {
let v = v.borrow();
- if v.is_null() { continue; }
+ if v.is_null() {
+ continue;
+ }
// Only true JSON integers qualify; floats (even integer-valued
// ones like `10.0`) are left for the lossless path so their exact
// representation round-trips.
- if v.as_i64().is_none() && v.as_u64().is_none() { return false; }
+ if v.as_i64().is_none() && v.as_u64().is_none() {
+ return false;
+ }
saw_num = true;
}
saw_num
}
fn all_scalar_dict_entries(entries: &[serde_json::Value]) -> bool {
- entries.iter().all(|v| v.is_null() || v.is_string() || v.is_number() || v.is_boolean())
+ entries
+ .iter()
+ .all(|v| v.is_null() || v.is_string() || v.is_number() || v.is_boolean())
}
/// Build a dictionary representation `(entries, ids)` where:
@@ -702,7 +749,7 @@ fn dict_encode_column<'a, B: std::borrow::Borrow<serde_json::Value>>(
continue;
}
non_null += 1;
- let Some(k) = dict_key(v) else { return None }; // object / array
+ let k = dict_key(v)?; // object / array
if let Some(&id) = map.get(&k) {
ids.push(id);
} else {
@@ -727,7 +774,9 @@ fn dict_encode_column<'a, B: std::borrow::Borrow<serde_json::Value>>(
// Replace u32::MAX with the reserved-null id.
let null_id = entries.len() as u32;
for id in ids.iter_mut() {
- if *id == u32::MAX { *id = null_id; }
+ if *id == u32::MAX {
+ *id = null_id;
+ }
}
Some((entries, ids))
}
@@ -741,11 +790,22 @@ fn best_cross_dep_source<B: std::borrow::Borrow<serde_json::Value>>(
target_col: &[B],
) -> Option<usize> {
for (src_ix, de) in dict_encoded.iter().enumerate() {
- if src_ix == target_ix { continue; }
- let (entries, ids) = match de { Some(e) => e, None => continue };
- if !all_scalar_dict_entries(entries) { continue; }
- if entries.len() < 2 { continue; } // constant source: useless
- if entries.len() > CROSS_DEP_MAX_SRC_CARDINALITY { continue; }
+ if src_ix == target_ix {
+ continue;
+ }
+ let (entries, ids) = match de {
+ Some(e) => e,
+ None => continue,
+ };
+ if !all_scalar_dict_entries(entries) {
+ continue;
+ }
+ if entries.len() < 2 {
+ continue;
+ } // constant source: useless
+ if entries.len() > CROSS_DEP_MAX_SRC_CARDINALITY {
+ continue;
+ }
// Sampled prefilter: tally only the first `SAMPLE` rows first.
// Cross-dep is a codec CHOICE — a false negative here just means
@@ -789,7 +849,9 @@ fn cross_dep_determinism<B: std::borrow::Borrow<serde_json::Value>>(
let mut total_numeric = 0usize;
for (row, t_val) in target_col.iter().enumerate() {
let t_val = t_val.borrow();
- let Some(t) = t_val.as_i64().or_else(|| t_val.as_f64().map(|f| f as i64)) else { continue };
+ let Some(t) = t_val.as_i64().or_else(|| t_val.as_f64().map(|f| f as i64)) else {
+ continue;
+ };
total_numeric += 1;
let sid = ids[row];
*mode_tally.entry(sid).or_default().entry(t).or_insert(0) += 1;
@@ -806,11 +868,13 @@ fn cross_dep_determinism<B: std::borrow::Borrow<serde_json::Value>>(
fn try_encode_constant<B: std::borrow::Borrow<serde_json::Value>>(col: &[B]) -> Option<Vec<u8>> {
let first = col.iter().map(|v| v.borrow()).find(|v| !v.is_null())?;
- if col.iter().all(|v| { let v = v.borrow(); v == first || v.is_null() })
- && !col.iter().any(|v| v.borrow().is_null())
+ if col.iter().all(|v| {
+ let v = v.borrow();
+ v == first || v.is_null()
+ }) && !col.iter().any(|v| v.borrow().is_null())
{
// Only encode as constant when there are no nulls (keep the codec simple).
- return Some(serde_json::to_vec(first).ok()?);
+ return serde_json::to_vec(first).ok();
}
None
}
@@ -827,7 +891,11 @@ fn encode_dict_bitpack(entries: &[serde_json::Value], ids: &[u32]) -> Vec<u8> {
// packed range, so we allocate `bit_width` large enough for dict_count
// inclusive).
let max_id = dict_count as u32;
- let bit_width = if max_id == 0 { 1 } else { 32 - max_id.leading_zeros() as u8 };
+ let bit_width = if max_id == 0 {
+ 1
+ } else {
+ 32 - max_id.leading_zeros() as u8
+ };
let mut out = Vec::new();
out.write_u32::<LittleEndian>(dict_count as u32).unwrap();
@@ -835,33 +903,43 @@ fn encode_dict_bitpack(entries: &[serde_json::Value], ids: &[u32]) -> Vec<u8> {
// Dict entries as zstd(json array).
let dict_json = serde_json::to_vec(entries).unwrap_or_default();
- let dict_zstd = zstd::encode_all(Cursor::new(&dict_json), STORED_ZSTD_LEVEL).unwrap_or(dict_json.clone());
- out.write_u32::<LittleEndian>(dict_zstd.len() as u32).unwrap();
+ let dict_zstd =
+ zstd::encode_all(Cursor::new(&dict_json), STORED_ZSTD_LEVEL).unwrap_or(dict_json.clone());
+ out.write_u32::<LittleEndian>(dict_zstd.len() as u32)
+ .unwrap();
out.extend_from_slice(&dict_zstd);
// Bit-packed ids.
let packed = bitpack_u32(ids, bit_width);
// zstd over the bit-packed stream — gives another 20-40 % on log
// data because repeated ids cluster.
- let packed_zstd = zstd::encode_all(Cursor::new(&packed), STORED_ZSTD_LEVEL).unwrap_or(packed.clone());
+ let packed_zstd =
+ zstd::encode_all(Cursor::new(&packed), STORED_ZSTD_LEVEL).unwrap_or(packed.clone());
out.write_u32::<LittleEndian>(ids.len() as u32).unwrap();
- out.write_u32::<LittleEndian>(packed_zstd.len() as u32).unwrap();
+ out.write_u32::<LittleEndian>(packed_zstd.len() as u32)
+ .unwrap();
out.extend_from_slice(&packed_zstd);
out
}
fn decode_dict_bitpack(payload: &[u8], num_docs: usize) -> Result<Vec<serde_json::Value>> {
let mut cur = Cursor::new(payload);
- let dict_count = cur.read_u32::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("dict_count: {e}")))? as usize;
- let bit_width = cur.read_u8()
+ let dict_count =
+ cur.read_u32::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("dict_count: {e}")))? as usize;
+ let bit_width = cur
+ .read_u8()
.map_err(|e| StorageError::Other(anyhow::anyhow!("bit_width: {e}")))?;
- let dict_zstd_len = cur.read_u32::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("dict_zstd_len: {e}")))? as usize;
+ let dict_zstd_len = cur
+ .read_u32::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("dict_zstd_len: {e}")))?
+ as usize;
let pos = cur.position() as usize;
if payload.len() < pos + dict_zstd_len {
- return Err(StorageError::Other(anyhow::anyhow!("dict bitpack truncated")));
+ return Err(StorageError::Other(anyhow::anyhow!(
+ "dict bitpack truncated"
+ )));
}
let dict_json = zstd::decode_all(&payload[pos..pos + dict_zstd_len])
.map_err(|e| StorageError::Other(anyhow::anyhow!("dict zstd decode: {e}")))?;
@@ -869,18 +947,25 @@ fn decode_dict_bitpack(payload: &[u8], num_docs: usize) -> Result<Vec<serde_json
.map_err(|e| StorageError::Other(anyhow::anyhow!("dict json decode: {e}")))?;
cur.set_position((pos + dict_zstd_len) as u64);
- let ids_len = cur.read_u32::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("ids_len: {e}")))? as usize;
+ let ids_len =
+ cur.read_u32::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("ids_len: {e}")))? as usize;
if ids_len != num_docs {
return Err(StorageError::Other(anyhow::anyhow!(
- "dict bitpack ids_len {} != num_docs {}", ids_len, num_docs
+ "dict bitpack ids_len {} != num_docs {}",
+ ids_len,
+ num_docs
)));
}
- let packed_zstd_len = cur.read_u32::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("packed_zstd_len: {e}")))? as usize;
+ let packed_zstd_len = cur
+ .read_u32::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("packed_zstd_len: {e}")))?
+ as usize;
let pos = cur.position() as usize;
if payload.len() < pos + packed_zstd_len {
- return Err(StorageError::Other(anyhow::anyhow!("dict bitpack packed truncated")));
+ return Err(StorageError::Other(anyhow::anyhow!(
+ "dict bitpack packed truncated"
+ )));
}
let packed = zstd::decode_all(&payload[pos..pos + packed_zstd_len])
.map_err(|e| StorageError::Other(anyhow::anyhow!("packed zstd decode: {e}")))?;
@@ -890,8 +975,14 @@ fn decode_dict_bitpack(payload: &[u8], num_docs: usize) -> Result<Vec<serde_json
let values: Vec<serde_json::Value> = ids
.into_iter()
.map(|id| {
- if id == null_id { serde_json::Value::Null }
- else { entries.get(id as usize).cloned().unwrap_or(serde_json::Value::Null) }
+ if id == null_id {
+ serde_json::Value::Null
+ } else {
+ entries
+ .get(id as usize)
+ .cloned()
+ .unwrap_or(serde_json::Value::Null)
+ }
})
.collect();
Ok(values)
@@ -917,7 +1008,9 @@ fn encode_cross_dep<B: std::borrow::Borrow<serde_json::Value>>(
let mut tally: Vec<HashMap<i64, u32>> = vec![HashMap::new(); dict_count + 1];
for (row, t) in target_col.iter().enumerate() {
let t = t.borrow();
- let Some(tv) = t.as_i64().or_else(|| t.as_f64().map(|f| f as i64)) else { continue };
+ let Some(tv) = t.as_i64().or_else(|| t.as_f64().map(|f| f as i64)) else {
+ continue;
+ };
let sid = src_ids[row] as usize;
*tally[sid.min(dict_count)].entry(tv).or_insert(0) += 1;
}
@@ -939,7 +1032,11 @@ fn encode_cross_dep<B: std::borrow::Borrow<serde_json::Value>>(
continue;
};
let sid = src_ids[row] as usize;
- let expected = if sid < dict_count { mode_values[sid] } else { i64::MIN };
+ let expected = if sid < dict_count {
+ mode_values[sid]
+ } else {
+ i64::MIN
+ };
if expected != tv {
exceptions.push((row as u32, tv));
}
@@ -949,8 +1046,11 @@ fn encode_cross_dep<B: std::borrow::Borrow<serde_json::Value>>(
let mut out = Vec::new();
out.write_u32::<LittleEndian>(src_ix as u32).unwrap();
out.write_u32::<LittleEndian>(dict_count as u32).unwrap();
- for &v in &mode_values { out.write_i64::<LittleEndian>(v).unwrap(); }
- out.write_u32::<LittleEndian>(exceptions.len() as u32).unwrap();
+ for &v in &mode_values {
+ out.write_i64::<LittleEndian>(v).unwrap();
+ }
+ out.write_u32::<LittleEndian>(exceptions.len() as u32)
+ .unwrap();
// delta-encode doc_ords
let mut prev_ord = 0u32;
for (ord, val) in &exceptions {
@@ -990,27 +1090,38 @@ fn decode_cross_dep(
payload[1..].to_vec()
};
let mut cur = Cursor::new(&body[..]);
- let src_ix = cur.read_u32::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("cross_dep src_ix: {e}")))? as usize;
+ let src_ix = cur
+ .read_u32::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("cross_dep src_ix: {e}")))?
+ as usize;
let _ = col_name_to_ix; // not strictly needed yet
- let dict_count = cur.read_u32::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("cross_dep dict_count: {e}")))? as usize;
+ let dict_count = cur
+ .read_u32::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("cross_dep dict_count: {e}")))?
+ as usize;
let mut mode_values: Vec<i64> = Vec::with_capacity(dict_count);
for _ in 0..dict_count {
- mode_values.push(cur.read_i64::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("cross_dep mode: {e}")))?);
+ mode_values.push(
+ cur.read_i64::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("cross_dep mode: {e}")))?,
+ );
}
- let exc_count = cur.read_u32::<LittleEndian>()
- .map_err(|e| StorageError::Other(anyhow::anyhow!("cross_dep exc_count: {e}")))? as usize;
+ let exc_count = cur
+ .read_u32::<LittleEndian>()
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("cross_dep exc_count: {e}")))?
+ as usize;
// Rebuild the source column's dict ids by re-running `dict_encode_column`
// on the already-decoded source column.
- let src_col = col_data.get(src_ix)
+ let src_col = col_data
+ .get(src_ix)
.ok_or_else(|| StorageError::Other(anyhow::anyhow!("cross_dep src missing")))?;
if src_col.len() != num_docs {
// Source column was itself a CROSS_DEP and not yet resolved. Caller
// should have resolved RHS first. Bail with a clear error.
- return Err(StorageError::Other(anyhow::anyhow!("cross_dep src not resolved")));
+ return Err(StorageError::Other(anyhow::anyhow!(
+ "cross_dep src not resolved"
+ )));
}
let (_src_entries, src_ids) = dict_encode_column(src_col)
.ok_or_else(|| StorageError::Other(anyhow::anyhow!("cross_dep re-dict src failed")))?;
@@ -1042,7 +1153,11 @@ fn decode_cross_dep(
continue;
}
let sid = src_ids.get(row).copied().unwrap_or(0) as usize;
- let v = if sid < dict_count { mode_values[sid] } else { i64::MIN };
+ let v = if sid < dict_count {
+ mode_values[sid]
+ } else {
+ i64::MIN
+ };
if v == i64::MIN {
result.push(serde_json::Value::Null);
} else {
@@ -1074,7 +1189,7 @@ fn bitpack_u32(ids: &[u32], bit_width: u8) -> Vec<u8> {
}
let bw = bit_width as usize;
let total_bits = ids.len() * bw;
- let total_bytes = (total_bits + 7) / 8;
+ let total_bytes = total_bits.div_ceil(8);
let mut out = vec![0u8; total_bytes];
let mut bit_pos = 0usize;
for &id in ids {
@@ -1083,7 +1198,7 @@ fn bitpack_u32(ids: &[u32], bit_width: u8) -> Vec<u8> {
let shift = bit_pos % 8;
let combined = val << shift;
// Write up to 64 bits starting at byte_ix.
- let n_bytes = ((bw + shift + 7) / 8).min(total_bytes - byte_ix);
+ let n_bytes = (bw + shift).div_ceil(8).min(total_bytes - byte_ix);
for i in 0..n_bytes {
out[byte_ix + i] |= ((combined >> (i * 8)) & 0xFF) as u8;
}
@@ -1104,7 +1219,7 @@ fn bitunpack_u32(packed: &[u8], bit_width: u8, count: usize) -> Vec<u32> {
let byte_ix = bit_pos / 8;
let shift = bit_pos % 8;
let mut combined: u64 = 0;
- let n_bytes = ((bw + shift + 7) / 8).min(packed.len() - byte_ix);
+ let n_bytes = (bw + shift).div_ceil(8).min(packed.len() - byte_ix);
for j in 0..n_bytes {
combined |= (packed[byte_ix + j] as u64) << (j * 8);
}
@@ -1120,7 +1235,10 @@ fn write_varint(out: &mut Vec<u8>, mut v: u64) {
loop {
let b = (v & 0x7F) as u8;
v >>= 7;
- if v == 0 { out.push(b); break; }
+ if v == 0 {
+ out.push(b);
+ break;
+ }
out.push(b | 0x80);
}
}
@@ -1132,7 +1250,9 @@ fn read_varint(data: &[u8], pos: &mut usize) -> u64 {
let b = data[*pos];
*pos += 1;
v |= ((b & 0x7F) as u64) << shift;
- if b & 0x80 == 0 { return v; }
+ if b & 0x80 == 0 {
+ return v;
+ }
shift += 7;
}
v
@@ -1197,8 +1317,12 @@ mod tests {
let raw = serde_json::to_vec(&docs).unwrap();
let encoded = encode_stored_v2(&raw);
// V2 should have meaningfully reduced the payload.
- assert!(encoded.len() < raw.len() / 2,
- "v2 encoded {} not < half of raw {}", encoded.len(), raw.len());
+ assert!(
+ encoded.len() < raw.len() / 2,
+ "v2 encoded {} not < half of raw {}",
+ encoded.len(),
+ raw.len()
+ );
let decoded = decode_stored(&encoded).unwrap();
let round: Vec<serde_json::Value> = serde_json::from_slice(&decoded).unwrap();
@@ -1206,12 +1330,18 @@ mod tests {
// Spot-check a few docs.
for i in [0, 1, 5, 42, 128, 255] {
assert_eq!(round[i]["_id"], docs[i]["_id"], "id mismatch at {i}");
- assert_eq!(round[i]["_source"]["status"], docs[i]["_source"]["status"],
- "status mismatch at {i}");
- assert_eq!(round[i]["_source"]["path"], docs[i]["_source"]["path"],
- "path mismatch at {i}");
- assert_eq!(round[i]["_source"]["bytes"], docs[i]["_source"]["bytes"],
- "bytes mismatch at {i}");
+ assert_eq!(
+ round[i]["_source"]["status"], docs[i]["_source"]["status"],
+ "status mismatch at {i}"
+ );
+ assert_eq!(
+ round[i]["_source"]["path"], docs[i]["_source"]["path"],
+ "path mismatch at {i}"
+ );
+ assert_eq!(
+ round[i]["_source"]["bytes"], docs[i]["_source"]["bytes"],
+ "bytes mismatch at {i}"
+ );
}
}
@@ -1255,15 +1385,22 @@ mod tests {
let round: Vec<serde_json::Value> = serde_json::from_slice(&decoded).unwrap();
assert_eq!(round.len(), docs.len());
- for i in 0..docs.len() {
+ for (i, doc) in round.iter().enumerate() {
let c = i % categories.len();
// Exact f64 fidelity — no truncation to integer.
- let got = round[i]["_source"]["cost_usd"].as_f64().unwrap();
+ let got = doc["_source"]["cost_usd"].as_f64().unwrap();
assert_eq!(got, costs[c], "cost_usd corrupted at row {i}");
// The integer sibling stays a genuine integer (not promoted to float).
- let cnt = &round[i]["_source"]["count"];
- assert!(cnt.is_i64() || cnt.is_u64(), "count became non-integer at row {i}: {cnt}");
- assert_eq!(cnt.as_i64().unwrap(), counts[c], "count value wrong at row {i}");
+ let cnt = &doc["_source"]["count"];
+ assert!(
+ cnt.is_i64() || cnt.is_u64(),
+ "count became non-integer at row {i}: {cnt}"
+ );
+ assert_eq!(
+ cnt.as_i64().unwrap(),
+ counts[c],
+ "count value wrong at row {i}"
+ );
}
}
@@ -1372,11 +1509,15 @@ mod tests {
});
if i >= 700 {
// late-discovered column → backfill path
- src.as_object_mut().unwrap().insert("late".into(), json!(i % 9));
+ src.as_object_mut()
+ .unwrap()
+ .insert("late".into(), json!(i % 9));
}
if i % 13 == 0 {
// reserved-name source field → must be ignored
- src.as_object_mut().unwrap().insert("__id".into(), json!("evil"));
+ src.as_object_mut()
+ .unwrap()
+ .insert("__id".into(), json!("evil"));
}
docs.push((format!("id-{}", i), 5_000 + i as u64, src));
}
@@ -1388,7 +1529,10 @@ mod tests {
let legacy = encode_stored_v2(&stored);
let from_values = encode_stored_v2_from_values(&stored, &refs);
- assert_eq!(legacy, from_values, "late-column build diverged from legacy");
+ assert_eq!(
+ legacy, from_values,
+ "late-column build diverged from legacy"
+ );
// And the no-JSON fast path must produce the same v2 bytes when
// v2 wins the size net (it does on this shape).
@@ -1398,7 +1542,10 @@ mod tests {
let decoded = decode_stored(&from_values).unwrap();
let round: Vec<serde_json::Value> = serde_json::from_slice(&decoded).unwrap();
assert_eq!(round.len(), docs.len());
- assert!(round[0]["_source"].get("late").map(|v| v.is_null()).unwrap_or(true));
+ assert!(round[0]["_source"]
+ .get("late")
+ .map(|v| v.is_null())
+ .unwrap_or(true));
assert_eq!(round[0]["_id"].as_str().unwrap(), "id-0");
assert_eq!(round[750]["_source"]["late"], json!(750 % 9));
}
diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rs
index 10185384a..709dddb38 100644
--- a/engine/crates/xerj-storage/src/version_map.rs
+++ b/engine/crates/xerj-storage/src/version_map.rs
@@ -37,8 +37,8 @@ use std::sync::Arc;
use dashmap::DashMap;
use tracing::debug;
-use crate::{Result, SeqNo, StorageError};
use crate::segment::SegmentId;
+use crate::{Result, SeqNo, StorageError};
/// Sentinel segment ID used for documents that are in the in-memory write buffer
/// (not yet flushed to any segment file).
@@ -108,7 +108,11 @@ pub struct VersionMap {
impl VersionMap {
/// Create an empty version map.
pub fn new() -> Self {
- Self { inner: DashMap::new(), live: AtomicI64::new(0), ghost_events: std::sync::atomic::AtomicU64::new(0) }
+ Self {
+ inner: DashMap::new(),
+ live: AtomicI64::new(0),
+ ghost_events: std::sync::atomic::AtomicU64::new(0),
+ }
}
/// Apply a live-count delta for an entry transition.
@@ -144,20 +148,25 @@ impl VersionMap {
) {
let doc_id = doc_id.into();
debug!(?doc_id, seq_no, "version_map::set");
- let old = self
- .inner
- .insert(doc_id, VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
+ let old = self.inner.insert(
+ doc_id,
+ VersionEntry {
+ seq_no,
+ segment_id: segment_id.into(),
+ deleted,
+ },
+ );
// Overwrite or delete — a superseded/tombstoned physical copy now
// exists somewhere until merged away. A same-seq_no re-set is NOT
// an overwrite: the flush path repoints every drained doc's entry
// from `__memtable__` to its segment id with the seq_no unchanged
// (live-verified: counting those flipped the gate ON for every
// flushed doc and re-disabled all fast paths under pure appends).
- let is_overwrite = old.as_ref().map_or(false, |e| e.seq_no != seq_no);
+ let is_overwrite = old.as_ref().is_some_and(|e| e.seq_no != seq_no);
if is_overwrite || deleted {
self.ghost_events.fetch_add(1, Ordering::Relaxed);
}
- self.live_delta(old.map_or(false, |e| !e.deleted), !deleted);
+ self.live_delta(old.is_some_and(|e| !e.deleted), !deleted);
}
/// Record `doc_id → (seq_no, segment_id)` only if `seq_no` is >= the
@@ -187,11 +196,19 @@ impl VersionMap {
match self.inner.entry(doc_id) {
Entry::Occupied(mut occ) => {
if seq_no >= occ.get().seq_no {
- occ.insert(VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
+ occ.insert(VersionEntry {
+ seq_no,
+ segment_id: segment_id.into(),
+ deleted,
+ });
}
}
Entry::Vacant(vac) => {
- vac.insert(VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
+ vac.insert(VersionEntry {
+ seq_no,
+ segment_id: segment_id.into(),
+ deleted,
+ });
}
}
}
@@ -243,7 +260,11 @@ impl VersionMap {
}
}
let was_live = !occ.get().deleted;
- occ.insert(VersionEntry { seq_no: new_seq_no, segment_id: new_segment_id, deleted });
+ occ.insert(VersionEntry {
+ seq_no: new_seq_no,
+ segment_id: new_segment_id,
+ deleted,
+ });
self.ghost_events.fetch_add(1, Ordering::Relaxed);
self.live_delta(was_live, !deleted);
Ok(new_seq_no)
@@ -257,7 +278,11 @@ impl VersionMap {
actual: 0,
});
}
- vac.insert(VersionEntry { seq_no: new_seq_no, segment_id: new_segment_id, deleted });
+ vac.insert(VersionEntry {
+ seq_no: new_seq_no,
+ segment_id: new_segment_id,
+ deleted,
+ });
if deleted {
self.ghost_events.fetch_add(1, Ordering::Relaxed);
}
@@ -316,7 +341,10 @@ impl VersionMap {
if removed_live != 0 {
self.live.fetch_sub(removed_live, Ordering::Relaxed);
}
- debug!(stale_count = stale.len(), "version_map: cleaned up stale segments");
+ debug!(
+ stale_count = stale.len(),
+ "version_map: cleaned up stale segments"
+ );
}
/// Bulk-load entries from a segment during recovery / replay.
@@ -333,9 +361,13 @@ impl VersionMap {
if should_insert {
let old = self.inner.insert(
doc_id,
- VersionEntry { seq_no, segment_id: Arc::from(segment_id), deleted },
+ VersionEntry {
+ seq_no,
+ segment_id: Arc::from(segment_id),
+ deleted,
+ },
);
- self.live_delta(old.map_or(false, |e| !e.deleted), !deleted);
+ self.live_delta(old.is_some_and(|e| !e.deleted), !deleted);
}
}
}
@@ -377,7 +409,9 @@ impl VersionMap {
}
impl Default for VersionMap {
- fn default() -> Self { Self::new() }
+ fn default() -> Self {
+ Self::new()
+ }
}
// ── Tests ─────────────────────────────────────────────────────────────────────
@@ -400,7 +434,9 @@ mod tests {
fn check_and_set_success() {
let vm = VersionMap::new();
vm.set("doc-1", 5, "seg-a", false);
- let new_seq = vm.check_and_set("doc-1", Some(5), 10, "seg-b", false).unwrap();
+ let new_seq = vm
+ .check_and_set("doc-1", Some(5), 10, "seg-b", false)
+ .unwrap();
assert_eq!(new_seq, 10);
assert_eq!(vm.get("doc-1").unwrap().seq_no, 10);
}
@@ -410,7 +446,14 @@ mod tests {
let vm = VersionMap::new();
vm.set("doc-1", 5, "seg-a", false);
let result = vm.check_and_set("doc-1", Some(3), 10, "seg-b", false);
- assert!(matches!(result, Err(StorageError::VersionConflict { actual: 5, expected: 3, .. })));
+ assert!(matches!(
+ result,
+ Err(StorageError::VersionConflict {
+ actual: 5,
+ expected: 3,
+ ..
+ })
+ ));
}
#[test]
@@ -491,7 +534,9 @@ mod tests {
vm.set(format!("doc-{i}"), i, "seg-x", false);
}));
}
- for h in handles { h.join().unwrap(); }
+ for h in handles {
+ h.join().unwrap();
+ }
assert_eq!(vm.len(), 8);
}
}
diff --git a/engine/crates/xerj-vector/src/hnsw.rs b/engine/crates/xerj-vector/src/hnsw.rs
index 2a77b5b28..365a03ec3 100644
--- a/engine/crates/xerj-vector/src/hnsw.rs
+++ b/engine/crates/xerj-vector/src/hnsw.rs
@@ -9,15 +9,14 @@
//! - Level selection uses the standard ln-based formula
//! - Graph is entirely in-memory; persistence is handled by the storage crate
-use std::collections::{BinaryHeap, HashMap, HashSet};
use std::cmp::Reverse;
+use std::collections::{BinaryHeap, HashMap, HashSet};
use std::io::{Read, Write};
use std::path::Path;
use std::sync::{Arc, RwLock};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use rand::Rng;
-use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use tracing::{debug, trace};
use xerj_common::XerjError;
@@ -102,6 +101,9 @@ impl HnswParams {
#[derive(Debug)]
struct Node {
+ /// The node's own id. Redundant with its key in the `nodes` map, so it is
+ /// never read back out — kept for `Debug` output and crash diagnostics.
+ #[allow(dead_code)]
id: u64,
vector: Vec<f32>,
/// Neighbor lists per layer. `neighbors[layer]` holds neighbor node IDs.
@@ -113,7 +115,11 @@ impl Node {
let neighbors = (0..=max_layer)
.map(|l| Vec::with_capacity(if l == 0 { 2 * m } else { m }))
.collect();
- Self { id, vector, neighbors }
+ Self {
+ id,
+ vector,
+ neighbors,
+ }
}
}
@@ -238,17 +244,13 @@ impl HnswIndex {
let ep_id = self.entry_point.read().unwrap().unwrap();
let ep_layer = *self.entry_layer.read().unwrap();
+ // Greedy descent tracks only the current entry point; the distance to
+ // it is recomputed by `search_layer`/`greedy_search_layer` at each step,
+ // so we don't carry it between layers.
let mut curr_ep = ep_id;
- let mut curr_dist = {
- let nodes = self.nodes.read().unwrap();
- let ep_node = nodes[&curr_ep].read().unwrap();
- self.dist(&vector, &ep_node.vector)
- };
for layer in (node_level + 1..=ep_layer).rev() {
- let (new_ep, new_dist) = self.greedy_search_layer(&vector, curr_ep, layer, 1);
- curr_ep = new_ep;
- curr_dist = new_dist;
+ curr_ep = self.greedy_search_layer(&vector, curr_ep, layer, 1).0;
}
// Search and connect from node_level down to 0
@@ -298,7 +300,6 @@ impl HnswIndex {
if !candidates.is_empty() {
curr_ep = candidates[0].1;
- curr_dist = candidates[0].0;
}
}
@@ -351,9 +352,7 @@ impl HnswIndex {
// mid-query. Cloning a u64 HashSet is cheap and avoids holding
// the read lock across the whole traversal.
let tombstoned: HashSet<u64> = self.tombstones.read().unwrap().clone();
- let combined_filter = move |id: u64| -> bool {
- !tombstoned.contains(&id) && filter(id)
- };
+ let combined_filter = move |id: u64| -> bool { !tombstoned.contains(&id) && filter(id) };
let ep_opt = *self.entry_point.read().unwrap();
let ep_id = match ep_opt {
@@ -437,29 +436,39 @@ impl HnswIndex {
let mut buf: Vec<u8> = Vec::with_capacity(1024 * 1024);
buf.write_all(HNSW_MAGIC).map_err(io_to_xerj)?;
- buf.write_u32::<LittleEndian>(HNSW_FORMAT_VERSION).map_err(io_to_xerj)?;
- buf.write_u32::<LittleEndian>(self.params.m as u32).map_err(io_to_xerj)?;
- buf.write_u32::<LittleEndian>(self.params.ef_construction as u32).map_err(io_to_xerj)?;
- buf.write_u8(metric_to_u8(self.params.metric)).map_err(io_to_xerj)?;
- buf.write_u32::<LittleEndian>(self.params.dim as u32).map_err(io_to_xerj)?;
+ buf.write_u32::<LittleEndian>(HNSW_FORMAT_VERSION)
+ .map_err(io_to_xerj)?;
+ buf.write_u32::<LittleEndian>(self.params.m as u32)
+ .map_err(io_to_xerj)?;
+ buf.write_u32::<LittleEndian>(self.params.ef_construction as u32)
+ .map_err(io_to_xerj)?;
+ buf.write_u8(metric_to_u8(self.params.metric))
+ .map_err(io_to_xerj)?;
+ buf.write_u32::<LittleEndian>(self.params.dim as u32)
+ .map_err(io_to_xerj)?;
let ep = *self.entry_point.read().unwrap();
let el = *self.entry_layer.read().unwrap();
- buf.write_u64::<LittleEndian>(ep.unwrap_or(u64::MAX)).map_err(io_to_xerj)?;
- buf.write_u32::<LittleEndian>(el as u32).map_err(io_to_xerj)?;
+ buf.write_u64::<LittleEndian>(ep.unwrap_or(u64::MAX))
+ .map_err(io_to_xerj)?;
+ buf.write_u32::<LittleEndian>(el as u32)
+ .map_err(io_to_xerj)?;
let nodes = self.nodes.read().unwrap();
- buf.write_u32::<LittleEndian>(nodes.len() as u32).map_err(io_to_xerj)?;
+ buf.write_u32::<LittleEndian>(nodes.len() as u32)
+ .map_err(io_to_xerj)?;
for (id, node_arc) in nodes.iter() {
let n = node_arc.read().unwrap();
buf.write_u64::<LittleEndian>(*id).map_err(io_to_xerj)?;
let max_layer = n.neighbors.len().saturating_sub(1) as u32;
- buf.write_u32::<LittleEndian>(max_layer).map_err(io_to_xerj)?;
+ buf.write_u32::<LittleEndian>(max_layer)
+ .map_err(io_to_xerj)?;
for &v in &n.vector {
buf.write_f32::<LittleEndian>(v).map_err(io_to_xerj)?;
}
for layer_neighbors in &n.neighbors {
- buf.write_u32::<LittleEndian>(layer_neighbors.len() as u32).map_err(io_to_xerj)?;
+ buf.write_u32::<LittleEndian>(layer_neighbors.len() as u32)
+ .map_err(io_to_xerj)?;
for &nid in layer_neighbors {
buf.write_u64::<LittleEndian>(nid).map_err(io_to_xerj)?;
}
@@ -471,7 +480,8 @@ impl HnswIndex {
// a delete; v1 readers don't know to skip past this, so the
// format version was bumped (loader checks).
let tombs = self.tombstones.read().unwrap();
- buf.write_u32::<LittleEndian>(tombs.len() as u32).map_err(io_to_xerj)?;
+ buf.write_u32::<LittleEndian>(tombs.len() as u32)
+ .map_err(io_to_xerj)?;
for &t in tombs.iter() {
buf.write_u64::<LittleEndian>(t).map_err(io_to_xerj)?;
}
@@ -521,7 +531,7 @@ impl HnswIndex {
)));
}
let format_ver = cur.read_u32::<LittleEndian>().map_err(io_to_xerj)?;
- if format_ver < 1 || format_ver > HNSW_FORMAT_VERSION {
+ if !(1..=HNSW_FORMAT_VERSION).contains(&format_ver) {
return Err(corrupt(&format!(
"unsupported format version {format_ver}, this build supports 1..={HNSW_FORMAT_VERSION}"
)));
@@ -535,7 +545,12 @@ impl HnswIndex {
let entry_layer = cur.read_u32::<LittleEndian>().map_err(io_to_xerj)? as usize;
let num_nodes = cur.read_u32::<LittleEndian>().map_err(io_to_xerj)? as usize;
- let params = HnswParams { m, ef_construction, metric, dim };
+ let params = HnswParams {
+ m,
+ ef_construction,
+ metric,
+ dim,
+ };
let mut node_map: HashMap<u64, Arc<RwLock<Node>>> = HashMap::with_capacity(num_nodes);
for _ in 0..num_nodes {
@@ -554,7 +569,14 @@ impl HnswIndex {
}
neighbors.push(layer);
}
- node_map.insert(id, Arc::new(RwLock::new(Node { id, vector, neighbors })));
+ node_map.insert(
+ id,
+ Arc::new(RwLock::new(Node {
+ id,
+ vector,
+ neighbors,
+ })),
+ );
}
// v2+: read trailing tombstone block. v1 files have no
@@ -571,7 +593,11 @@ impl HnswIndex {
Ok(Self {
params,
nodes: RwLock::new(node_map),
- entry_point: RwLock::new(if ep_raw == u64::MAX { None } else { Some(ep_raw) }),
+ entry_point: RwLock::new(if ep_raw == u64::MAX {
+ None
+ } else {
+ Some(ep_raw)
+ }),
entry_layer: RwLock::new(entry_layer),
tombstones: RwLock::new(tombstones),
})
@@ -628,13 +654,7 @@ impl HnswIndex {
}
/// Beam search at a given layer, returning candidates sorted by distance.
- fn search_layer(
- &self,
- query: &[f32],
- entry: u64,
- ef: usize,
- layer: usize,
- ) -> Vec<(f32, u64)> {
+ fn search_layer(&self, query: &[f32], entry: u64, ef: usize, layer: usize) -> Vec<(f32, u64)> {
self.search_layer_filtered(query, entry, ef, layer, &|_| true)
}
@@ -716,10 +736,7 @@ impl HnswIndex {
}
// Collect results sorted by distance ascending
- let mut results: Vec<(f32, u64)> = w
- .into_iter()
- .map(|(d, id)| (d.0, id))
- .collect();
+ let mut results: Vec<(f32, u64)> = w.into_iter().map(|(d, id)| (d.0, id)).collect();
results.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
results
}
@@ -753,7 +770,9 @@ mod ordered_float {
impl Ord for OrderedFloat {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.0.partial_cmp(&other.0).unwrap_or(std::cmp::Ordering::Equal)
+ self.0
+ .partial_cmp(&other.0)
+ .unwrap_or(std::cmp::Ordering::Equal)
}
}
}
diff --git a/engine/crates/xerj-vector/src/lib.rs b/engine/crates/xerj-vector/src/lib.rs
index 526deebd1..7605cbdee 100644
--- a/engine/crates/xerj-vector/src/lib.rs
+++ b/engine/crates/xerj-vector/src/lib.rs
@@ -15,7 +15,7 @@ pub mod search;
pub use distance::{compute_distance, cosine, dot_product, l2_squared, DistanceMetric};
pub use hnsw::{HnswIndex, HnswParams};
-pub use quantizer::{NoneQuantizer, Quantizer, QuantizedData, QuantizedVectors, Scalar8Quantizer};
+pub use quantizer::{NoneQuantizer, QuantizedData, QuantizedVectors, Quantizer, Scalar8Quantizer};
pub use search::{SearchResult, VectorSearcher};
pub use xerj_common::Result;
diff --git a/engine/crates/xerj-vector/src/quantizer.rs b/engine/crates/xerj-vector/src/quantizer.rs
index f8771f5ca..72ff4e9e6 100644
--- a/engine/crates/xerj-vector/src/quantizer.rs
+++ b/engine/crates/xerj-vector/src/quantizer.rs
@@ -69,7 +69,11 @@ impl QuantizedData {
None
}
}
- QuantizedData::U8 { bytes, mins, scales } => {
+ QuantizedData::U8 {
+ bytes,
+ mins,
+ scales,
+ } => {
let start = idx * dim;
let end = start + dim;
if end <= bytes.len() {
@@ -83,9 +87,13 @@ impl QuantizedData {
None
}
}
- QuantizedData::Nibble { bytes, mins, scales } => {
+ QuantizedData::Nibble {
+ bytes,
+ mins,
+ scales,
+ } => {
// Each vector uses `ceil(dim/2)` bytes.
- let bytes_per_vec = (dim + 1) / 2;
+ let bytes_per_vec = dim.div_ceil(2);
let byte_start = idx * bytes_per_vec;
let byte_end = byte_start + bytes_per_vec;
if byte_end > bytes.len() {
@@ -223,12 +231,20 @@ impl Quantizer for Scalar8Quantizer {
));
}
for (d, &v) in vec.iter().enumerate() {
- if v < mins[d] { mins[d] = v; }
- if v > maxs[d] { maxs[d] = v; }
+ if v < mins[d] {
+ mins[d] = v;
+ }
+ if v > maxs[d] {
+ maxs[d] = v;
+ }
}
}
- let scales: Vec<f32> = mins.iter().zip(maxs.iter()).map(|(&mn, &mx)| mx - mn).collect();
+ let scales: Vec<f32> = mins
+ .iter()
+ .zip(maxs.iter())
+ .map(|(&mn, &mx)| mx - mn)
+ .collect();
// Encode all vectors
let mut bytes = Vec::with_capacity(vectors.len() * dim);
@@ -241,14 +257,26 @@ impl Quantizer for Scalar8Quantizer {
Ok(QuantizedVectors {
count: vectors.len(),
dim,
- data: QuantizedData::U8 { bytes, mins, scales },
+ data: QuantizedData::U8 {
+ bytes,
+ mins,
+ scales,
+ },
})
}
fn distance(&self, query: &[f32], quantized: &QuantizedData, id: usize) -> Result<f32> {
let (bytes, mins, scales) = match quantized {
- QuantizedData::U8 { bytes, mins, scales } => (bytes, mins, scales),
- _ => return Err(XerjError::internal("Scalar8Quantizer: wrong QuantizedData variant")),
+ QuantizedData::U8 {
+ bytes,
+ mins,
+ scales,
+ } => (bytes, mins, scales),
+ _ => {
+ return Err(XerjError::internal(
+ "Scalar8Quantizer: wrong QuantizedData variant",
+ ))
+ }
};
let dim = query.len();
@@ -345,15 +373,23 @@ impl Quantizer for Scalar4Quantizer {
));
}
for (d, &v) in vec.iter().enumerate() {
- if v < mins[d] { mins[d] = v; }
- if v > maxs[d] { maxs[d] = v; }
+ if v < mins[d] {
+ mins[d] = v;
+ }
+ if v > maxs[d] {
+ maxs[d] = v;
+ }
}
}
- let scales: Vec<f32> = mins.iter().zip(maxs.iter()).map(|(&mn, &mx)| mx - mn).collect();
+ let scales: Vec<f32> = mins
+ .iter()
+ .zip(maxs.iter())
+ .map(|(&mn, &mx)| mx - mn)
+ .collect();
// Each vector takes `ceil(dim/2)` bytes.
- let bytes_per_vec = (dim + 1) / 2;
+ let bytes_per_vec = dim.div_ceil(2);
let mut bytes = Vec::with_capacity(vectors.len() * bytes_per_vec);
for vec in vectors {
@@ -373,18 +409,30 @@ impl Quantizer for Scalar4Quantizer {
Ok(QuantizedVectors {
count: vectors.len(),
dim,
- data: QuantizedData::Nibble { bytes, mins, scales },
+ data: QuantizedData::Nibble {
+ bytes,
+ mins,
+ scales,
+ },
})
}
fn distance(&self, query: &[f32], quantized: &QuantizedData, id: usize) -> Result<f32> {
let (bytes, mins, scales) = match quantized {
- QuantizedData::Nibble { bytes, mins, scales } => (bytes, mins, scales),
- _ => return Err(XerjError::internal("Scalar4Quantizer: wrong QuantizedData variant")),
+ QuantizedData::Nibble {
+ bytes,
+ mins,
+ scales,
+ } => (bytes, mins, scales),
+ _ => {
+ return Err(XerjError::internal(
+ "Scalar4Quantizer: wrong QuantizedData variant",
+ ))
+ }
};
let dim = query.len();
- let bytes_per_vec = (dim + 1) / 2;
+ let bytes_per_vec = dim.div_ceil(2);
let byte_start = id * bytes_per_vec;
let byte_end = byte_start + bytes_per_vec;
@@ -444,19 +492,13 @@ mod tests {
#[test]
fn scalar8_roundtrip_approx() {
let q = Scalar8Quantizer;
- let vecs = vec![
- vec![0.0, 0.5, 1.0],
- vec![0.2, 0.7, 0.9],
- ];
+ let vecs = vec![vec![0.0, 0.5, 1.0], vec![0.2, 0.7, 0.9]];
let qv = q.quantize(&vecs).unwrap();
// Dequantized values should be close to originals (within ~1% error)
let recovered = qv.data.get_f32(0, 3).unwrap();
for (orig, rec) in vecs[0].iter().zip(recovered.iter()) {
- assert!(
- approx_eq(*orig, *rec, 0.02),
- "expected ~{orig}, got {rec}"
- );
+ assert!(approx_eq(*orig, *rec, 0.02), "expected ~{orig}, got {rec}");
}
}
@@ -485,18 +527,12 @@ mod tests {
#[test]
fn scalar4_roundtrip_approx() {
let q = Scalar4Quantizer;
- let vecs = vec![
- vec![0.0, 0.5, 1.0],
- vec![0.2, 0.7, 0.9],
- ];
+ let vecs = vec![vec![0.0, 0.5, 1.0], vec![0.2, 0.7, 0.9]];
let qv = q.quantize(&vecs).unwrap();
// 4-bit gives 16 levels over the range — tolerance of ~7%
let recovered = qv.data.get_f32(0, 3).unwrap();
for (orig, rec) in vecs[0].iter().zip(recovered.iter()) {
- assert!(
- (orig - rec).abs() < 0.08,
- "expected ~{orig}, got {rec}"
- );
+ assert!((orig - rec).abs() < 0.08, "expected ~{orig}, got {rec}");
}
}
@@ -519,7 +555,10 @@ mod tests {
let query = vec![1.0, 0.0, 0.0];
let d0 = q.distance(&query, &qv.data, 0).unwrap();
let d1 = q.distance(&query, &qv.data, 1).unwrap();
- assert!(d0 < d1, "closer vector should have smaller distance (4-bit)");
+ assert!(
+ d0 < d1,
+ "closer vector should have smaller distance (4-bit)"
+ );
}
#[test]
@@ -534,8 +573,14 @@ mod tests {
let qv4 = q4.quantize(&vecs).unwrap();
let qv8 = q8.quantize(&vecs).unwrap();
- let bytes4 = match &qv4.data { QuantizedData::Nibble { bytes, .. } => bytes.len(), _ => panic!() };
- let bytes8 = match &qv8.data { QuantizedData::U8 { bytes, .. } => bytes.len(), _ => panic!() };
+ let bytes4 = match &qv4.data {
+ QuantizedData::Nibble { bytes, .. } => bytes.len(),
+ _ => panic!(),
+ };
+ let bytes8 = match &qv8.data {
+ QuantizedData::U8 { bytes, .. } => bytes.len(),
+ _ => panic!(),
+ };
// scalar4 should use ≤ 55% of scalar8 byte count (target is 50%)
assert!(
diff --git a/engine/crates/xerj-vector/src/search.rs b/engine/crates/xerj-vector/src/search.rs
index e20330234..64fafd520 100644
--- a/engine/crates/xerj-vector/src/search.rs
+++ b/engine/crates/xerj-vector/src/search.rs
@@ -25,7 +25,11 @@ pub struct SearchResult {
impl SearchResult {
fn new(doc_id: u64, distance: f32, metric: DistanceMetric) -> Self {
let score = distance_to_score(distance, metric);
- Self { doc_id, distance, score }
+ Self {
+ doc_id,
+ distance,
+ score,
+ }
}
}
@@ -59,7 +63,10 @@ impl VectorSearcher {
/// Create a searcher with a custom ef.
pub fn with_ef(index: HnswIndex, ef: usize) -> Self {
- Self { index, default_ef: ef }
+ Self {
+ index,
+ default_ef: ef,
+ }
}
/// Get a reference to the underlying index.
@@ -93,7 +100,9 @@ impl VectorSearcher {
.collect();
// Sort by score descending
results.sort_unstable_by(|a, b| {
- b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)
+ b.score
+ .partial_cmp(&a.score)
+ .unwrap_or(std::cmp::Ordering::Equal)
});
Ok(results)
}
@@ -129,7 +138,9 @@ impl VectorSearcher {
.map(|(id, dist)| SearchResult::new(id, dist, metric))
.collect();
results.sort_unstable_by(|a, b| {
- b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)
+ b.score
+ .partial_cmp(&a.score)
+ .unwrap_or(std::cmp::Ordering::Equal)
});
Ok(results)
}
diff --git a/engine/crates/xerj-wasm/examples/example_custom_plugin.rs b/engine/crates/xerj-wasm/examples/example_custom_plugin.rs
index acebb2b96..b5ad69fd2 100644
--- a/engine/crates/xerj-wasm/examples/example_custom_plugin.rs
+++ b/engine/crates/xerj-wasm/examples/example_custom_plugin.rs
@@ -72,5 +72,8 @@ fn main() {
let action = pipeline.process(&mut doc);
println!("Action: {action:?}");
- println!("Transformed doc: {}", serde_json::to_string_pretty(&doc).unwrap());
+ println!(
+ "Transformed doc: {}",
+ serde_json::to_string_pretty(&doc).unwrap()
+ );
}
diff --git a/engine/crates/xerj-wasm/src/builtins.rs b/engine/crates/xerj-wasm/src/builtins.rs
index bcdbd547a..5711d3c6f 100644
--- a/engine/crates/xerj-wasm/src/builtins.rs
+++ b/engine/crates/xerj-wasm/src/builtins.rs
@@ -98,11 +98,7 @@ pub struct TimestampParsePlugin {
}
impl TimestampParsePlugin {
- pub fn new(
- field: impl Into<String>,
- formats: Vec<String>,
- target: Option<String>,
- ) -> Self {
+ pub fn new(field: impl Into<String>, formats: Vec<String>, target: Option<String>) -> Self {
Self {
field: field.into(),
formats,
@@ -187,7 +183,11 @@ impl TransformPlugin for TimestampParsePlugin {
return ProcessAction::Pass;
}
- debug!(field = self.field.as_str(), raw = raw.as_str(), "timestamp_parse: no format matched");
+ debug!(
+ field = self.field.as_str(),
+ raw = raw.as_str(),
+ "timestamp_parse: no format matched"
+ );
ProcessAction::Pass
}
}
@@ -426,8 +426,16 @@ fn grok_pattern(name: &str) -> (&'static str, &'static [&'static str]) {
"NGINX_COMBINED" | "APACHE_COMBINED" => (
r#"^(?P<remote_addr>\S+) - (?P<remote_user>\S+) \[(?P<time_local>[^\]]+)\] "(?P<method>\S+) (?P<request_uri>\S+) (?P<http_version>[^"]+)" (?P<status>\d{3}) (?P<body_bytes_sent>\d+) "(?P<http_referer>[^"]*)" "(?P<http_user_agent>[^"]*)"#,
&[
- "remote_addr", "remote_user", "time_local", "method", "request_uri",
- "http_version", "status", "body_bytes_sent", "http_referer", "http_user_agent",
+ "remote_addr",
+ "remote_user",
+ "time_local",
+ "method",
+ "request_uri",
+ "http_version",
+ "status",
+ "body_bytes_sent",
+ "http_referer",
+ "http_user_agent",
],
),
"SYSLOG" => (
@@ -436,7 +444,14 @@ fn grok_pattern(name: &str) -> (&'static str, &'static [&'static str]) {
),
"POSTGRESQL" => (
r#"^(?P<log_time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ \w+) \[(?P<pid>\d+)\] (?P<user_name>\S+)@(?P<database_name>\S+) (?P<severity>\w+): (?P<message>.+)$"#,
- &["log_time", "pid", "user_name", "database_name", "severity", "message"],
+ &[
+ "log_time",
+ "pid",
+ "user_name",
+ "database_name",
+ "severity",
+ "message",
+ ],
),
_ => (
// Generic: capture everything as `message`.
@@ -730,7 +745,9 @@ pub struct LowercasePlugin {
impl LowercasePlugin {
pub fn new(field: impl Into<String>) -> Self {
- Self { field: field.into() }
+ Self {
+ field: field.into(),
+ }
}
}
@@ -758,7 +775,9 @@ pub struct UppercasePlugin {
impl UppercasePlugin {
pub fn new(field: impl Into<String>) -> Self {
- Self { field: field.into() }
+ Self {
+ field: field.into(),
+ }
}
}
@@ -806,10 +825,7 @@ impl TransformPlugin for SetPlugin {
}
fn process(&self, doc: &mut Value) -> ProcessAction {
- let exists = doc
- .get(&self.field)
- .map(|v| !v.is_null())
- .unwrap_or(false);
+ let exists = doc.get(&self.field).map(|v| !v.is_null()).unwrap_or(false);
if self.override_existing || !exists {
doc[&self.field] = self.value.clone();
@@ -853,7 +869,9 @@ pub struct UrlDecodePlugin {
impl UrlDecodePlugin {
pub fn new(field: impl Into<String>) -> Self {
- Self { field: field.into() }
+ Self {
+ field: field.into(),
+ }
}
fn percent_decode(s: &str) -> Option<String> {
@@ -984,7 +1002,9 @@ mod tests {
#[test]
fn field_rename_basic() {
let plugin = FieldRenamePlugin::new(
- [("old_name".into(), "new_name".into())].into_iter().collect(),
+ [("old_name".into(), "new_name".into())]
+ .into_iter()
+ .collect(),
);
let mut doc = json!({ "old_name": "value", "other": 1 });
assert_eq!(plugin.process(&mut doc), ProcessAction::Pass);
@@ -996,7 +1016,9 @@ mod tests {
#[test]
fn field_rename_missing_field_no_op() {
let plugin = FieldRenamePlugin::new(
- [("does_not_exist".into(), "target".into())].into_iter().collect(),
+ [("does_not_exist".into(), "target".into())]
+ .into_iter()
+ .collect(),
);
let mut doc = json!({ "a": 1 });
assert_eq!(plugin.process(&mut doc), ProcessAction::Pass);
@@ -1053,18 +1075,20 @@ mod tests {
let plugin = RoutePlugin::new("level", routes, None);
let mut doc = json!({ "level": "error", "msg": "oops" });
- assert_eq!(plugin.process(&mut doc), ProcessAction::Route("logs-errors".into()));
+ assert_eq!(
+ plugin.process(&mut doc),
+ ProcessAction::Route("logs-errors".into())
+ );
}
#[test]
fn route_fallback_to_default() {
- let plugin = RoutePlugin::new(
- "level",
- HashMap::new(),
- Some("logs-misc".into()),
- );
+ let plugin = RoutePlugin::new("level", HashMap::new(), Some("logs-misc".into()));
let mut doc = json!({ "level": "debug" });
- assert_eq!(plugin.process(&mut doc), ProcessAction::Route("logs-misc".into()));
+ assert_eq!(
+ plugin.process(&mut doc),
+ ProcessAction::Route("logs-misc".into())
+ );
}
#[test]
@@ -1117,7 +1141,10 @@ mod tests {
let mut doc = json!({ "msg": "contact [email protected] for info" });
assert_eq!(plugin.process(&mut doc), ProcessAction::Pass);
let msg = doc["msg"].as_str().unwrap();
- assert!(!msg.contains("[email protected]"), "email not redacted: {msg}");
+ assert!(
+ !msg.contains("[email protected]"),
+ "email not redacted: {msg}"
+ );
assert!(msg.contains("[REDACTED_EMAIL]"), "got: {msg}");
}
diff --git a/engine/crates/xerj-wasm/src/pipeline.rs b/engine/crates/xerj-wasm/src/pipeline.rs
index b4c4d38d7..ce3fad251 100644
--- a/engine/crates/xerj-wasm/src/pipeline.rs
+++ b/engine/crates/xerj-wasm/src/pipeline.rs
@@ -106,10 +106,12 @@ impl Pipeline {
let mut stages: Vec<Arc<dyn TransformPlugin>> = Vec::new();
for stage_cfg in &config.stages {
- let plugin = build_plugin(&stage_cfg.stage_type, &stage_cfg.config)
- .map_err(|reason| WasmError::InvalidConfig {
- plugin: stage_cfg.stage_type.clone(),
- reason,
+ let plugin =
+ build_plugin(&stage_cfg.stage_type, &stage_cfg.config).map_err(|reason| {
+ WasmError::InvalidConfig {
+ plugin: stage_cfg.stage_type.clone(),
+ reason,
+ }
})?;
stages.push(plugin);
}
@@ -132,7 +134,11 @@ impl Pipeline {
/// [`ProcessAction::Pass`] if all stages pass.
pub fn process(&self, doc: &mut Value) -> ProcessAction {
for stage in &self.stages {
- debug!(pipeline = self.name.as_str(), stage = stage.name(), "running stage");
+ debug!(
+ pipeline = self.name.as_str(),
+ stage = stage.name(),
+ "running stage"
+ );
match stage.process(doc) {
ProcessAction::Pass => continue,
action => {
@@ -393,10 +399,7 @@ mod tests {
fn pipeline_short_circuits_on_drop() {
let pl = make_pipeline(&[
// First stage drops the document
- (
- "drop_field",
- serde_json::json!({ "fields": ["drop_me"] }),
- ),
+ ("drop_field", serde_json::json!({ "fields": ["drop_me"] })),
// This stage would add a field, but we'll test with a route stage
// that triggers drop via a route mismatch — use a simple 2-stage test
(
@@ -416,10 +419,7 @@ mod tests {
"add_field",
serde_json::json!({ "field": "pipeline", "value": "default" }),
)]);
- let mut docs = vec![
- serde_json::json!({ "a": 1 }),
- serde_json::json!({ "b": 2 }),
- ];
+ let mut docs = vec![serde_json::json!({ "a": 1 }), serde_json::json!({ "b": 2 })];
let actions = pl.process_batch(&mut docs);
assert!(actions.iter().all(|a| *a == ProcessAction::Pass));
assert_eq!(docs[0]["pipeline"], "default");
diff --git a/engine/crates/xerj-wasm/tests/integration.rs b/engine/crates/xerj-wasm/tests/integration.rs
index 76866013a..307746bb9 100644
--- a/engine/crates/xerj-wasm/tests/integration.rs
+++ b/engine/crates/xerj-wasm/tests/integration.rs
@@ -1,7 +1,9 @@
//! Integration tests for the xerj-wasm pipeline system.
use serde_json::json;
-use xerj_wasm::pipeline::{ErrorPolicy, Pipeline, PipelineConfig, PipelineStageConfig, ProcessAction};
+use xerj_wasm::pipeline::{
+ ErrorPolicy, Pipeline, PipelineConfig, PipelineStageConfig, ProcessAction,
+};
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -33,14 +35,8 @@ fn test_full_pipeline_nginx_log() {
"grok",
json!({ "pattern": "NGINX_COMBINED", "field": "message" }),
),
- (
- "timestamp_parse",
- json!({ "field": "time_local" }),
- ),
- (
- "pii_redaction",
- json!({ "types": ["ip", "email"] }),
- ),
+ ("timestamp_parse", json!({ "field": "time_local" })),
+ ("pii_redaction", json!({ "types": ["ip", "email"] })),
(
"add_field",
json!({ "field": "pipeline", "value": "nginx-access" }),
@@ -56,7 +52,10 @@ fn test_full_pipeline_nginx_log() {
// Grok fields extracted
assert_eq!(doc["method"], "GET", "method field not extracted");
assert_eq!(doc["status"], "200", "status field not extracted");
- assert_eq!(doc["request_uri"], "/index.html", "request_uri not extracted");
+ assert_eq!(
+ doc["request_uri"], "/index.html",
+ "request_uri not extracted"
+ );
// Timestamp parsed from time_local
assert!(
@@ -65,7 +64,10 @@ fn test_full_pipeline_nginx_log() {
doc["@timestamp"]
);
let ts = doc["@timestamp"].as_str().unwrap();
- assert!(ts.contains("2026-04-10"), "timestamp not parsed correctly: {ts}");
+ assert!(
+ ts.contains("2026-04-10"),
+ "timestamp not parsed correctly: {ts}"
+ );
// PII redacted — remote_addr should be masked
let remote = doc["remote_addr"].as_str().unwrap_or("");
@@ -129,17 +131,35 @@ fn test_pipeline_with_routing() {
fn test_pipeline_chaining_all_plugins() {
let pl = make_pipeline(&[
// json_parse: parse a JSON payload field
- ("json_parse", json!({ "field": "payload", "target": "parsed" })),
+ (
+ "json_parse",
+ json!({ "field": "payload", "target": "parsed" }),
+ ),
// add_field: add a static tag
- ("add_field", json!({ "field": "env", "value": "production" })),
+ (
+ "add_field",
+ json!({ "field": "env", "value": "production" }),
+ ),
// copy_field: copy env → environment
- ("copy_field", json!({ "source": "env", "target": "environment" })),
+ (
+ "copy_field",
+ json!({ "source": "env", "target": "environment" }),
+ ),
// set: only set if missing
- ("set", json!({ "field": "region", "value": "us-east-1", "override": false })),
+ (
+ "set",
+ json!({ "field": "region", "value": "us-east-1", "override": false }),
+ ),
// set: override always
- ("set", json!({ "field": "env", "value": "prod", "override": true })),
+ (
+ "set",
+ json!({ "field": "env", "value": "prod", "override": true }),
+ ),
// convert: status code to integer
- ("convert", json!({ "field": "status_code", "type": "integer" })),
+ (
+ "convert",
+ json!({ "field": "status_code", "type": "integer" }),
+ ),
// split: split tags
("split", json!({ "field": "tags", "separator": "," })),
// lowercase: lowercase method
@@ -159,7 +179,10 @@ fn test_pipeline_chaining_all_plugins() {
// timestamp_parse
("timestamp_parse", json!({ "field": "ts" })),
// grok on log line
- ("grok", json!({ "pattern": "SYSLOG", "field": "syslog_line" })),
+ (
+ "grok",
+ json!({ "pattern": "SYSLOG", "field": "syslog_line" }),
+ ),
]);
let mut doc = json!({
@@ -176,10 +199,17 @@ fn test_pipeline_chaining_all_plugins() {
});
let action = pl.process(&mut doc);
- assert_eq!(action, ProcessAction::Pass, "all-plugin pipeline should pass");
+ assert_eq!(
+ action,
+ ProcessAction::Pass,
+ "all-plugin pipeline should pass"
+ );
// json_parse + field_rename
- assert_eq!(doc["data"]["key"], "value", "json_parse + field_rename failed");
+ assert_eq!(
+ doc["data"]["key"], "value",
+ "json_parse + field_rename failed"
+ );
// add_field
assert_eq!(doc["environment"], "production", "copy_field failed");
// set override=false keeps existing
@@ -201,7 +231,10 @@ fn test_pipeline_chaining_all_plugins() {
assert!(doc.get("null_field").is_none(), "remove_null failed");
// pii_redaction
let contact = doc["contact"].as_str().unwrap();
- assert!(contact.contains("[REDACTED_EMAIL]"), "pii_redaction failed: {contact}");
+ assert!(
+ contact.contains("[REDACTED_EMAIL]"),
+ "pii_redaction failed: {contact}"
+ );
// timestamp_parse
assert!(doc["@timestamp"].is_string(), "@timestamp not set");
// grok
@@ -225,7 +258,10 @@ fn test_pipeline_error_handling() {
timeout_ms: 0,
};
let result = Pipeline::from_config("bad", &cfg);
- assert!(result.is_err(), "building pipeline with unknown plugin should fail");
+ assert!(
+ result.is_err(),
+ "building pipeline with unknown plugin should fail"
+ );
// With on_error = Pass, a missing-field stage is a no-op (passes through).
let pl = make_pipeline(&[
@@ -236,7 +272,10 @@ fn test_pipeline_error_handling() {
]);
let mut doc = json!({ "msg": "hello" });
assert_eq!(pl.process(&mut doc), ProcessAction::Pass);
- assert_eq!(doc["survived"], true, "document should survive after no-op stage");
+ assert_eq!(
+ doc["survived"], true,
+ "document should survive after no-op stage"
+ );
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -290,7 +329,10 @@ fn test_simulate_pipeline() {
#[test]
fn test_copy_field() {
- let pl = make_pipeline(&[("copy_field", json!({ "source": "msg", "target": "original" }))]);
+ let pl = make_pipeline(&[(
+ "copy_field",
+ json!({ "source": "msg", "target": "original" }),
+ )]);
let mut doc = json!({ "msg": "hello" });
assert_eq!(pl.process(&mut doc), ProcessAction::Pass);
assert_eq!(doc["msg"], "hello", "source should be preserved");
@@ -305,10 +347,13 @@ fn test_convert_types() {
("convert", json!({ "field": "s", "type": "string" })),
("convert", json!({ "field": "b", "type": "boolean" })),
]);
- let mut doc = json!({ "n": "42", "f": "3.14", "s": 99, "b": "true" });
+ let mut doc = json!({ "n": "42", "f": "2.5", "s": 99, "b": "true" });
assert_eq!(pl.process(&mut doc), ProcessAction::Pass);
assert_eq!(doc["n"], 42, "integer convert failed");
- assert!((doc["f"].as_f64().unwrap() - 3.14).abs() < 0.001, "float convert failed");
+ assert!(
+ (doc["f"].as_f64().unwrap() - 2.5).abs() < 0.001,
+ "float convert failed"
+ );
assert_eq!(doc["s"], "99", "string convert failed");
assert_eq!(doc["b"], true, "boolean convert failed");
}
@@ -329,7 +374,10 @@ fn test_split_and_case_plugins() {
#[test]
fn test_set_plugin_no_override() {
- let pl = make_pipeline(&[("set", json!({ "field": "env", "value": "production", "override": false }))]);
+ let pl = make_pipeline(&[(
+ "set",
+ json!({ "field": "env", "value": "production", "override": false }),
+ )]);
// Field absent → should be set
let mut doc = json!({ "msg": "hi" });
@@ -339,15 +387,24 @@ fn test_set_plugin_no_override() {
// Field present → should NOT be overwritten
let mut doc2 = json!({ "env": "staging" });
pl.process(&mut doc2);
- assert_eq!(doc2["env"], "staging", "set with override=false should not overwrite");
+ assert_eq!(
+ doc2["env"], "staging",
+ "set with override=false should not overwrite"
+ );
}
#[test]
fn test_set_plugin_with_override() {
- let pl = make_pipeline(&[("set", json!({ "field": "env", "value": "production", "override": true }))]);
+ let pl = make_pipeline(&[(
+ "set",
+ json!({ "field": "env", "value": "production", "override": true }),
+ )]);
let mut doc = json!({ "env": "staging" });
pl.process(&mut doc);
- assert_eq!(doc["env"], "production", "set with override=true should overwrite");
+ assert_eq!(
+ doc["env"], "production",
+ "set with override=true should overwrite"
+ );
}
#[test]
@@ -356,7 +413,10 @@ fn test_remove_null_plugin() {
let mut doc = json!({ "a": 1, "b": null, "c": "hello", "d": "" });
pl.process(&mut doc);
assert!(doc.get("b").is_none(), "null field should be removed");
- assert!(doc.get("d").is_none(), "empty string field should be removed");
+ assert!(
+ doc.get("d").is_none(),
+ "empty string field should be removed"
+ );
assert_eq!(doc["a"], 1, "non-null field should remain");
assert_eq!(doc["c"], "hello", "non-null field should remain");
}
@@ -366,5 +426,8 @@ fn test_url_decode_plugin() {
let pl = make_pipeline(&[("url_decode", json!({ "field": "path" }))]);
let mut doc = json!({ "path": "/search%3Fq%3Dhello+world%26lang%3Den" });
pl.process(&mut doc);
- assert_eq!(doc["path"], "/search?q=hello world&lang=en", "url_decode failed");
+ assert_eq!(
+ doc["path"], "/search?q=hello world&lang=en",
+ "url_decode failed"
+ );
}
diff --git a/engine/tests/es-compat-yaml/src/main.rs b/engine/tests/es-compat-yaml/src/main.rs
index 1663791e5..f15465583 100644
--- a/engine/tests/es-compat-yaml/src/main.rs
+++ b/engine/tests/es-compat-yaml/src/main.rs
@@ -15,8 +15,8 @@
use clap::Parser;
use colored::*;
-use serde_json::Value;
use serde::Deserialize;
+use serde_json::Value;
use serde_yaml::Value as YamlValue;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
@@ -58,11 +58,20 @@ fn main() {
println!(
"\n{}",
- format!("ES-COMPAT YAML RUNNER · {} files · {}", files.len(), cli.url).bold()
+ format!(
+ "ES-COMPAT YAML RUNNER · {} files · {}",
+ files.len(),
+ cli.url
+ )
+ .bold()
);
println!("{}\n", "=".repeat(60));
- let mut total = Stats { passed: 0, failed: 0, skipped: 0 };
+ let mut total = Stats {
+ passed: 0,
+ failed: 0,
+ skipped: 0,
+ };
for path in &files {
run_file(&client, &cli.url, path, cli.verbose, &mut total);
@@ -90,19 +99,18 @@ fn collect_files(cli: &Cli) -> Vec<PathBuf> {
}
}
if let Some(d) = &cli.dir {
- for entry in glob::glob(&format!("{}/**/*.yml", d.display())).unwrap() {
- if let Ok(p) = entry {
- files.push(p);
- }
+ for p in glob::glob(&format!("{}/**/*.yml", d.display()))
+ .unwrap()
+ .flatten()
+ {
+ files.push(p);
}
}
if files.is_empty() && cli.file.is_none() && cli.dir.is_none() {
let default = Path::new("yaml");
if default.exists() {
- for entry in glob::glob("yaml/**/*.yml").unwrap() {
- if let Ok(p) = entry {
- files.push(p);
- }
+ for p in glob::glob("yaml/**/*.yml").unwrap().flatten() {
+ files.push(p);
}
}
}
@@ -110,15 +118,18 @@ fn collect_files(cli: &Cli) -> Vec<PathBuf> {
files
}
+// Helper kept for the per-test index-isolation work (wire-up pending); not
+// yet called from the runner loop, so silence dead_code under `-D warnings`.
+#[allow(dead_code)]
fn extract_setup_indices(steps: &[YamlValue]) -> Vec<String> {
let mut indices = Vec::new();
for step in steps {
if let YamlValue::Mapping(m) = step {
- if let Some(YamlValue::Mapping(action)) = m.get(&YamlValue::String("do".into())) {
+ if let Some(YamlValue::Mapping(action)) = m.get(YamlValue::String("do".into())) {
for (key, val) in action {
let action_name = yaml_to_string(key);
if let YamlValue::Mapping(params) = val {
- if let Some(idx) = params.get(&YamlValue::String("index".into())) {
+ if let Some(idx) = params.get(YamlValue::String("index".into())) {
let idx_str = yaml_to_string(idx);
if !idx_str.is_empty() && !indices.contains(&idx_str) {
indices.push(idx_str);
@@ -126,7 +137,10 @@ fn extract_setup_indices(steps: &[YamlValue]) -> Vec<String> {
}
}
// Also check bulk and index actions in test body steps
- if action_name == "indices.create" || action_name == "bulk" || action_name == "index" {
+ if action_name == "indices.create"
+ || action_name == "bulk"
+ || action_name == "index"
+ {
// already handled above
}
}
@@ -137,7 +151,7 @@ fn extract_setup_indices(steps: &[YamlValue]) -> Vec<String> {
}
fn cleanup_indices(client: &reqwest::blocking::Client, base_url: &str) {
- let _ = client.delete(&format!("{}/_all", base_url)).send();
+ let _ = client.delete(format!("{}/_all", base_url)).send();
}
fn run_file(
@@ -183,9 +197,8 @@ fn run_file(
for doc in &docs {
if let YamlValue::Mapping(m) = doc {
- if m.contains_key(&YamlValue::String("setup".into())) {
- if let Some(YamlValue::Sequence(steps)) = m.get(&YamlValue::String("setup".into()))
- {
+ if m.contains_key(YamlValue::String("setup".into())) {
+ if let Some(YamlValue::Sequence(steps)) = m.get(YamlValue::String("setup".into())) {
setup_steps = steps.clone();
}
continue;
@@ -205,16 +218,26 @@ fn run_file(
let mut setup_response: Value = Value::Null;
let mut setup_vars: HashMap<String, String> = HashMap::new();
for step in &setup_steps {
- if let Err(e) = execute_step(client, base_url, step, &mut setup_response, &mut setup_vars) {
+ if let Err(e) =
+ execute_step(client, base_url, step, &mut setup_response, &mut setup_vars)
+ {
if verbose {
- println!(" {} {} · {} — setup: {}", "SKIP".yellow(), short_name, name, e);
+ println!(
+ " {} {} · {} — setup: {}",
+ "SKIP".yellow(),
+ short_name,
+ name,
+ e
+ );
}
stats.skipped += 1;
setup_ok = false;
break;
}
}
- if !setup_ok { continue; }
+ if !setup_ok {
+ continue;
+ }
let result = run_test_case(client, base_url, &[], steps, verbose);
match result {
@@ -225,12 +248,24 @@ fn run_file(
stats.passed += 1;
}
TestResult::Fail(reason) => {
- println!(" {} {} · {} — {}", "FAIL".red().bold(), short_name, name, reason);
+ println!(
+ " {} {} · {} — {}",
+ "FAIL".red().bold(),
+ short_name,
+ name,
+ reason
+ );
stats.failed += 1;
}
TestResult::Skip(reason) => {
if verbose {
- println!(" {} {} · {} — {}", "SKIP".yellow(), short_name, name, reason);
+ println!(
+ " {} {} · {} — {}",
+ "SKIP".yellow(),
+ short_name,
+ name,
+ reason
+ );
}
stats.skipped += 1;
}
@@ -292,7 +327,7 @@ fn execute_step(
};
// `set` — store response field into a variable
- if let Some(YamlValue::Mapping(m)) = map.get(&YamlValue::String("set".into())) {
+ if let Some(YamlValue::Mapping(m)) = map.get(YamlValue::String("set".into())) {
for (path, var_name) in m {
let path_str = yaml_to_string(path);
let var_str = yaml_to_string(var_name);
@@ -301,15 +336,14 @@ fn execute_step(
// the parent path, fetching its first key, and assigning
// THAT to the variable.
if path_str.ends_with("._arbitrary_key_") || path_str == "_arbitrary_key_" {
- let parent_path = path_str
- .strip_suffix("._arbitrary_key_")
- .unwrap_or("");
+ let parent_path = path_str.strip_suffix("._arbitrary_key_").unwrap_or("");
let parent_val = if parent_path.is_empty() {
last_response.clone()
} else {
json_path(last_response, parent_path)
};
- let key_str = parent_val.as_object()
+ let key_str = parent_val
+ .as_object()
.and_then(|o| o.keys().next().cloned())
.unwrap_or_default();
vars.insert(var_str, key_str);
@@ -328,7 +362,7 @@ fn execute_step(
return Ok(());
}
- if let Some(action) = map.get(&YamlValue::String("do".into())) {
+ if let Some(action) = map.get(YamlValue::String("do".into())) {
return execute_do(client, base_url, action, last_response, vars);
}
@@ -336,7 +370,7 @@ fn execute_step(
//
// Special paths: `$body` refers to the entire response body,
// `""` (empty) also resolves to the whole body.
- if let Some(YamlValue::Mapping(m)) = map.get(&YamlValue::String("match".into())) {
+ if let Some(YamlValue::Mapping(m)) = map.get(YamlValue::String("match".into())) {
for (path, expected) in m {
let raw_path_str = yaml_to_string(path);
let path_str = substitute_vars(&raw_path_str, vars);
@@ -383,7 +417,7 @@ fn execute_step(
// `contains` — assert path contains a value. For strings, substring
// match; for arrays, element must appear; for objects, expected must
// be a subset of the actual map.
- if let Some(YamlValue::Mapping(m)) = map.get(&YamlValue::String("contains".into())) {
+ if let Some(YamlValue::Mapping(m)) = map.get(YamlValue::String("contains".into())) {
for (path, expected) in m {
let path_str = yaml_to_string(path);
// Paths that are a bare `$var` reference (a stashed value) resolve
@@ -414,7 +448,7 @@ fn execute_step(
(Value::Array(a), e) => a.iter().any(|item| values_match(item, e)),
(Value::Object(a), Value::Object(e)) => e
.iter()
- .all(|(k, v)| a.get(k).map_or(false, |av| values_match(av, v))),
+ .all(|(k, v)| a.get(k).is_some_and(|av| values_match(av, v))),
_ => values_match(&actual, &expected_json),
};
if !ok {
@@ -428,7 +462,7 @@ fn execute_step(
}
// `length` — assert array/object length
- if let Some(YamlValue::Mapping(m)) = map.get(&YamlValue::String("length".into())) {
+ if let Some(YamlValue::Mapping(m)) = map.get(YamlValue::String("length".into())) {
for (path, expected) in m {
let raw_path_str = yaml_to_string(path);
let path_str = substitute_vars(&raw_path_str, vars);
@@ -481,7 +515,7 @@ fn execute_step(
}
}
}
- if let Some(path) = map.get(&YamlValue::String("is_true".into())) {
+ if let Some(path) = map.get(YamlValue::String("is_true".into())) {
let path_str = yaml_to_string(path);
let resolved_path = substitute_vars(&path_str, vars);
let val = if resolved_path == "$body" || resolved_path.is_empty() {
@@ -496,7 +530,7 @@ fn execute_step(
}
// `is_false` — assert field is falsy
- if let Some(path) = map.get(&YamlValue::String("is_false".into())) {
+ if let Some(path) = map.get(YamlValue::String("is_false".into())) {
let path_str = yaml_to_string(path);
let resolved_path = substitute_vars(&path_str, vars);
let val = if resolved_path == "$body" || resolved_path.is_empty() {
@@ -517,7 +551,7 @@ fn execute_step(
("lte", |a, b| a <= b),
("lt", |a, b| a < b),
] {
- if let Some(YamlValue::Mapping(m)) = map.get(&YamlValue::String(op_name.into())) {
+ if let Some(YamlValue::Mapping(m)) = map.get(YamlValue::String(op_name.into())) {
for (path, expected) in m {
let raw_path_str = yaml_to_string(path);
let path_str = substitute_vars(&raw_path_str, vars);
@@ -533,7 +567,7 @@ fn execute_step(
}
// `close_to` — assert floating-point value is within tolerance
- if let Some(YamlValue::Mapping(m)) = map.get(&YamlValue::String("close_to".into())) {
+ if let Some(YamlValue::Mapping(m)) = map.get(YamlValue::String("close_to".into())) {
for (path, spec) in m {
let raw_path_str = yaml_to_string(path);
let path_str = substitute_vars(&raw_path_str, vars);
@@ -544,12 +578,17 @@ fn execute_step(
let expected = match spec_json.get("value") {
Some(Value::String(s)) if s.starts_with('$') => {
let name = &s[1..];
- vars.get(name).and_then(|v| v.parse::<f64>().ok()).unwrap_or(0.0)
+ vars.get(name)
+ .and_then(|v| v.parse::<f64>().ok())
+ .unwrap_or(0.0)
}
Some(v) => v.as_f64().unwrap_or(0.0),
None => 0.0,
};
- let error = spec_json.get("error").and_then(|v| v.as_f64()).unwrap_or(0.001);
+ let error = spec_json
+ .get("error")
+ .and_then(|v| v.as_f64())
+ .unwrap_or(0.001);
if (actual_f - expected).abs() > error {
return Err(format!(
"close_to {}: expected {:.6} ± {}, got {:.6}",
@@ -564,12 +603,13 @@ fn execute_step(
// explicitly awaiting a fix in the upstream project.
// `skip: { features: [...] }` is runner-capability metadata (e.g.,
// "headers", "allowed_warnings") — not a reason to skip on XERJ.
- if let Some(skip_val) = map.get(&YamlValue::String("skip".into())) {
+ if let Some(skip_val) = map.get(YamlValue::String("skip".into())) {
if let YamlValue::Mapping(skip_map) = skip_val {
- let has_version = skip_map.contains_key(&YamlValue::String("version".into()));
- let has_features = skip_map.contains_key(&YamlValue::String("features".into()));
- let has_cluster_features = skip_map.contains_key(&YamlValue::String("cluster_features".into()));
- let has_awaits_fix = skip_map.contains_key(&YamlValue::String("awaits_fix".into()));
+ let has_version = skip_map.contains_key(YamlValue::String("version".into()));
+ let has_features = skip_map.contains_key(YamlValue::String("features".into()));
+ let has_cluster_features =
+ skip_map.contains_key(YamlValue::String("cluster_features".into()));
+ let has_awaits_fix = skip_map.contains_key(YamlValue::String("awaits_fix".into()));
// awaits_fix marks a test upstream is tracking as broken — skip.
if has_awaits_fix {
return Err("skip directive (awaits_fix)".into());
@@ -582,7 +622,9 @@ fn execute_step(
// as an ES-8.x-equivalent cluster, so any `gte_vX.Y` constraint
// where X<=8 is considered satisfied → skip.
if has_cluster_features {
- if let Some(YamlValue::Sequence(items)) = skip_map.get(&YamlValue::String("cluster_features".into())) {
+ if let Some(YamlValue::Sequence(items)) =
+ skip_map.get(YamlValue::String("cluster_features".into()))
+ {
for f in items {
if let YamlValue::String(s) = f {
if cluster_feature_satisfied(s) {
@@ -590,7 +632,9 @@ fn execute_step(
}
}
}
- } else if let Some(YamlValue::String(s)) = skip_map.get(&YamlValue::String("cluster_features".into())) {
+ } else if let Some(YamlValue::String(s)) =
+ skip_map.get(YamlValue::String("cluster_features".into()))
+ {
if cluster_feature_satisfied(s) {
return Err("skip directive (cluster_features)".into());
}
@@ -608,7 +652,7 @@ fn execute_step(
// `requires` — tests may enumerate cluster versions / features they
// depend on. We permissively attempt every `requires` test (we aim
// for 100% on the broadest feature set), so this block is a no-op.
- if map.contains_key(&YamlValue::String("requires".into())) {
+ if map.contains_key(YamlValue::String("requires".into())) {
return Ok(());
}
@@ -623,7 +667,9 @@ fn execute_step(
/// are still outstanding for us)
/// - gte_v9+ is NOT satisfied
fn cluster_feature_satisfied(feature: &str) -> bool {
- let Some(rest) = feature.strip_prefix("gte_v") else { return false };
+ let Some(rest) = feature.strip_prefix("gte_v") else {
+ return false;
+ };
let parts: Vec<&str> = rest.split('.').collect();
let major: u32 = parts.first().and_then(|s| s.parse().ok()).unwrap_or(0);
let minor: u32 = parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(0);
@@ -658,14 +704,17 @@ fn execute_do(
// Skip `catch` expectations and `headers` for now
let mut catch: Option<String> = None;
- if let Some(c) = map.get(&YamlValue::String("catch".into())) {
+ if let Some(c) = map.get(YamlValue::String("catch".into())) {
catch = Some(yaml_to_string(c));
}
for (key, val) in map {
let action_name = yaml_to_string(key);
- if action_name == "catch" || action_name == "headers" || action_name == "warnings"
- || action_name == "allowed_warnings" || action_name == "allowed_warnings_regex"
+ if action_name == "catch"
+ || action_name == "headers"
+ || action_name == "warnings"
+ || action_name == "allowed_warnings"
+ || action_name == "allowed_warnings_regex"
|| action_name == "node_selector"
{
continue;
@@ -704,7 +753,11 @@ fn execute_do(
let resp_text = resp.text().unwrap_or_default();
if std::env::var("XERJ_DEBUG_RESP").is_ok() {
- eprintln!("--- RESP {} ---\n{}\n--- END ---", status, &resp_text[..resp_text.len().min(1000)]);
+ eprintln!(
+ "--- RESP {} ---\n{}\n--- END ---",
+ status,
+ &resp_text[..resp_text.len().min(1000)]
+ );
}
let mut resp_json: Value = serde_json::from_str(&resp_text).unwrap_or(Value::Null);
@@ -725,7 +778,13 @@ fn execute_do(
// 404 on delete = idempotent cleanup (OK)
// 409 on create = index exists from prior test (OK after cleanup)
if !action_name.contains("delete") && !action_name.contains("create") {
- return Err(format!("{} {} → {} {}", method, path, status, &resp_text[..resp_text.len().min(200)]));
+ return Err(format!(
+ "{} {} → {} {}",
+ method,
+ path,
+ status,
+ &resp_text[..resp_text.len().min(200)]
+ ));
}
}
@@ -740,7 +799,12 @@ fn execute_do(
// collide with URL path semantics. Percent-encode them so the server sees
// one path segment. Also handle `+` for index name addition syntax.
fn encode_index_seg(s: &str) -> String {
- if !s.contains('<') && !s.contains('|') && !s.contains('>') && !s.contains('{') && !s.contains('}') {
+ if !s.contains('<')
+ && !s.contains('|')
+ && !s.contains('>')
+ && !s.contains('{')
+ && !s.contains('}')
+ {
return s.to_string();
}
let mut out = String::with_capacity(s.len() * 2);
@@ -770,30 +834,63 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
"indices.delete" => {
let mut path = format!("/{}", index);
let mut qp = Vec::new();
- if let Some(v) = get_param_str(params, "ignore_unavailable") { qp.push(format!("ignore_unavailable={}", v)); }
- if let Some(v) = get_param_str(params, "allow_no_indices") { qp.push(format!("allow_no_indices={}", v)); }
- if let Some(v) = get_param_str(params, "expand_wildcards") { qp.push(format!("expand_wildcards={}", v)); }
- if !qp.is_empty() { path.push('?'); path.push_str(&qp.join("&")); }
+ if let Some(v) = get_param_str(params, "ignore_unavailable") {
+ qp.push(format!("ignore_unavailable={}", v));
+ }
+ if let Some(v) = get_param_str(params, "allow_no_indices") {
+ qp.push(format!("allow_no_indices={}", v));
+ }
+ if let Some(v) = get_param_str(params, "expand_wildcards") {
+ qp.push(format!("expand_wildcards={}", v));
+ }
+ if !qp.is_empty() {
+ path.push('?');
+ path.push_str(&qp.join("&"));
+ }
("DELETE".into(), path, None)
}
- "indices.refresh" => ("POST".into(), format!("/{}/_refresh", if index.is_empty() { "*" } else { &index }), None),
+ "indices.refresh" => (
+ "POST".into(),
+ format!("/{}/_refresh", if index.is_empty() { "*" } else { &index }),
+ None,
+ ),
"indices.disk_usage" => {
let mut path = format!("/{}/_disk_usage", index);
let mut qp = Vec::new();
- if let Some(v) = get_param_str(params, "run_expensive_tasks") { qp.push(format!("run_expensive_tasks={}", v)); }
- if let Some(v) = get_param_str(params, "flush") { qp.push(format!("flush={}", v)); }
- if !qp.is_empty() { path.push('?'); path.push_str(&qp.join("&")); }
+ if let Some(v) = get_param_str(params, "run_expensive_tasks") {
+ qp.push(format!("run_expensive_tasks={}", v));
+ }
+ if let Some(v) = get_param_str(params, "flush") {
+ qp.push(format!("flush={}", v));
+ }
+ if !qp.is_empty() {
+ path.push('?');
+ path.push_str(&qp.join("&"));
+ }
("POST".into(), path, None)
- },
+ }
"indices.get" => {
let mut path = format!("/{}", index);
let mut qp = Vec::new();
- if let Some(v) = get_param_str(params, "human") { qp.push(format!("human={}", v)); }
- if let Some(v) = get_param_str(params, "features") { qp.push(format!("features={}", v)); }
- if let Some(v) = get_param_str(params, "ignore_unavailable") { qp.push(format!("ignore_unavailable={}", v)); }
- if let Some(v) = get_param_str(params, "allow_no_indices") { qp.push(format!("allow_no_indices={}", v)); }
- if let Some(v) = get_param_str(params, "expand_wildcards") { qp.push(format!("expand_wildcards={}", v)); }
- if !qp.is_empty() { path.push('?'); path.push_str(&qp.join("&")); }
+ if let Some(v) = get_param_str(params, "human") {
+ qp.push(format!("human={}", v));
+ }
+ if let Some(v) = get_param_str(params, "features") {
+ qp.push(format!("features={}", v));
+ }
+ if let Some(v) = get_param_str(params, "ignore_unavailable") {
+ qp.push(format!("ignore_unavailable={}", v));
+ }
+ if let Some(v) = get_param_str(params, "allow_no_indices") {
+ qp.push(format!("allow_no_indices={}", v));
+ }
+ if let Some(v) = get_param_str(params, "expand_wildcards") {
+ qp.push(format!("expand_wildcards={}", v));
+ }
+ if !qp.is_empty() {
+ path.push('?');
+ path.push_str(&qp.join("&"));
+ }
("GET".into(), path, None)
}
"indices.put_mapping" => ("PUT".into(), format!("/{}/_mapping", index), body),
@@ -853,7 +950,11 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
"indices.put_settings" => ("PUT".into(), format!("/{}/_settings", index), body),
"indices.get_field_mapping" => {
let fields = get_param_str(params, "fields").unwrap_or_default();
- ("GET".into(), format!("/{}/_mapping/field/{}", index, fields), None)
+ (
+ "GET".into(),
+ format!("/{}/_mapping/field/{}", index, fields),
+ None,
+ )
}
"indices.put_index_template" | "indices.put_index_template.json" => {
let name = get_param_str(params, "name").unwrap_or_default();
@@ -879,17 +980,41 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
// Document operations
"index" => {
let mut qp: Vec<String> = Vec::new();
- if let Some(v) = get_param_str(params, "refresh") { qp.push(format!("refresh={}", v)); }
- if let Some(v) = get_param_str(params, "routing") { qp.push(format!("routing={}", v)); }
- if let Some(v) = get_param_str(params, "pipeline") { qp.push(format!("pipeline={}", v)); }
- if let Some(v) = get_param_str(params, "version") { qp.push(format!("version={}", v)); }
- if let Some(v) = get_param_str(params, "version_type") { qp.push(format!("version_type={}", v)); }
- if let Some(v) = get_param_str(params, "op_type") { qp.push(format!("op_type={}", v)); }
- if let Some(v) = get_param_str(params, "if_seq_no") { qp.push(format!("if_seq_no={}", v)); }
- if let Some(v) = get_param_str(params, "if_primary_term") { qp.push(format!("if_primary_term={}", v)); }
- if let Some(v) = get_param_str(params, "timeout") { qp.push(format!("timeout={}", v)); }
- if let Some(v) = get_param_str(params, "require_alias") { qp.push(format!("require_alias={}", v)); }
- let qs = if qp.is_empty() { String::new() } else { format!("?{}", qp.join("&")) };
+ if let Some(v) = get_param_str(params, "refresh") {
+ qp.push(format!("refresh={}", v));
+ }
+ if let Some(v) = get_param_str(params, "routing") {
+ qp.push(format!("routing={}", v));
+ }
+ if let Some(v) = get_param_str(params, "pipeline") {
+ qp.push(format!("pipeline={}", v));
+ }
+ if let Some(v) = get_param_str(params, "version") {
+ qp.push(format!("version={}", v));
+ }
+ if let Some(v) = get_param_str(params, "version_type") {
+ qp.push(format!("version_type={}", v));
+ }
+ if let Some(v) = get_param_str(params, "op_type") {
+ qp.push(format!("op_type={}", v));
+ }
+ if let Some(v) = get_param_str(params, "if_seq_no") {
+ qp.push(format!("if_seq_no={}", v));
+ }
+ if let Some(v) = get_param_str(params, "if_primary_term") {
+ qp.push(format!("if_primary_term={}", v));
+ }
+ if let Some(v) = get_param_str(params, "timeout") {
+ qp.push(format!("timeout={}", v));
+ }
+ if let Some(v) = get_param_str(params, "require_alias") {
+ qp.push(format!("require_alias={}", v));
+ }
+ let qs = if qp.is_empty() {
+ String::new()
+ } else {
+ format!("?{}", qp.join("&"))
+ };
if id.is_empty() {
("POST".into(), format!("/{}/_doc{}", index, qs), body)
} else {
@@ -910,44 +1035,112 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
};
// Forward query params that ES YAML tests use
let mut qp = Vec::new();
- if let Some(v) = get_param_str(params, "rest_total_hits_as_int") { qp.push(format!("rest_total_hits_as_int={}", v)); }
- if let Some(v) = get_param_str(params, "track_total_hits") { qp.push(format!("track_total_hits={}", v)); }
- if let Some(v) = get_param_str(params, "scroll") { qp.push(format!("scroll={}", v)); }
- if let Some(v) = get_param_str(params, "size") { qp.push(format!("size={}", v)); }
- if let Some(v) = get_param_str(params, "from") { qp.push(format!("from={}", v)); }
- if let Some(v) = get_param_str(params, "sort") { qp.push(format!("sort={}", v)); }
- if let Some(v) = get_param_str(params, "_source") { qp.push(format!("_source={}", v)); }
- if let Some(v) = get_param_str(params, "_source_includes") { qp.push(format!("_source_includes={}", v)); }
- if let Some(v) = get_param_str(params, "_source_excludes") { qp.push(format!("_source_excludes={}", v)); }
- if let Some(v) = get_param_str(params, "typed_keys") { qp.push(format!("typed_keys={}", v)); }
- if let Some(v) = get_param_str(params, "docvalue_fields") { qp.push(format!("docvalue_fields={}", v)); }
- if let Some(v) = get_param_str(params, "stored_fields") { qp.push(format!("stored_fields={}", v)); }
- if let Some(v) = get_param_str(params, "q") { qp.push(format!("q={}", v)); }
- if let Some(v) = get_param_str(params, "df") { qp.push(format!("df={}", v)); }
- if let Some(v) = get_param_str(params, "default_operator") { qp.push(format!("default_operator={}", v)); }
- if let Some(v) = get_param_str(params, "pre_filter_shard_size") { qp.push(format!("pre_filter_shard_size={}", v)); }
- if let Some(v) = get_param_str(params, "batched_reduce_size") { qp.push(format!("batched_reduce_size={}", v)); }
- if let Some(v) = get_param_str(params, "ccs_minimize_roundtrips") { qp.push(format!("ccs_minimize_roundtrips={}", v)); }
- if let Some(v) = get_param_str(params, "explain") { qp.push(format!("explain={}", v)); }
- if let Some(v) = get_param_str(params, "ignore_unavailable") { qp.push(format!("ignore_unavailable={}", v)); }
- if let Some(v) = get_param_str(params, "allow_no_indices") { qp.push(format!("allow_no_indices={}", v)); }
- if let Some(v) = get_param_str(params, "expand_wildcards") { qp.push(format!("expand_wildcards={}", v)); }
- if let Some(v) = get_param_str(params, "seq_no_primary_term") { qp.push(format!("seq_no_primary_term={}", v)); }
- if let Some(v) = get_param_str(params, "version") { qp.push(format!("version={}", v)); }
- if let Some(v) = get_param_str(params, "preference") { qp.push(format!("preference={}", v)); }
- if let Some(v) = get_param_str(params, "routing") { qp.push(format!("routing={}", v)); }
- if let Some(v) = get_param_str(params, "search_type") { qp.push(format!("search_type={}", v)); }
- if let Some(v) = get_param_str(params, "request_cache") { qp.push(format!("request_cache={}", v)); }
+ if let Some(v) = get_param_str(params, "rest_total_hits_as_int") {
+ qp.push(format!("rest_total_hits_as_int={}", v));
+ }
+ if let Some(v) = get_param_str(params, "track_total_hits") {
+ qp.push(format!("track_total_hits={}", v));
+ }
+ if let Some(v) = get_param_str(params, "scroll") {
+ qp.push(format!("scroll={}", v));
+ }
+ if let Some(v) = get_param_str(params, "size") {
+ qp.push(format!("size={}", v));
+ }
+ if let Some(v) = get_param_str(params, "from") {
+ qp.push(format!("from={}", v));
+ }
+ if let Some(v) = get_param_str(params, "sort") {
+ qp.push(format!("sort={}", v));
+ }
+ if let Some(v) = get_param_str(params, "_source") {
+ qp.push(format!("_source={}", v));
+ }
+ if let Some(v) = get_param_str(params, "_source_includes") {
+ qp.push(format!("_source_includes={}", v));
+ }
+ if let Some(v) = get_param_str(params, "_source_excludes") {
+ qp.push(format!("_source_excludes={}", v));
+ }
+ if let Some(v) = get_param_str(params, "typed_keys") {
+ qp.push(format!("typed_keys={}", v));
+ }
+ if let Some(v) = get_param_str(params, "docvalue_fields") {
+ qp.push(format!("docvalue_fields={}", v));
+ }
+ if let Some(v) = get_param_str(params, "stored_fields") {
+ qp.push(format!("stored_fields={}", v));
+ }
+ if let Some(v) = get_param_str(params, "q") {
+ qp.push(format!("q={}", v));
+ }
+ if let Some(v) = get_param_str(params, "df") {
+ qp.push(format!("df={}", v));
+ }
+ if let Some(v) = get_param_str(params, "default_operator") {
+ qp.push(format!("default_operator={}", v));
+ }
+ if let Some(v) = get_param_str(params, "pre_filter_shard_size") {
+ qp.push(format!("pre_filter_shard_size={}", v));
+ }
+ if let Some(v) = get_param_str(params, "batched_reduce_size") {
+ qp.push(format!("batched_reduce_size={}", v));
+ }
+ if let Some(v) = get_param_str(params, "ccs_minimize_roundtrips") {
+ qp.push(format!("ccs_minimize_roundtrips={}", v));
+ }
+ if let Some(v) = get_param_str(params, "explain") {
+ qp.push(format!("explain={}", v));
+ }
+ if let Some(v) = get_param_str(params, "ignore_unavailable") {
+ qp.push(format!("ignore_unavailable={}", v));
+ }
+ if let Some(v) = get_param_str(params, "allow_no_indices") {
+ qp.push(format!("allow_no_indices={}", v));
+ }
+ if let Some(v) = get_param_str(params, "expand_wildcards") {
+ qp.push(format!("expand_wildcards={}", v));
+ }
+ if let Some(v) = get_param_str(params, "seq_no_primary_term") {
+ qp.push(format!("seq_no_primary_term={}", v));
+ }
+ if let Some(v) = get_param_str(params, "version") {
+ qp.push(format!("version={}", v));
+ }
+ if let Some(v) = get_param_str(params, "preference") {
+ qp.push(format!("preference={}", v));
+ }
+ if let Some(v) = get_param_str(params, "routing") {
+ qp.push(format!("routing={}", v));
+ }
+ if let Some(v) = get_param_str(params, "search_type") {
+ qp.push(format!("search_type={}", v));
+ }
+ if let Some(v) = get_param_str(params, "request_cache") {
+ qp.push(format!("request_cache={}", v));
+ }
if let Some(v) = get_param_str(params, "filter_path") {
// Commas and `*` are part of filter_path syntax; encode
// only what reqwest or the server would choke on.
- let e = v.replace(' ', "%20").replace('#', "%23").replace('&', "%26");
+ let e = v
+ .replace(' ', "%20")
+ .replace('#', "%23")
+ .replace('&', "%26");
qp.push(format!("filter_path={}", e));
}
- if let Some(v) = get_param_str(params, "allow_partial_search_results") { qp.push(format!("allow_partial_search_results={}", v)); }
- if let Some(v) = get_param_str(params, "include_named_queries_score") { qp.push(format!("include_named_queries_score={}", v)); }
- if let Some(v) = get_param_str(params, "force_synthetic_source") { qp.push(format!("force_synthetic_source={}", v)); }
- if !qp.is_empty() { path.push('?'); path.push_str(&qp.join("&")); }
+ if let Some(v) = get_param_str(params, "allow_partial_search_results") {
+ qp.push(format!("allow_partial_search_results={}", v));
+ }
+ if let Some(v) = get_param_str(params, "include_named_queries_score") {
+ qp.push(format!("include_named_queries_score={}", v));
+ }
+ if let Some(v) = get_param_str(params, "force_synthetic_source") {
+ qp.push(format!("force_synthetic_source={}", v));
+ }
+ if !qp.is_empty() {
+ path.push('?');
+ path.push_str(&qp.join("&"));
+ }
("POST".into(), path, body)
}
"count" => {
@@ -958,26 +1151,46 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
};
let mut qp = Vec::new();
if let Some(v) = get_param_str(params, "filter_path") {
- let e = v.replace(' ', "%20").replace('#', "%23").replace('&', "%26");
+ let e = v
+ .replace(' ', "%20")
+ .replace('#', "%23")
+ .replace('&', "%26");
qp.push(format!("filter_path={}", e));
}
- if !qp.is_empty() { path.push('?'); path.push_str(&qp.join("&")); }
+ if !qp.is_empty() {
+ path.push('?');
+ path.push_str(&qp.join("&"));
+ }
("POST".into(), path, body)
}
"scroll" => {
let mut path = "/_search/scroll".to_string();
let mut qp = Vec::new();
- if let Some(v) = get_param_str(params, "rest_total_hits_as_int") { qp.push(format!("rest_total_hits_as_int={}", v)); }
- if let Some(v) = get_param_str(params, "scroll_id") { qp.push(format!("scroll_id={}", v)); }
- if let Some(v) = get_param_str(params, "scroll") { qp.push(format!("scroll={}", v)); }
- if !qp.is_empty() { path.push('?'); path.push_str(&qp.join("&")); }
+ if let Some(v) = get_param_str(params, "rest_total_hits_as_int") {
+ qp.push(format!("rest_total_hits_as_int={}", v));
+ }
+ if let Some(v) = get_param_str(params, "scroll_id") {
+ qp.push(format!("scroll_id={}", v));
+ }
+ if let Some(v) = get_param_str(params, "scroll") {
+ qp.push(format!("scroll={}", v));
+ }
+ if !qp.is_empty() {
+ path.push('?');
+ path.push_str(&qp.join("&"));
+ }
("POST".into(), path, body)
}
"clear_scroll" => {
let mut path = "/_search/scroll".to_string();
let mut qp = Vec::new();
- if let Some(v) = get_param_str(params, "scroll_id") { qp.push(format!("scroll_id={}", v)); }
- if !qp.is_empty() { path.push('?'); path.push_str(&qp.join("&")); }
+ if let Some(v) = get_param_str(params, "scroll_id") {
+ qp.push(format!("scroll_id={}", v));
+ }
+ if !qp.is_empty() {
+ path.push('?');
+ path.push_str(&qp.join("&"));
+ }
("DELETE".into(), path, body)
}
@@ -989,15 +1202,34 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
format!("/{}/_bulk", index)
};
let mut qp = Vec::new();
- if let Some(v) = get_param_str(params, "refresh") { qp.push(format!("refresh={}", v)); }
- if let Some(v) = get_param_str(params, "require_alias") { qp.push(format!("require_alias={}", v)); }
- if let Some(v) = get_param_str(params, "routing") { qp.push(format!("routing={}", v)); }
- if let Some(v) = get_param_str(params, "pipeline") { qp.push(format!("pipeline={}", v)); }
- if let Some(v) = get_param_str(params, "timeout") { qp.push(format!("timeout={}", v)); }
- if let Some(v) = get_param_str(params, "_source") { qp.push(format!("_source={}", v)); }
- if let Some(v) = get_param_str(params, "_source_includes") { qp.push(format!("_source_includes={}", v)); }
- if let Some(v) = get_param_str(params, "_source_excludes") { qp.push(format!("_source_excludes={}", v)); }
- if !qp.is_empty() { path.push('?'); path.push_str(&qp.join("&")); }
+ if let Some(v) = get_param_str(params, "refresh") {
+ qp.push(format!("refresh={}", v));
+ }
+ if let Some(v) = get_param_str(params, "require_alias") {
+ qp.push(format!("require_alias={}", v));
+ }
+ if let Some(v) = get_param_str(params, "routing") {
+ qp.push(format!("routing={}", v));
+ }
+ if let Some(v) = get_param_str(params, "pipeline") {
+ qp.push(format!("pipeline={}", v));
+ }
+ if let Some(v) = get_param_str(params, "timeout") {
+ qp.push(format!("timeout={}", v));
+ }
+ if let Some(v) = get_param_str(params, "_source") {
+ qp.push(format!("_source={}", v));
+ }
+ if let Some(v) = get_param_str(params, "_source_includes") {
+ qp.push(format!("_source_includes={}", v));
+ }
+ if let Some(v) = get_param_str(params, "_source_excludes") {
+ qp.push(format!("_source_excludes={}", v));
+ }
+ if !qp.is_empty() {
+ path.push('?');
+ path.push_str(&qp.join("&"));
+ }
let ndjson = get_param_ndjson(params);
("POST".into(), path, ndjson)
}
@@ -1009,15 +1241,34 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
path = format!("/_cluster/health/{}", index);
}
let mut qp = Vec::new();
- if let Some(v) = get_param_str(params, "level") { qp.push(format!("level={}", v)); }
- if let Some(v) = get_param_str(params, "wait_for_status") { qp.push(format!("wait_for_status={}", v)); }
- if let Some(v) = get_param_str(params, "wait_for_no_relocating_shards") { qp.push(format!("wait_for_no_relocating_shards={}", v)); }
- if let Some(v) = get_param_str(params, "wait_for_no_initializing_shards") { qp.push(format!("wait_for_no_initializing_shards={}", v)); }
- if let Some(v) = get_param_str(params, "wait_for_active_shards") { qp.push(format!("wait_for_active_shards={}", v)); }
- if let Some(v) = get_param_str(params, "wait_for_nodes") { qp.push(format!("wait_for_nodes={}", v)); }
- if let Some(v) = get_param_str(params, "expand_wildcards") { qp.push(format!("expand_wildcards={}", v)); }
- if let Some(v) = get_param_str(params, "timeout") { qp.push(format!("timeout={}", v)); }
- if !qp.is_empty() { path.push('?'); path.push_str(&qp.join("&")); }
+ if let Some(v) = get_param_str(params, "level") {
+ qp.push(format!("level={}", v));
+ }
+ if let Some(v) = get_param_str(params, "wait_for_status") {
+ qp.push(format!("wait_for_status={}", v));
+ }
+ if let Some(v) = get_param_str(params, "wait_for_no_relocating_shards") {
+ qp.push(format!("wait_for_no_relocating_shards={}", v));
+ }
+ if let Some(v) = get_param_str(params, "wait_for_no_initializing_shards") {
+ qp.push(format!("wait_for_no_initializing_shards={}", v));
+ }
+ if let Some(v) = get_param_str(params, "wait_for_active_shards") {
+ qp.push(format!("wait_for_active_shards={}", v));
+ }
+ if let Some(v) = get_param_str(params, "wait_for_nodes") {
+ qp.push(format!("wait_for_nodes={}", v));
+ }
+ if let Some(v) = get_param_str(params, "expand_wildcards") {
+ qp.push(format!("expand_wildcards={}", v));
+ }
+ if let Some(v) = get_param_str(params, "timeout") {
+ qp.push(format!("timeout={}", v));
+ }
+ if !qp.is_empty() {
+ path.push('?');
+ path.push_str(&qp.join("&"));
+ }
("GET".into(), path, None)
}
"cluster.state" => ("GET".into(), "/_cluster/state".into(), None),
@@ -1033,7 +1284,9 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
"nodes.stats" => ("GET".into(), "/_nodes/stats".into(), None),
// Internal cluster APIs
- "_internal.get_desired_balance" => ("GET".into(), "/_internal/desired_balance".into(), None),
+ "_internal.get_desired_balance" => {
+ ("GET".into(), "/_internal/desired_balance".into(), None)
+ }
// Ingest
"ingest.put_pipeline" => {
@@ -1052,8 +1305,14 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
format!("/_ingest/pipeline/{}/_simulate", pid)
};
let mut qp = Vec::new();
- if let Some(v) = get_param_str(params, "verbose") { qp.push(format!("verbose={}", v)); }
- let final_path = if qp.is_empty() { path } else { format!("{}?{}", path, qp.join("&")) };
+ if let Some(v) = get_param_str(params, "verbose") {
+ qp.push(format!("verbose={}", v));
+ }
+ let final_path = if qp.is_empty() {
+ path
+ } else {
+ format!("{}?{}", path, qp.join("&"))
+ };
("POST".into(), final_path, body)
}
"ingest.get_pipeline" => {
@@ -1098,7 +1357,7 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
}
fn get_param_str(params: &serde_yaml::Mapping, key: &str) -> Option<String> {
- params.get(&YamlValue::String(key.into())).map(|v| {
+ params.get(YamlValue::String(key.into())).map(|v| {
match v {
// YAML arrays like [test_1, test_2] → comma-separated "test_1,test_2"
YamlValue::Sequence(items) => items
@@ -1113,7 +1372,7 @@ fn get_param_str(params: &serde_yaml::Mapping, key: &str) -> Option<String> {
fn get_param_body(params: &serde_yaml::Mapping) -> Option<String> {
params
- .get(&YamlValue::String("body".into()))
+ .get(YamlValue::String("body".into()))
.map(|v| match v {
// A YAML string that already looks like a JSON object/array is
// passed through unchanged — ES YAML tests use `body: > {...}`
@@ -1132,7 +1391,7 @@ fn get_param_body(params: &serde_yaml::Mapping) -> Option<String> {
}
fn get_param_ndjson(params: &serde_yaml::Mapping) -> Option<String> {
- if let Some(YamlValue::Sequence(items)) = params.get(&YamlValue::String("body".into())) {
+ if let Some(YamlValue::Sequence(items)) = params.get(YamlValue::String("body".into())) {
let mut ndjson = String::new();
for item in items {
match item {
@@ -1149,7 +1408,9 @@ fn get_param_ndjson(params: &serde_yaml::Mapping) -> Option<String> {
// doesn't parse, fall back to replacing newlines
// with spaces so the JSON remains intact.
let compact = match serde_json::from_str::<serde_json::Value>(s) {
- Ok(v) => serde_json::to_string(&v).unwrap_or_else(|_| s.replace('\n', " ")),
+ Ok(v) => {
+ serde_json::to_string(&v).unwrap_or_else(|_| s.replace('\n', " "))
+ }
Err(_) => s.replace('\n', " "),
};
ndjson.push_str(&compact);
@@ -1277,7 +1538,9 @@ fn values_match(actual: &Value, expected: &Value) -> bool {
(Value::Number(a), Value::Number(b)) => {
let af = a.as_f64().unwrap_or(0.0);
let bf = b.as_f64().unwrap_or(0.0);
- if af == bf { return true; }
+ if af == bf {
+ return true;
+ }
// serde_yaml and serde_json disagree by up to 1 ULP on the
// parse of identical decimal literals (serde_json's parser
// is not always correctly rounded). Treat numbers that are
@@ -1310,20 +1573,16 @@ fn values_match(actual: &Value, expected: &Value) -> bool {
a == b
}
}
- (Value::String(a), Value::Number(b)) => {
- a.parse::<f64>().ok() == b.as_f64()
- }
- (Value::Number(a), Value::String(b)) => {
- b.parse::<f64>().ok() == a.as_f64()
- }
+ (Value::String(a), Value::Number(b)) => a.parse::<f64>().ok() == b.as_f64(),
+ (Value::Number(a), Value::String(b)) => b.parse::<f64>().ok() == a.as_f64(),
(Value::Bool(a), Value::Bool(b)) => a == b,
(Value::Null, Value::Null) => true,
(Value::Array(a), Value::Array(b)) => {
a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| values_match(x, y))
}
- (Value::Object(a), Value::Object(b)) => {
- b.iter().all(|(k, v)| a.get(k).map_or(false, |av| values_match(av, v)))
- }
+ (Value::Object(a), Value::Object(b)) => b
+ .iter()
+ .all(|(k, v)| a.get(k).is_some_and(|av| values_match(av, v))),
_ => actual == expected,
}
}