AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(es-compat): API-fidelity minors — _cat/indices uuid+bytes, pattern route, snapshot shape (item 13)
xerj-org573e5035116d
diff --git a/docs/examples/migrate-from-elasticsearch/migrate_demo.sh b/docs/examples/migrate-from-elasticsearch/migrate_demo.shindex 5068e6837..970ddc9bc 100755--- a/docs/examples/migrate-from-elasticsearch/migrate_demo.sh+++ b/docs/examples/migrate-from-elasticsearch/migrate_demo.sh@@ -113,14 +113,11 @@ for b in agg["buckets"]:'# 6) _cat/indices — operational sanity check, human-readable table.-# Note: XERJ serves the whole-cluster _cat/indices listing (which also-# includes its .xerj_* system indices). It does NOT serve the per-index-# _cat/indices/<name> path (404), so filter client-side as shown.-say "6. GET /_cat/indices?v (ops table — includes .xerj_* system indices)"+say "6. GET /_cat/indices?v (ops table)"curl -s "$ES/_cat/indices?v"echo-echo "--- your index only (client-side filter) ---"-curl -s "$ES/_cat/indices?v" | grep -E ' products ' || true+echo "--- your index only (server-side, standard ES path) ---"+curl -s "$ES/_cat/indices/products?v"echoecho "All standard Elasticsearch calls succeeded against XERJ at $ES"diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex a3c1cb970..1aef28304 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -389,26 +389,89 @@ async fn cluster_health_inner(// ─────────────────────────────────────────────────────────────────────────────pub async fn cat_indices(State(state): State<AppState>) -> impl IntoResponse {+ cat_indices_inner(state, None).await+}++/// `GET /_cat/indices/{pattern}` — the per-index / pattern-scoped variant.+///+/// Historically 404 (only the whole-cluster `/_cat/indices` was routed), which+/// forced clients to fetch the entire listing and filter client-side — the+/// exact workaround baked into our own migrate example. ES serves this path+/// directly, so we do too.+pub async fn cat_indices_pattern(+ State(state): State<AppState>,+ Path(pattern): Path<String>,+) -> impl IntoResponse {+ cat_indices_inner(state, Some(pattern)).await+}++async fn cat_indices_inner(state: AppState, pattern: Option<String>) -> axum::response::Response {let indices = state.engine.list_indices().await;++ // Narrow to the path pattern when present (`/_cat/indices/{pattern}`):+ // a comma-separated list of concrete names and/or `*` globs. ES rules:+ // • a CONCRETE name matching nothing → 404 index_not_found_exception+ // • a WILDCARD matching nothing → empty 200+ // Absent pattern lists the whole cluster (unchanged behavior).+ let selected: Vec<&xerj_engine::engine::IndexInfo> = match pattern.as_deref() {+ None => indices.iter().collect(),+ Some(pat) => {+ let parts: Vec<&str> = pat+ .split(',')+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .collect();+ for &p in &parts {+ let is_wildcard = p == "_all" || p == "*" || p.contains('*');+ if !is_wildcard && !indices.iter().any(|i| i.name == p) {+ return ApiError::new(xerj_common::XerjError::index_not_found(p))+ .into_response();+ }+ }+ indices+ .iter()+ .filter(|info| {+ parts.iter().any(|&p| {+ p == "_all"+ || p == "*"+ || info.name == p+ || glob_match_simple(p, &info.name)+ })+ })+ .collect()+ }+ };+let mut lines = Vec::new();- for info in &indices {+ for info in &selected {// Real on-disk size: recursive byte sum of the index's data_dir.- // Single-shard (pri=1, rep=0), so store.size == pri.store.size.+ // Single-shard (pri=1, rep=0), so store.size == pri.store.size, and+ // dataset.size (the 8.13 column) equals store.size for a fully-mounted+ // (non-partial, non-searchable-snapshot) shard.let size = state.engine.get_index(&info.name).map(|idx| dir_size_bytes(idx.data_dir())).unwrap_or(0);+ let hsize = human_bytes(size);lines.push(format!(- "green open {} {} 1 0 {} 0 {}b {}b",+ "green open {} {} 1 0 {} 0 {} {} {}",info.name,- Uuid::new_v4(),+ // Emit the stable REAL settings uuid (the same value+ // GET /{index}/_settings returns), not a fresh random one per call.+ stable_index_uuid(&info.name),info.doc_count,- size,- size,+ hsize,+ hsize,+ hsize,));}- let body = lines.join("\n") + "\n";+ // ES returns an empty body (not a bare newline) when nothing matches.+ let body = if lines.is_empty() {+ String::new()+ } else {+ lines.join("\n") + "\n"+ };(StatusCode::OK,[(@@ -420,6 +483,54 @@ pub async fn cat_indices(State(state): State<AppState>) -> impl IntoResponse {.into_response()}+/// Format a raw byte count the way ES `_cat` columns do (`ByteSizeValue`):+/// the largest unit that keeps the value ≥ 1, one decimal place with a+/// trailing `.0` dropped; plain bytes carry no decimal (`260b`, `5.1kb`,+/// `1.9mb`). Previously `_cat/indices` emitted raw `{n}b` for every size.+fn human_bytes(bytes: u64) -> String {+ const KB: f64 = 1024.0;+ const MB: f64 = 1024.0 * 1024.0;+ const GB: f64 = 1024.0 * 1024.0 * 1024.0;+ const TB: f64 = 1024.0 * 1024.0 * 1024.0 * 1024.0;+ let b = bytes as f64;+ let (val, unit) = if b >= TB {+ (b / TB, "tb")+ } else if b >= GB {+ (b / GB, "gb")+ } else if b >= MB {+ (b / MB, "mb")+ } else if b >= KB {+ (b / KB, "kb")+ } else {+ return format!("{bytes}b");+ };+ let s = format!("{val:.1}");+ let s = s.strip_suffix(".0").unwrap_or(&s);+ format!("{s}{unit}")+}++/// A stable, deterministic UUID for an index, derived from its name.+///+/// `_cat/indices` used to mint a fresh `Uuid::new_v4()` on every request, so+/// the uuid column changed each call and never matched the `index.uuid`+/// returned by `GET /{index}/_settings`. ES returns the same real settings+/// uuid in both places. We derive it deterministically from the index name so+/// both endpoints agree and the value is stable across requests and restarts+/// (no persistence path needed). `DefaultHasher` uses fixed keys, so this is+/// reproducible across processes.+fn stable_index_uuid(name: &str) -> String {+ use std::hash::{Hash, Hasher};+ let mut hi = std::collections::hash_map::DefaultHasher::new();+ name.hash(&mut hi);+ let high = hi.finish();+ let mut lo = std::collections::hash_map::DefaultHasher::new();+ // Distinct salt so the two 64-bit halves are independent.+ 0x9E37_79B9_7F4A_7C15u64.hash(&mut lo);+ name.hash(&mut lo);+ let low = lo.finish();+ Uuid::from_u128(((high as u128) << 64) | (low as u128)).to_string()+}+// ─────────────────────────────────────────────────────────────────────────────// PUT /{index}// ─────────────────────────────────────────────────────────────────────────────@@ -1004,6 +1115,13 @@ fn merge_settings_defaults(user_settings: &Value, provided_name: &str, human: boif let Some(obj) = raw_inner.as_object() {for (k, v) in obj {+ // Internal bookkeeping keys (index-sort hints written at create+ // time) are not ES index settings — never surface them in the+ // settings response. They were leaking into GET /{index}/_settings+ // for any index with an @timestamp-derived or explicit index sort.+ if k.starts_with("__xy_") {+ continue;+ }// ES coerces numeric settings to strings in the response.let coerced = match v {Value::Number(n) => Value::String(n.to_string()),@@ -1019,9 +1137,12 @@ fn merge_settings_defaults(user_settings: &Value, provided_name: &str, human: boinner.entry("creation_date".to_string()).or_insert_with(|| Value::String(now_ms.to_string()));- inner- .entry("uuid".to_string())- .or_insert_with(|| Value::String(Uuid::new_v4().to_string()));+ // Emit the stable, deterministic index uuid — the SAME value+ // _cat/indices reports — rather than a fresh random one per request.+ inner.insert(+ "uuid".to_string(),+ Value::String(stable_index_uuid(provided_name)),+ );let version = inner.entry("version".to_string()).or_insert_with(|| json!({ "created": "8130099" }));diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rsindex 39f5775b5..58c5af1bd 100644--- a/engine/crates/xerj-api/src/router.rs+++ b/engine/crates/xerj-api/src/router.rs@@ -165,6 +165,7 @@ pub fn build_native_router(state: AppState) -> Router {/// GET / es_info/// GET /_cluster/health cluster_health/// GET /_cat/indices cat_indices+/// GET /_cat/indices/:pattern cat_indices_pattern/// POST /_bulk global_bulk/// PUT /:index create_index/// DELETE /:index delete_index@@ -198,6 +199,10 @@ pub fn build_es_compat_router(state: AppState) -> Router {get(es_compat::cluster_health_for_index),).route("/_cat/indices", get(es_compat::cat_indices))+ .route(+ "/_cat/indices/:pattern",+ get(es_compat::cat_indices_pattern),+ ).route("/_cat/health", get(es_compat::cat_health)).route("/_cat/nodes", get(es_compat::cat_nodes)).route("/_cat/aliases", get(es_compat::cat_aliases))diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 8da25acbb..d3ea85ce9 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -1175,13 +1175,25 @@ impl Engine {let snap_dir = std::path::Path::new(repo_path).join(name);std::fs::create_dir_all(&snap_dir).map_err(EngineError::Io)?;+ // Wall-clock start, captured BEFORE the flush+copy work so+ // duration_in_millis reflects the real elapsed time (it was hardcoded+ // to 0 because start_time == end_time were sampled at the same instant+ // after the copy).+ let start_ms = now_millis();+let target_indices: Vec<String> = match indices {Some(list) if !list.is_empty() => list,- _ => self.indices.iter().map(|e| e.key().clone()).collect(),+ // ES excludes system indices from an all-indices snapshot; exclude+ // our `.xerj_*` internals (auth, sessions, dashboards, …) by+ // default so a plain snapshot captures only user data.+ _ => self+ .indices+ .iter()+ .map(|e| e.key().clone())+ .filter(|n| !is_system_index(n))+ .collect(),};- let mut manifest_indices: Vec<Value> = Vec::new();-for idx_name in &target_indices {let idx = match self.indices.get(idx_name.as_str()) {Some(i) => i,@@ -1195,21 +1207,16 @@ impl Engine {let dst_dir = snap_dir.join(idx_name);std::fs::create_dir_all(&dst_dir).map_err(EngineError::Io)?;+ // Copy everything recursively (WAL + segments + schema). The+ // per-file list is intentionally discarded: it was serialized into+ // the response as `index_files` (~170 KB of dead weight that ES+ // never returns and `restore_snapshot` never reads — restore copies+ // the snapshot dir back wholesale and reopens by index name).let mut files: Vec<String> = Vec::new();-- // Copy everything recursively (WAL + segments + schema).copy_dir_recursive(&src_dir, &dst_dir, &mut files).map_err(EngineError::Io)?;-- manifest_indices.push(serde_json::json!({- "name": idx_name,- "files": files,- }));}- let now_ms = std::time::SystemTime::now()- .duration_since(std::time::UNIX_EPOCH)- .unwrap_or_default()- .as_millis() as i64;+ let end_ms = now_millis();let manifest = serde_json::json!({"snapshot": name,@@ -1217,16 +1224,15 @@ impl Engine {"version": "8.13.0","indices": target_indices,"state": "SUCCESS",- "start_time_in_millis": now_ms,- "end_time_in_millis": now_ms,- "duration_in_millis": 0,+ "start_time_in_millis": start_ms,+ "end_time_in_millis": end_ms,+ "duration_in_millis": (end_ms - start_ms).max(0),"failures": [],"shards": {"total": target_indices.len(),"failed": 0,"successful": target_indices.len(),},- "index_files": manifest_indices,});let manifest_path = snap_dir.join("manifest.json");@@ -1286,13 +1292,25 @@ impl Engine {.collect();let index_names: Vec<String> = if patterns.is_empty() {+ // Default restore: every index in the snapshot EXCEPT `.xerj_*`+ // system indices (ES excludes system indices from a default+ // restore). Snapshots taken by this build already omit them, but a+ // snapshot written by an older build may still carry them.snapshot_indices+ .into_iter()+ .filter(|n| !is_system_index(n))+ .collect()} else {let mut selected: Vec<String> = Vec::new();for pat in &patterns {+ // A wildcard never matches a `.xerj_*` system index (ES `*`+ // semantics); only a pattern that explicitly targets the+ // dot-namespace (`.xerj_*`, `.xerj_users`, …) may.+ let targets_system = pat.starts_with('.');let matched: Vec<&String> = snapshot_indices.iter().filter(|n| glob_match(pat, n))+ .filter(|n| targets_system || !is_system_index(n)).collect();if matched.is_empty() && !pat.contains('*') {// ES: restore of an index absent from the snapshot fails@@ -1362,6 +1380,21 @@ impl Engine {// ── Private helpers ───────────────────────────────────────────────────────────+/// Milliseconds since the Unix epoch (wall clock).+fn now_millis() -> i64 {+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap_or_default()+ .as_millis() as i64+}++/// A XERJ-internal system index (`.xerj_*`: auth, sessions, dashboards,+/// audit, …). ES hides its own system indices from default snapshot/restore;+/// we do the same for ours so `_snapshot`/`_restore` operate on user data.+pub(crate) fn is_system_index(name: &str) -> bool {+ name.starts_with(".xerj_")+}+/// Recursively copy all files from `src` to `dst`, recording relative paths in `files`.fn copy_dir_recursive(src: &std::path::Path,