Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(es-compat): API-fidelity minors — _cat/indices uuid+bytes, pattern route, snapshot shape (item 13)

xerj-org573e5035116d
diff --git a/docs/examples/migrate-from-elasticsearch/migrate_demo.sh b/docs/examples/migrate-from-elasticsearch/migrate_demo.sh
index 5068e6837..970ddc9bc 100755
--- a/docs/examples/migrate-from-elasticsearch/migrate_demo.sh
+++ b/docs/examples/migrate-from-elasticsearch/migrate_demo.sh
@@ -113,14 +113,11 @@ for b in agg["buckets"]:
'
# 6) _cat/indices — operational sanity check, human-readable table.
-# Note: XERJ serves the whole-cluster _cat/indices listing (which also
-# includes its .xerj_* system indices). It does NOT serve the per-index
-# _cat/indices/<name> path (404), so filter client-side as shown.
-say "6. GET /_cat/indices?v (ops table — includes .xerj_* system indices)"
+say "6. GET /_cat/indices?v (ops table)"
curl -s "$ES/_cat/indices?v"
echo
-echo "--- your index only (client-side filter) ---"
-curl -s "$ES/_cat/indices?v" | grep -E ' products ' || true
+echo "--- your index only (server-side, standard ES path) ---"
+curl -s "$ES/_cat/indices/products?v"
echo
echo "All standard Elasticsearch calls succeeded against XERJ at $ES"
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index a3c1cb970..1aef28304 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -389,26 +389,89 @@ async fn cluster_health_inner(
// ─────────────────────────────────────────────────────────────────────────────
pub async fn cat_indices(State(state): State<AppState>) -> impl IntoResponse {
+ cat_indices_inner(state, None).await
+}
+
+/// `GET /_cat/indices/{pattern}` — the per-index / pattern-scoped variant.
+///
+/// Historically 404 (only the whole-cluster `/_cat/indices` was routed), which
+/// forced clients to fetch the entire listing and filter client-side — the
+/// exact workaround baked into our own migrate example. ES serves this path
+/// directly, so we do too.
+pub async fn cat_indices_pattern(
+ State(state): State<AppState>,
+ Path(pattern): Path<String>,
+) -> impl IntoResponse {
+ cat_indices_inner(state, Some(pattern)).await
+}
+
+async fn cat_indices_inner(state: AppState, pattern: Option<String>) -> axum::response::Response {
let indices = state.engine.list_indices().await;
+
+ // Narrow to the path pattern when present (`/_cat/indices/{pattern}`):
+ // a comma-separated list of concrete names and/or `*` globs. ES rules:
+ // • a CONCRETE name matching nothing → 404 index_not_found_exception
+ // • a WILDCARD matching nothing → empty 200
+ // Absent pattern lists the whole cluster (unchanged behavior).
+ let selected: Vec<&xerj_engine::engine::IndexInfo> = match pattern.as_deref() {
+ None => indices.iter().collect(),
+ Some(pat) => {
+ let parts: Vec<&str> = pat
+ .split(',')
+ .map(str::trim)
+ .filter(|s| !s.is_empty())
+ .collect();
+ for &p in &parts {
+ let is_wildcard = p == "_all" || p == "*" || p.contains('*');
+ if !is_wildcard && !indices.iter().any(|i| i.name == p) {
+ return ApiError::new(xerj_common::XerjError::index_not_found(p))
+ .into_response();
+ }
+ }
+ indices
+ .iter()
+ .filter(|info| {
+ parts.iter().any(|&p| {
+ p == "_all"
+ || p == "*"
+ || info.name == p
+ || glob_match_simple(p, &info.name)
+ })
+ })
+ .collect()
+ }
+ };
+
let mut lines = Vec::new();
- for info in &indices {
+ for info in &selected {
// Real on-disk size: recursive byte sum of the index's data_dir.
- // Single-shard (pri=1, rep=0), so store.size == pri.store.size.
+ // Single-shard (pri=1, rep=0), so store.size == pri.store.size, and
+ // dataset.size (the 8.13 column) equals store.size for a fully-mounted
+ // (non-partial, non-searchable-snapshot) shard.
let size = state
.engine
.get_index(&info.name)
.map(|idx| dir_size_bytes(idx.data_dir()))
.unwrap_or(0);
+ let hsize = human_bytes(size);
lines.push(format!(
- "green open {} {} 1 0 {} 0 {}b {}b",
+ "green open {} {} 1 0 {} 0 {} {} {}",
info.name,
- Uuid::new_v4(),
+ // Emit the stable REAL settings uuid (the same value
+ // GET /{index}/_settings returns), not a fresh random one per call.
+ stable_index_uuid(&info.name),
info.doc_count,
- size,
- size,
+ hsize,
+ hsize,
+ hsize,
));
}
- let body = lines.join("\n") + "\n";
+ // ES returns an empty body (not a bare newline) when nothing matches.
+ let body = if lines.is_empty() {
+ String::new()
+ } else {
+ lines.join("\n") + "\n"
+ };
(
StatusCode::OK,
[(
@@ -420,6 +483,54 @@ pub async fn cat_indices(State(state): State<AppState>) -> impl IntoResponse {
.into_response()
}
+/// Format a raw byte count the way ES `_cat` columns do (`ByteSizeValue`):
+/// the largest unit that keeps the value ≥ 1, one decimal place with a
+/// trailing `.0` dropped; plain bytes carry no decimal (`260b`, `5.1kb`,
+/// `1.9mb`). Previously `_cat/indices` emitted raw `{n}b` for every size.
+fn human_bytes(bytes: u64) -> String {
+ const KB: f64 = 1024.0;
+ const MB: f64 = 1024.0 * 1024.0;
+ const GB: f64 = 1024.0 * 1024.0 * 1024.0;
+ const TB: f64 = 1024.0 * 1024.0 * 1024.0 * 1024.0;
+ let b = bytes as f64;
+ let (val, unit) = if b >= TB {
+ (b / TB, "tb")
+ } else if b >= GB {
+ (b / GB, "gb")
+ } else if b >= MB {
+ (b / MB, "mb")
+ } else if b >= KB {
+ (b / KB, "kb")
+ } else {
+ return format!("{bytes}b");
+ };
+ let s = format!("{val:.1}");
+ let s = s.strip_suffix(".0").unwrap_or(&s);
+ format!("{s}{unit}")
+}
+
+/// A stable, deterministic UUID for an index, derived from its name.
+///
+/// `_cat/indices` used to mint a fresh `Uuid::new_v4()` on every request, so
+/// the uuid column changed each call and never matched the `index.uuid`
+/// returned by `GET /{index}/_settings`. ES returns the same real settings
+/// uuid in both places. We derive it deterministically from the index name so
+/// both endpoints agree and the value is stable across requests and restarts
+/// (no persistence path needed). `DefaultHasher` uses fixed keys, so this is
+/// reproducible across processes.
+fn stable_index_uuid(name: &str) -> String {
+ use std::hash::{Hash, Hasher};
+ let mut hi = std::collections::hash_map::DefaultHasher::new();
+ name.hash(&mut hi);
+ let high = hi.finish();
+ let mut lo = std::collections::hash_map::DefaultHasher::new();
+ // Distinct salt so the two 64-bit halves are independent.
+ 0x9E37_79B9_7F4A_7C15u64.hash(&mut lo);
+ name.hash(&mut lo);
+ let low = lo.finish();
+ Uuid::from_u128(((high as u128) << 64) | (low as u128)).to_string()
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// PUT /{index}
// ─────────────────────────────────────────────────────────────────────────────
@@ -1004,6 +1115,13 @@ fn merge_settings_defaults(user_settings: &Value, provided_name: &str, human: bo
if let Some(obj) = raw_inner.as_object() {
for (k, v) in obj {
+ // Internal bookkeeping keys (index-sort hints written at create
+ // time) are not ES index settings — never surface them in the
+ // settings response. They were leaking into GET /{index}/_settings
+ // for any index with an @timestamp-derived or explicit index sort.
+ if k.starts_with("__xy_") {
+ continue;
+ }
// ES coerces numeric settings to strings in the response.
let coerced = match v {
Value::Number(n) => Value::String(n.to_string()),
@@ -1019,9 +1137,12 @@ fn merge_settings_defaults(user_settings: &Value, provided_name: &str, human: bo
inner
.entry("creation_date".to_string())
.or_insert_with(|| Value::String(now_ms.to_string()));
- inner
- .entry("uuid".to_string())
- .or_insert_with(|| Value::String(Uuid::new_v4().to_string()));
+ // Emit the stable, deterministic index uuid — the SAME value
+ // _cat/indices reports — rather than a fresh random one per request.
+ inner.insert(
+ "uuid".to_string(),
+ Value::String(stable_index_uuid(provided_name)),
+ );
let version = inner
.entry("version".to_string())
.or_insert_with(|| json!({ "created": "8130099" }));
diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rs
index 39f5775b5..58c5af1bd 100644
--- a/engine/crates/xerj-api/src/router.rs
+++ b/engine/crates/xerj-api/src/router.rs
@@ -165,6 +165,7 @@ pub fn build_native_router(state: AppState) -> Router {
/// GET / es_info
/// GET /_cluster/health cluster_health
/// GET /_cat/indices cat_indices
+/// GET /_cat/indices/:pattern cat_indices_pattern
/// POST /_bulk global_bulk
/// PUT /:index create_index
/// DELETE /:index delete_index
@@ -198,6 +199,10 @@ pub fn build_es_compat_router(state: AppState) -> Router {
get(es_compat::cluster_health_for_index),
)
.route("/_cat/indices", get(es_compat::cat_indices))
+ .route(
+ "/_cat/indices/:pattern",
+ get(es_compat::cat_indices_pattern),
+ )
.route("/_cat/health", get(es_compat::cat_health))
.route("/_cat/nodes", get(es_compat::cat_nodes))
.route("/_cat/aliases", get(es_compat::cat_aliases))
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 8da25acbb..d3ea85ce9 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -1175,13 +1175,25 @@ impl Engine {
let snap_dir = std::path::Path::new(repo_path).join(name);
std::fs::create_dir_all(&snap_dir).map_err(EngineError::Io)?;
+ // Wall-clock start, captured BEFORE the flush+copy work so
+ // duration_in_millis reflects the real elapsed time (it was hardcoded
+ // to 0 because start_time == end_time were sampled at the same instant
+ // after the copy).
+ let start_ms = now_millis();
+
let target_indices: Vec<String> = match indices {
Some(list) if !list.is_empty() => list,
- _ => self.indices.iter().map(|e| e.key().clone()).collect(),
+ // ES excludes system indices from an all-indices snapshot; exclude
+ // our `.xerj_*` internals (auth, sessions, dashboards, …) by
+ // default so a plain snapshot captures only user data.
+ _ => self
+ .indices
+ .iter()
+ .map(|e| e.key().clone())
+ .filter(|n| !is_system_index(n))
+ .collect(),
};
- let mut manifest_indices: Vec<Value> = Vec::new();
-
for idx_name in &target_indices {
let idx = match self.indices.get(idx_name.as_str()) {
Some(i) => i,
@@ -1195,21 +1207,16 @@ impl Engine {
let dst_dir = snap_dir.join(idx_name);
std::fs::create_dir_all(&dst_dir).map_err(EngineError::Io)?;
+ // Copy everything recursively (WAL + segments + schema). The
+ // per-file list is intentionally discarded: it was serialized into
+ // the response as `index_files` (~170 KB of dead weight that ES
+ // never returns and `restore_snapshot` never reads — restore copies
+ // the snapshot dir back wholesale and reopens by index name).
let mut files: Vec<String> = Vec::new();
-
- // Copy everything recursively (WAL + segments + schema).
copy_dir_recursive(&src_dir, &dst_dir, &mut files).map_err(EngineError::Io)?;
-
- manifest_indices.push(serde_json::json!({
- "name": idx_name,
- "files": files,
- }));
}
- let now_ms = std::time::SystemTime::now()
- .duration_since(std::time::UNIX_EPOCH)
- .unwrap_or_default()
- .as_millis() as i64;
+ let end_ms = now_millis();
let manifest = serde_json::json!({
"snapshot": name,
@@ -1217,16 +1224,15 @@ impl Engine {
"version": "8.13.0",
"indices": target_indices,
"state": "SUCCESS",
- "start_time_in_millis": now_ms,
- "end_time_in_millis": now_ms,
- "duration_in_millis": 0,
+ "start_time_in_millis": start_ms,
+ "end_time_in_millis": end_ms,
+ "duration_in_millis": (end_ms - start_ms).max(0),
"failures": [],
"shards": {
"total": target_indices.len(),
"failed": 0,
"successful": target_indices.len(),
},
- "index_files": manifest_indices,
});
let manifest_path = snap_dir.join("manifest.json");
@@ -1286,13 +1292,25 @@ impl Engine {
.collect();
let index_names: Vec<String> = if patterns.is_empty() {
+ // Default restore: every index in the snapshot EXCEPT `.xerj_*`
+ // system indices (ES excludes system indices from a default
+ // restore). Snapshots taken by this build already omit them, but a
+ // snapshot written by an older build may still carry them.
snapshot_indices
+ .into_iter()
+ .filter(|n| !is_system_index(n))
+ .collect()
} else {
let mut selected: Vec<String> = Vec::new();
for pat in &patterns {
+ // A wildcard never matches a `.xerj_*` system index (ES `*`
+ // semantics); only a pattern that explicitly targets the
+ // dot-namespace (`.xerj_*`, `.xerj_users`, …) may.
+ let targets_system = pat.starts_with('.');
let matched: Vec<&String> = snapshot_indices
.iter()
.filter(|n| glob_match(pat, n))
+ .filter(|n| targets_system || !is_system_index(n))
.collect();
if matched.is_empty() && !pat.contains('*') {
// ES: restore of an index absent from the snapshot fails
@@ -1362,6 +1380,21 @@ impl Engine {
// ── Private helpers ───────────────────────────────────────────────────────────
+/// Milliseconds since the Unix epoch (wall clock).
+fn now_millis() -> i64 {
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_millis() as i64
+}
+
+/// A XERJ-internal system index (`.xerj_*`: auth, sessions, dashboards,
+/// audit, …). ES hides its own system indices from default snapshot/restore;
+/// we do the same for ours so `_snapshot`/`_restore` operate on user data.
+pub(crate) fn is_system_index(name: &str) -> bool {
+ name.starts_with(".xerj_")
+}
+
/// Recursively copy all files from `src` to `dst`, recording relative paths in `files`.
fn copy_dir_recursive(
src: &std::path::Path,