Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): persist alias filter metadata so it survives a restart (#524)

xerj-orgb745cef2d68f
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 6f9c6e385..c2fccb717 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -1146,6 +1146,7 @@ pub async fn create_index(
.engine
.index_alias_metadata
.insert(index.clone(), Value::Object(resolved_aliases));
+ state.engine.flush_alias_metadata();
}
// ES-shape lifecycle attach: `index.lifecycle.name` in create-time
@@ -19574,6 +19575,7 @@ pub async fn put_alias(
.engine
.index_alias_metadata
.insert(idx_name.to_string(), existing);
+ state.engine.flush_alias_metadata();
}
Ok(())
};
diff --git a/engine/crates/xerj-api/tests/alias_filter_survives_restart.rs b/engine/crates/xerj-api/tests/alias_filter_survives_restart.rs
new file mode 100644
index 000000000..9245a0ea4
--- /dev/null
+++ b/engine/crates/xerj-api/tests/alias_filter_survives_restart.rs
@@ -0,0 +1,107 @@
+//! Issue #524: an alias's `filter` must survive a restart.
+//!
+//! The alias→members topology is persisted to `aliases.json`, but each alias's
+//! `filter` lived only in the in-memory `index_alias_metadata` map and was
+//! never written to disk. So a filtered alias came back after a restart with
+//! its members but no filter: `_search`/`_count` through it silently returned
+//! the whole backing corpus instead of the slice the filter names, and once
+//! the by-query paths honour the filter (#453) a `_delete_by_query` through
+//! such an alias would empty the whole backing index. The fix persists the
+//! metadata to a `alias_metadata.json` sidecar and reloads it on boot.
+//!
+//! This test creates a filtered alias, drops the engine, opens a fresh one on
+//! the same data dir, and asserts the filter is still there. Before the fix it
+//! comes back empty.
+//!
+//! Elasticsearch is referenced for wire semantics only; no ES code is
+//! reproduced here.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+fn config_for(dir: &std::path::Path) -> xerj_common::config::Config {
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ config
+}
+
+/// Build a fresh app over `dir`. Calling this after the previous app has been
+/// dropped is a restart: nothing is handed over in memory, the second boot
+/// sees only what reached the disk.
+fn app_over(dir: &std::path::Path) -> axum::Router {
+ let config = config_for(dir);
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ xerj_api::router::build_es_compat_router(state)
+}
+
+async fn send(app: &axum::Router, req: Request<Body>) -> (StatusCode, Value) {
+ let response = app.clone().oneshot(req).await.expect("response");
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .expect("body");
+ let body = if bytes.is_empty() {
+ Value::Null
+ } else {
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null)
+ };
+ (status, body)
+}
+
+fn put_json(path: &str, body: Value) -> Request<Body> {
+ Request::put(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request")
+}
+
+fn get(path: &str) -> Request<Body> {
+ Request::get(path).body(Body::empty()).expect("request")
+}
+
+#[tokio::test]
+async fn an_alias_filter_survives_a_restart() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let the_filter = json!({ "term": { "tier": "gold" } });
+
+ // ── Session 1: create an index and a FILTERED alias over it. ──
+ {
+ let app = app_over(dir.path());
+
+ let (st, body) = send(&app, put_json("/orders", json!({}))).await;
+ assert_eq!(st, StatusCode::OK, "create index: {body}");
+
+ let (st, body) = send(
+ &app,
+ put_json("/orders/_alias/gold", json!({ "filter": the_filter })),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "create filtered alias: {body}");
+
+ // Sanity BEFORE the restart: the filter is visible, so a failure after
+ // the restart is unambiguously a persistence failure, not a write that
+ // never landed.
+ let (st, body) = send(&app, get("/_alias/gold")).await;
+ assert_eq!(st, StatusCode::OK, "read alias before restart: {body}");
+ assert_eq!(
+ body.pointer("/orders/aliases/gold/filter"),
+ Some(&the_filter),
+ "the filter must be present before the restart: {body}"
+ );
+ } // app — and with it the engine and its node lock — dropped here.
+
+ // ── Session 2: a fresh engine on the same dir sees only what reached disk. ──
+ let app = app_over(dir.path());
+ let (st, body) = send(&app, get("/_alias/gold")).await;
+ assert_eq!(st, StatusCode::OK, "read alias after restart: {body}");
+ assert_eq!(
+ body.pointer("/orders/aliases/gold/filter"),
+ Some(&the_filter),
+ "the alias filter must survive a restart, not come back empty: {body}"
+ );
+}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index d0ab8b6e8..164f6a1f7 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -988,6 +988,10 @@ impl Engine {
// restart, mistaking a missing-alias 404 for a still-in-progress
// migration by another instance).
engine.load_persisted_aliases();
+ // …and the alias filter metadata that pairs with that topology, or
+ // a filtered alias comes back unfiltered — wrong reads, and a
+ // by-query that empties the whole backing index (#524).
+ engine.load_alias_metadata();
// Restore the cluster-level management state — index templates,
// legacy templates, component templates, ingest pipelines, data
@@ -1756,6 +1760,73 @@ impl Engine {
}
}
+ // ── Alias filter metadata durability (issue #524) ───────────────────────
+
+ /// Path of the persisted alias-metadata sidecar
+ /// (`<data_dir>/alias_metadata.json`).
+ ///
+ /// A SEPARATE file from `aliases.json` (which persists only the
+ /// alias→members topology) on purpose. Each alias's `filter` (and the
+ /// other create-time metadata) lived only in the in-memory
+ /// `index_alias_metadata` map and evaporated on restart, so a filtered
+ /// alias came back with its members but no filter — wrong `_search` /
+ /// `_count` results, and once the by-query paths honour the filter, a
+ /// `_delete_by_query` that empties the whole backing index instead of the
+ /// slice the filter names (#524). A separate file keeps `aliases.json`'s
+ /// shape unchanged; an older binary simply ignores this one, the same
+ /// downgrade-safe call the ingest-pipeline-refusals sidecar makes.
+ fn alias_metadata_path(&self) -> PathBuf {
+ self.data_dir.join("alias_metadata.json")
+ }
+
+ /// Serialize `index_alias_metadata` to disk atomically (temp-file +
+ /// rename), mirroring [`Engine::flush_aliases`]. Called after every
+ /// mutation of the map; a write failure is logged but non-fatal (the
+ /// filter still applies in-memory until the next restart).
+ pub fn flush_alias_metadata(&self) {
+ let snapshot: std::collections::HashMap<String, serde_json::Value> = self
+ .index_alias_metadata
+ .iter()
+ .map(|e| (e.key().clone(), e.value().clone()))
+ .collect();
+ let bytes = match serde_json::to_vec_pretty(&snapshot) {
+ Ok(b) => b,
+ Err(e) => {
+ warn!(error = %e, "failed to serialize alias metadata for persistence");
+ return;
+ }
+ };
+ if let Err(e) = crate::index::write_file_atomic(&self.alias_metadata_path(), &bytes) {
+ warn!(error = %e, "failed to persist alias_metadata.json (alias filters work until restart)");
+ }
+ }
+
+ /// Load persisted alias metadata from `<data_dir>/alias_metadata.json` into
+ /// the in-memory map on boot. A missing file is normal (fresh node, or a
+ /// node whose aliases were all created by a build that predates this
+ /// sidecar); a corrupt file is logged and ignored.
+ fn load_alias_metadata(&self) {
+ let path = self.alias_metadata_path();
+ let Ok(bytes) = std::fs::read(&path) else {
+ return;
+ };
+ match serde_json::from_slice::<std::collections::HashMap<String, serde_json::Value>>(&bytes)
+ {
+ Ok(map) => {
+ let n = map.len();
+ for (index, meta) in map {
+ self.index_alias_metadata.insert(index, meta);
+ }
+ if n > 0 {
+ info!(count = n, "restored persisted alias metadata");
+ }
+ }
+ Err(e) => {
+ warn!(error = %e, "ignoring corrupt alias_metadata.json");
+ }
+ }
+ }
+
// ── Ingest-pipeline refusal provenance (issue #204) ─────────────────────
/// Sidecar holding [`Engine::definition_time_refusals`].
@@ -2958,6 +3029,7 @@ impl Engine {
self.index_settings.remove(name);
self.index_mappings.remove(name);
self.index_alias_metadata.remove(name);
+ self.flush_alias_metadata();
// The WAL tap's cursor is a byte offset into a WAL stream that has
// just ceased to exist. Dropping it HERE, rather than waiting for a