AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): persist alias filter metadata so it survives a restart (#524)
xerj-orgb745cef2d68f
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 6f9c6e385..c2fccb717 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -1146,6 +1146,7 @@ pub async fn create_index(.engine.index_alias_metadata.insert(index.clone(), Value::Object(resolved_aliases));+ state.engine.flush_alias_metadata();}// ES-shape lifecycle attach: `index.lifecycle.name` in create-time@@ -19574,6 +19575,7 @@ pub async fn put_alias(.engine.index_alias_metadata.insert(idx_name.to_string(), existing);+ state.engine.flush_alias_metadata();}Ok(())};diff --git a/engine/crates/xerj-api/tests/alias_filter_survives_restart.rs b/engine/crates/xerj-api/tests/alias_filter_survives_restart.rsnew file mode 100644index 000000000..9245a0ea4--- /dev/null+++ b/engine/crates/xerj-api/tests/alias_filter_survives_restart.rs@@ -0,0 +1,107 @@+//! Issue #524: an alias's `filter` must survive a restart.+//!+//! The alias→members topology is persisted to `aliases.json`, but each alias's+//! `filter` lived only in the in-memory `index_alias_metadata` map and was+//! never written to disk. So a filtered alias came back after a restart with+//! its members but no filter: `_search`/`_count` through it silently returned+//! the whole backing corpus instead of the slice the filter names, and once+//! the by-query paths honour the filter (#453) a `_delete_by_query` through+//! such an alias would empty the whole backing index. The fix persists the+//! metadata to a `alias_metadata.json` sidecar and reloads it on boot.+//!+//! This test creates a filtered alias, drops the engine, opens a fresh one on+//! the same data dir, and asserts the filter is still there. Before the fix it+//! comes back empty.+//!+//! Elasticsearch is referenced for wire semantics only; no ES code is+//! reproduced here.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++fn config_for(dir: &std::path::Path) -> xerj_common::config::Config {+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ config+}++/// Build a fresh app over `dir`. Calling this after the previous app has been+/// dropped is a restart: nothing is handed over in memory, the second boot+/// sees only what reached the disk.+fn app_over(dir: &std::path::Path) -> axum::Router {+ let config = config_for(dir);+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ xerj_api::router::build_es_compat_router(state)+}++async fn send(app: &axum::Router, req: Request<Body>) -> (StatusCode, Value) {+ let response = app.clone().oneshot(req).await.expect("response");+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .expect("body");+ let body = if bytes.is_empty() {+ Value::Null+ } else {+ serde_json::from_slice(&bytes).unwrap_or(Value::Null)+ };+ (status, body)+}++fn put_json(path: &str, body: Value) -> Request<Body> {+ Request::put(path)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .expect("request")+}++fn get(path: &str) -> Request<Body> {+ Request::get(path).body(Body::empty()).expect("request")+}++#[tokio::test]+async fn an_alias_filter_survives_a_restart() {+ let dir = tempfile::tempdir().expect("tempdir");+ let the_filter = json!({ "term": { "tier": "gold" } });++ // ── Session 1: create an index and a FILTERED alias over it. ──+ {+ let app = app_over(dir.path());++ let (st, body) = send(&app, put_json("/orders", json!({}))).await;+ assert_eq!(st, StatusCode::OK, "create index: {body}");++ let (st, body) = send(+ &app,+ put_json("/orders/_alias/gold", json!({ "filter": the_filter })),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "create filtered alias: {body}");++ // Sanity BEFORE the restart: the filter is visible, so a failure after+ // the restart is unambiguously a persistence failure, not a write that+ // never landed.+ let (st, body) = send(&app, get("/_alias/gold")).await;+ assert_eq!(st, StatusCode::OK, "read alias before restart: {body}");+ assert_eq!(+ body.pointer("/orders/aliases/gold/filter"),+ Some(&the_filter),+ "the filter must be present before the restart: {body}"+ );+ } // app — and with it the engine and its node lock — dropped here.++ // ── Session 2: a fresh engine on the same dir sees only what reached disk. ──+ let app = app_over(dir.path());+ let (st, body) = send(&app, get("/_alias/gold")).await;+ assert_eq!(st, StatusCode::OK, "read alias after restart: {body}");+ assert_eq!(+ body.pointer("/orders/aliases/gold/filter"),+ Some(&the_filter),+ "the alias filter must survive a restart, not come back empty: {body}"+ );+}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex d0ab8b6e8..164f6a1f7 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -988,6 +988,10 @@ impl Engine {// restart, mistaking a missing-alias 404 for a still-in-progress// migration by another instance).engine.load_persisted_aliases();+ // …and the alias filter metadata that pairs with that topology, or+ // a filtered alias comes back unfiltered — wrong reads, and a+ // by-query that empties the whole backing index (#524).+ engine.load_alias_metadata();// Restore the cluster-level management state — index templates,// legacy templates, component templates, ingest pipelines, data@@ -1756,6 +1760,73 @@ impl Engine {}}+ // ── Alias filter metadata durability (issue #524) ───────────────────────++ /// Path of the persisted alias-metadata sidecar+ /// (`<data_dir>/alias_metadata.json`).+ ///+ /// A SEPARATE file from `aliases.json` (which persists only the+ /// alias→members topology) on purpose. Each alias's `filter` (and the+ /// other create-time metadata) lived only in the in-memory+ /// `index_alias_metadata` map and evaporated on restart, so a filtered+ /// alias came back with its members but no filter — wrong `_search` /+ /// `_count` results, and once the by-query paths honour the filter, a+ /// `_delete_by_query` that empties the whole backing index instead of the+ /// slice the filter names (#524). A separate file keeps `aliases.json`'s+ /// shape unchanged; an older binary simply ignores this one, the same+ /// downgrade-safe call the ingest-pipeline-refusals sidecar makes.+ fn alias_metadata_path(&self) -> PathBuf {+ self.data_dir.join("alias_metadata.json")+ }++ /// Serialize `index_alias_metadata` to disk atomically (temp-file ++ /// rename), mirroring [`Engine::flush_aliases`]. Called after every+ /// mutation of the map; a write failure is logged but non-fatal (the+ /// filter still applies in-memory until the next restart).+ pub fn flush_alias_metadata(&self) {+ let snapshot: std::collections::HashMap<String, serde_json::Value> = self+ .index_alias_metadata+ .iter()+ .map(|e| (e.key().clone(), e.value().clone()))+ .collect();+ let bytes = match serde_json::to_vec_pretty(&snapshot) {+ Ok(b) => b,+ Err(e) => {+ warn!(error = %e, "failed to serialize alias metadata for persistence");+ return;+ }+ };+ if let Err(e) = crate::index::write_file_atomic(&self.alias_metadata_path(), &bytes) {+ warn!(error = %e, "failed to persist alias_metadata.json (alias filters work until restart)");+ }+ }++ /// Load persisted alias metadata from `<data_dir>/alias_metadata.json` into+ /// the in-memory map on boot. A missing file is normal (fresh node, or a+ /// node whose aliases were all created by a build that predates this+ /// sidecar); a corrupt file is logged and ignored.+ fn load_alias_metadata(&self) {+ let path = self.alias_metadata_path();+ let Ok(bytes) = std::fs::read(&path) else {+ return;+ };+ match serde_json::from_slice::<std::collections::HashMap<String, serde_json::Value>>(&bytes)+ {+ Ok(map) => {+ let n = map.len();+ for (index, meta) in map {+ self.index_alias_metadata.insert(index, meta);+ }+ if n > 0 {+ info!(count = n, "restored persisted alias metadata");+ }+ }+ Err(e) => {+ warn!(error = %e, "ignoring corrupt alias_metadata.json");+ }+ }+ }+// ── Ingest-pipeline refusal provenance (issue #204) ─────────────────────/// Sidecar holding [`Engine::definition_time_refusals`].@@ -2958,6 +3029,7 @@ impl Engine {self.index_settings.remove(name);self.index_mappings.remove(name);self.index_alias_metadata.remove(name);+ self.flush_alias_metadata();// The WAL tap's cursor is a byte offset into a WAL stream that has// just ceased to exist. Dropping it HERE, rather than waiting for a