Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): persist templates, pipelines, data streams and ILM policies (#203)

xerj-orge9798abf2cbd
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 3d84e0549..2d7cf9d92 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -17374,8 +17374,10 @@ pub async fn put_index_template(
mappings,
priority: body.priority.unwrap_or(0),
};
- state.engine.templates.insert(name, tmpl);
- Json(json!({ "acknowledged": true })).into_response()
+ match state.engine.put_index_template(name, tmpl) {
+ Ok(()) => Json(json!({ "acknowledged": true })).into_response(),
+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
+ }
}
pub async fn get_index_template(
@@ -17435,11 +17437,14 @@ pub async fn delete_index_template(
State(state): State<AppState>,
Path(name): Path<String>,
) -> impl IntoResponse {
- if state.engine.templates.remove(&name).is_some() {
- Json(json!({ "acknowledged": true })).into_response()
- } else {
- let e = xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));
- ApiError::new(e).into_response()
+ match state.engine.delete_index_template(&name) {
+ Ok(true) => Json(json!({ "acknowledged": true })).into_response(),
+ Ok(false) => {
+ let e =
+ xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));
+ ApiError::new(e).into_response()
+ }
+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
}
@@ -22151,8 +22156,6 @@ pub async fn put_ingest_pipeline(
Path(id): Path<String>,
Json(body): Json<Value>,
) -> impl IntoResponse {
- // Store the raw JSON for GET /_ingest/pipeline.
- state.engine.pipelines.insert(id.clone(), body.clone());
// Convert ES processor format → xerj stage format, then compile.
// ES: {"processors": [{"set": {"field":"x","value":"y"}}]}
// xerj: {"stages": [{"type": "set", "config": {"field":"x","value":"y"}}]}
@@ -22207,10 +22210,17 @@ pub async fn put_ingest_pipeline(
} else {
body.clone()
};
- if let Err(e) = state.engine.create_pipeline(&id, xerj_config) {
- tracing::warn!(pipeline = %id, error = %e, "pipeline stored but failed to compile");
+ // `Some(body)` keeps this surface's long-standing behaviour: a definition
+ // that will not compile is still stored and still readable through
+ // `GET /_ingest/pipeline/{id}` (the engine logs the compile error and
+ // drops the executable form, so `?pipeline={id}` behaves identically
+ // before and after a restart). An `Err` means it did not reach disk —
+ // answering `acknowledged` there would promise a pipeline that is gone at
+ // the next boot, which is the whole of issue #203.
+ match state.engine.put_pipeline(&id, xerj_config, Some(body)) {
+ Ok(_) => Json(json!({ "acknowledged": true })).into_response(),
+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
- Json(json!({ "acknowledged": true })).into_response()
}
pub async fn get_ingest_pipeline(
@@ -22240,11 +22250,13 @@ pub async fn delete_ingest_pipeline(
State(state): State<AppState>,
Path(id): Path<String>,
) -> impl IntoResponse {
- if state.engine.pipelines.remove(&id).is_some() {
- Json(json!({ "acknowledged": true })).into_response()
- } else {
- let e = xerj_common::XerjError::index_not_found(format!("pipeline [{id}] is missing"));
- ApiError::new(e).into_response()
+ match state.engine.delete_pipeline(&id) {
+ Ok(true) => Json(json!({ "acknowledged": true })).into_response(),
+ Ok(false) => {
+ let e = xerj_common::XerjError::index_not_found(format!("pipeline [{id}] is missing"));
+ ApiError::new(e).into_response()
+ }
+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
}
@@ -24062,8 +24074,15 @@ pub async fn put_ilm_policy(
) -> impl IntoResponse {
// Persist the raw policy verbatim in the ILM store; `get_ilm_policy`
// reads it back out of this same DashMap, so PUT then GET round-trips
- // faithfully regardless of whether translation below succeeds.
- state.engine.ilm_policies.insert(name.clone(), body.clone());
+ // faithfully regardless of whether translation below succeeds — and the
+ // store is written to `<data_dir>/cluster_state.json`, so it round-trips
+ // after a restart too (issue #203). This write goes first because it is
+ // the one that can fail loudly: if the document cannot be persisted the
+ // request is a 500 and nothing has happened yet, rather than an
+ // `acknowledged` that a reboot silently undoes.
+ if let Err(e) = state.engine.put_ilm_policy(name.clone(), body.clone()) {
+ return ApiError::new(xerj_common::XerjError::from(e)).into_response();
+ }
// Translate into the shared internal model (`xerj_engine::lifecycle`)
// so the SAME background job that drives native ISM policies also
@@ -24131,12 +24150,21 @@ pub async fn delete_ilm_policy(
State(state): State<AppState>,
Path(name): Path<String>,
) -> impl IntoResponse {
- if state.engine.ilm_policies.remove(&name).is_some() {
- state.engine.remove_ism_policy(&name);
- Json(json!({ "acknowledged": true })).into_response()
- } else {
- let e = xerj_common::XerjError::index_not_found(format!("policy [{name}] not found"));
- ApiError::new(e).into_response()
+ // Remove the verbatim document first: that is the guarded, persisted
+ // path, so a failure there is a 500 with nothing removed anywhere. Only
+ // once it is gone from `cluster_state.json` is the translated executor
+ // policy dropped, so the two files can never disagree in the direction
+ // that keeps running a policy `GET _ilm/policy` reports as deleted.
+ match state.engine.delete_ilm_policy(&name) {
+ Ok(true) => {
+ state.engine.remove_ism_policy(&name);
+ Json(json!({ "acknowledged": true })).into_response()
+ }
+ Ok(false) => {
+ let e = xerj_common::XerjError::index_not_found(format!("policy [{name}] not found"));
+ ApiError::new(e).into_response()
+ }
+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
}
@@ -24235,8 +24263,10 @@ pub async fn put_component_template(
Path(name): Path<String>,
Json(body): Json<Value>,
) -> impl IntoResponse {
- state.engine.component_templates.insert(name, body);
- Json(json!({ "acknowledged": true })).into_response()
+ match state.engine.put_component_template(name, body) {
+ Ok(()) => Json(json!({ "acknowledged": true })).into_response(),
+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
+ }
}
pub async fn get_component_template(
@@ -24278,12 +24308,15 @@ pub async fn delete_component_template(
State(state): State<AppState>,
Path(name): Path<String>,
) -> impl IntoResponse {
- if state.engine.component_templates.remove(&name).is_some() {
- Json(json!({ "acknowledged": true })).into_response()
- } else {
- let e =
- xerj_common::XerjError::index_not_found(format!("component template [{name}] missing"));
- ApiError::new(e).into_response()
+ match state.engine.delete_component_template(&name) {
+ Ok(true) => Json(json!({ "acknowledged": true })).into_response(),
+ Ok(false) => {
+ let e = xerj_common::XerjError::index_not_found(format!(
+ "component template [{name}] missing"
+ ));
+ ApiError::new(e).into_response()
+ }
+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
}
@@ -33932,8 +33965,10 @@ pub async fn put_legacy_template(
if let Some(settings) = body.get("settings").cloned() {
body["settings"] = flatten_template_settings(&settings);
}
- state.engine.legacy_templates.insert(name, body);
- Json(json!({ "acknowledged": true }))
+ match state.engine.put_legacy_template(name, body) {
+ Ok(()) => Json(json!({ "acknowledged": true })).into_response(),
+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
+ }
}
/// Normalize a template's `settings` block to ES's storage form: every
@@ -34030,11 +34065,14 @@ pub async fn delete_legacy_template(
State(state): State<AppState>,
Path(name): Path<String>,
) -> impl IntoResponse {
- if state.engine.legacy_templates.remove(&name).is_some() {
- Json(json!({ "acknowledged": true })).into_response()
- } else {
- let e = xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));
- ApiError::new(e).into_response()
+ match state.engine.delete_legacy_template(&name) {
+ Ok(true) => Json(json!({ "acknowledged": true })).into_response(),
+ Ok(false) => {
+ let e =
+ xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));
+ ApiError::new(e).into_response()
+ }
+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
}
diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rs
index c008077bc..3aa0f1534 100644
--- a/engine/crates/xerj-api/src/native.rs
+++ b/engine/crates/xerj-api/src/native.rs
@@ -2486,8 +2486,13 @@ pub async fn put_pipeline(
let started = Instant::now();
let request_id = Uuid::new_v4().to_string();
- match state.engine.create_pipeline(&name, body) {
- Ok(()) => {
+ // `None` — this surface rejects a definition it cannot compile, so
+ // nothing is stored on a compile error. On success the pipeline is
+ // written to `<data_dir>/cluster_state.json` before we acknowledge it
+ // (issue #203): it used to live only in memory and vanish on restart,
+ // exactly like the ES-compat one.
+ match state.engine.put_pipeline(&name, body, None) {
+ Ok(None) => {
let took_ms = started.elapsed().as_millis() as u64;
let resp = NativeResponse::new(
serde_json::json!({ "pipeline": name, "acknowledged": true }),
@@ -2496,6 +2501,13 @@ pub async fn put_pipeline(
);
(StatusCode::OK, Json(resp)).into_response()
}
+ // The definition does not compile — same 500 this surface has always
+ // returned, and nothing was stored.
+ Ok(Some(compile_err)) => {
+ let ze = xerj_common::XerjError::internal(compile_err.to_string());
+ native_error(ze, Some(&request_id), started.elapsed().as_millis() as u64)
+ .into_response()
+ }
Err(e) => {
let ze = xerj_common::XerjError::internal(e.to_string());
native_error(ze, Some(&request_id), started.elapsed().as_millis() as u64)
diff --git a/engine/crates/xerj-api/tests/cluster_metadata_survives_restart.rs b/engine/crates/xerj-api/tests/cluster_metadata_survives_restart.rs
new file mode 100644
index 000000000..19a5f83f0
--- /dev/null
+++ b/engine/crates/xerj-api/tests/cluster_metadata_survives_restart.rs
@@ -0,0 +1,447 @@
+//! Issue #203: index templates, legacy templates, component templates,
+//! ingest pipelines, data streams and ILM policies must survive a restart.
+//!
+//! Before the fix each of these lived only in an in-memory `DashMap` on
+//! `Engine`. `Engine::new` restored index directories, `es_mapping.json`,
+//! API keys and aliases — nothing else — so a `PUT _index_template/logs`
+//! answered `200 {"acknowledged": true}` and then evaporated on the next
+//! boot. The operator got no error, and the next index that should have
+//! matched the template was created without it.
+//!
+//! Elasticsearch is referenced for wire semantics only; no ES code is
+//! reproduced here.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+fn config_for(dir: &std::path::Path) -> xerj_common::config::Config {
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ config
+}
+
+/// Build a fresh app over `dir`. Calling this twice (with the first engine
+/// dropped in between) is a restart: nothing is handed over in memory, the
+/// second boot sees only what reached the disk.
+fn app_over(dir: &std::path::Path) -> axum::Router {
+ let config = config_for(dir);
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ xerj_api::router::build_es_compat_router(state)
+}
+
+async fn send(app: &axum::Router, req: Request<Body>) -> (StatusCode, Value) {
+ let response = app.clone().oneshot(req).await.expect("response");
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .expect("body");
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+async fn put_json(app: &axum::Router, path: &str, body: Value) -> (StatusCode, Value) {
+ send(
+ app,
+ Request::put(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request"),
+ )
+ .await
+}
+
+async fn get(app: &axum::Router, path: &str) -> (StatusCode, Value) {
+ send(
+ app,
+ Request::get(path).body(Body::empty()).expect("request"),
+ )
+ .await
+}
+
+async fn delete(app: &axum::Router, path: &str) -> (StatusCode, Value) {
+ send(
+ app,
+ Request::delete(path).body(Body::empty()).expect("request"),
+ )
+ .await
+}
+
+/// Write one of each managed object. Kept separate from the assertions so
+/// the same fixture can be replayed after a restart.
+async fn write_all_the_config(app: &axum::Router) {
+ let (st, _) = put_json(
+ app,
+ "/_index_template/logs",
+ json!({
+ "index_patterns": ["logs-*"],
+ "priority": 200,
+ "template": {
+ "settings": { "number_of_replicas": 0 },
+ "mappings": { "properties": { "host": { "type": "keyword" } } }
+ }
+ }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "PUT _index_template");
+
+ let (st, _) = put_json(
+ app,
+ "/_template/legacy-logs",
+ json!({
+ "index_patterns": ["legacy-*"],
+ "settings": { "number_of_shards": 1 },
+ "mappings": { "properties": { "msg": { "type": "text" } } }
+ }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "PUT _template");
+
+ let (st, _) = put_json(
+ app,
+ "/_component_template/base-settings",
+ json!({ "template": { "settings": { "number_of_replicas": 1 } } }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "PUT _component_template");
+
+ let (st, _) = put_json(
+ app,
+ "/_ingest/pipeline/tagger",
+ json!({
+ "description": "tag every doc",
+ "processors": [ { "set": { "field": "tagged", "value": "yes" } } ]
+ }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "PUT _ingest/pipeline");
+
+ let (st, _) = put_json(
+ app,
+ "/_ilm/policy/hot-warm",
+ json!({ "policy": { "phases": { "hot": { "actions": {} } } } }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "PUT _ilm/policy");
+
+ let (st, _) = put_json(app, "/_data_stream/metrics-app", json!({})).await;
+ assert_eq!(st, StatusCode::OK, "PUT _data_stream");
+}
+
+/// The read side of each managed object.
+const READ_BACK: [&str; 6] = [
+ "/_index_template/logs",
+ "/_template/legacy-logs",
+ "/_component_template/base-settings",
+ "/_ingest/pipeline/tagger",
+ "/_ilm/policy/hot-warm",
+ "/_data_stream/metrics-app",
+];
+
+/// Read every managed object back, asserting each is present, and return the
+/// exact responses so a later boot can be compared against them verbatim.
+///
+/// Comparing whole bodies is deliberate: it catches a partial restore (a
+/// template that comes back without its mappings, a data stream that comes
+/// back at generation 1) that a spot-check on one field would wave through.
+async fn read_back_everything(app: &axum::Router, when: &str) -> Vec<Value> {
+ let mut out = Vec::new();
+ for path in READ_BACK {
+ let (st, body) = get(app, path).await;
+ assert_eq!(st, StatusCode::OK, "GET {path} {when}: {body}");
+ out.push(body);
+ }
+
+ // Enough content assertions to prove the fixture really was written, so
+ // an all-empty "equal" comparison cannot pass for a restore.
+ let tmpl = &out[0]["index_templates"][0]["index_template"];
+ assert_eq!(tmpl["index_patterns"][0], "logs-*", "{when}: {}", out[0]);
+ assert_eq!(tmpl["priority"], 200, "{when}: {}", out[0]);
+ assert_eq!(
+ tmpl["template"]["mappings"]["properties"]["host"]["type"], "keyword",
+ "{when}: {}",
+ out[0]
+ );
+ assert_eq!(
+ out[1]["legacy-logs"]["index_patterns"][0], "legacy-*",
+ "{when}: {}",
+ out[1]
+ );
+ assert_eq!(
+ out[2]["component_templates"][0]["component_template"]["template"]["settings"]
+ ["number_of_replicas"],
+ 1,
+ "{when}: {}",
+ out[2]
+ );
+ assert_eq!(
+ out[5]["data_streams"][0]["name"], "metrics-app",
+ "{when}: {}",
+ out[5]
+ );
+ out
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn templates_pipelines_streams_and_policies_survive_a_restart() {
+ let dir = tempfile::tempdir().expect("tempdir");
+
+ let before = {
+ let app = app_over(dir.path());
+ write_all_the_config(&app).await;
+ // Sanity: it is all readable *before* the restart, so a failure
+ // below is about durability and not about the write path.
+ read_back_everything(&app, "before restart").await
+ }; // engine dropped — the node.lock is released, nothing is handed over
+
+ let app = app_over(dir.path());
+ let after = read_back_everything(&app, "after restart").await;
+
+ for (path, (b, a)) in READ_BACK.iter().zip(before.iter().zip(after.iter())) {
+ assert_eq!(
+ b, a,
+ "GET {path} answered differently after a restart\nbefore: {b}\nafter: {a}"
+ );
+ }
+}
+
+/// A restored ingest pipeline must actually *run*, not merely round-trip
+/// through `GET /_ingest/pipeline`. Storing the raw JSON without
+/// recompiling it would leave `?pipeline=tagger` accepted and ignored.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_restored_pipeline_still_transforms_documents() {
+ let dir = tempfile::tempdir().expect("tempdir");
+
+ {
+ let app = app_over(dir.path());
+ write_all_the_config(&app).await;
+ }
+
+ let app = app_over(dir.path());
+ let (st, body) = send(
+ &app,
+ Request::post("/_ingest/pipeline/tagger/_simulate")
+ .header("content-type", "application/json")
+ .body(Body::from(
+ json!({ "docs": [ { "_source": { "host": "a" } } ] }).to_string(),
+ ))
+ .expect("request"),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "simulate after restart: {body}");
+ assert_eq!(
+ body["docs"][0]["doc"]["_source"]["tagged"], "yes",
+ "the restored pipeline must still execute its processors: {body}"
+ );
+}
+
+/// Deletes must be durable too — a deleted template that comes back on the
+/// next boot is the same class of surprise as one that disappears.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_deleted_template_stays_deleted_across_a_restart() {
+ let dir = tempfile::tempdir().expect("tempdir");
+
+ {
+ let app = app_over(dir.path());
+ write_all_the_config(&app).await;
+ let (st, _) = delete(&app, "/_index_template/logs").await;
+ assert_eq!(st, StatusCode::OK, "DELETE _index_template");
+ let (st, _) = delete(&app, "/_ingest/pipeline/tagger").await;
+ assert_eq!(st, StatusCode::OK, "DELETE _ingest/pipeline");
+ }
+
+ let app = app_over(dir.path());
+ let (st, _) = get(&app, "/_index_template/logs").await;
+ assert_eq!(
+ st,
+ StatusCode::NOT_FOUND,
+ "a deleted index template must not resurrect on restart"
+ );
+ let (st, _) = get(&app, "/_ingest/pipeline/tagger").await;
+ assert_eq!(
+ st,
+ StatusCode::NOT_FOUND,
+ "a deleted pipeline must not resurrect on restart"
+ );
+}
+
+async fn post(app: &axum::Router, path: &str, body: Value) -> (StatusCode, Value) {
+ send(
+ app,
+ Request::post(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request"),
+ )
+ .await
+}
+
+/// A rollover interrupted between "backing index created" and "generation
+/// written" must not wedge the stream.
+///
+/// `rollover_data_stream` creates `.ds-<name>-00000N` on disk first and only
+/// then flushes the new generation, so a `kill -9` in that window leaves the
+/// index present and the document still saying N-1. Restoring N-1 verbatim
+/// makes the next rollover recompute the *same* name, which `create_index`
+/// refuses as `index_already_exists` — and it does so on every subsequent
+/// attempt, so the stream can never roll again.
+///
+/// The crash window is reproduced exactly rather than approximated: the
+/// backing index is left on disk and the persisted generation is rewound by
+/// hand, which is byte-for-byte the state a crash at that instant leaves.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_rollover_interrupted_before_its_flush_does_not_wedge_the_stream() {
+ let dir = tempfile::tempdir().expect("tempdir");
+
+ {
+ let app = app_over(dir.path());
+ let (st, _) = put_json(&app, "/_data_stream/metrics-app", json!({})).await;
+ assert_eq!(st, StatusCode::OK, "PUT _data_stream");
+ let (st, body) = post(&app, "/metrics-app/_rollover", json!({})).await;
+ assert_eq!(st, StatusCode::OK, "rollover: {body}");
+ }
+
+ // Rewind only the generation, exactly as a crash between `create_index`
+ // and `flush_cluster_state` would have left it: `.ds-metrics-app-000002`
+ // is on disk, the document still reads generation 1.
+ let state_path = dir.path().join("cluster_state.json");
+ let mut state: Value =
+ serde_json::from_slice(&std::fs::read(&state_path).expect("read state")).expect("parse");
+ let ds = &mut state["data_streams"]["metrics-app"];
+ ds["generation"] = json!(1);
+ ds["backing_indices"] = json!([".ds-metrics-app-000001"]);
+ std::fs::write(&state_path, serde_json::to_vec(&state).expect("serialize")).expect("write");
+ assert!(
+ dir.path().join(".ds-metrics-app-000002").is_dir(),
+ "fixture is wrong: the generation-2 backing index should still be on disk"
+ );
+
+ let app = app_over(dir.path());
+ let (st, body) = post(&app, "/metrics-app/_rollover", json!({})).await;
+ assert_eq!(
+ st,
+ StatusCode::OK,
+ "a stream recovered from an interrupted rollover must still roll: {body}"
+ );
+ assert!(
+ body.to_string().contains(".ds-metrics-app-000003"),
+ "the next rollover must skip past the backing index the crash left \
+ behind, not reissue it: {body}"
+ );
+}
+
+/// A corrupt `cluster_state.json` must not be destroyed by the next write.
+///
+/// The node boots with empty maps and logs the failure, so the first
+/// `PUT /_index_template/...` rewrites the file — taking whatever was still
+/// legible in it along. Hand-recovery out of a damaged document is the
+/// operator's last option; it should not be silently removed.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_corrupt_cluster_state_is_preserved_before_it_is_overwritten() {
+ let dir = tempfile::tempdir().expect("tempdir");
+
+ {
+ let app = app_over(dir.path());
+ write_all_the_config(&app).await;
+ }
+
+ // Truncate mid-document — the shape a torn write would have had, if the
+ // rewrite were not atomic, and the shape media damage leaves.
+ let state_path = dir.path().join("cluster_state.json");
+ let original = std::fs::read_to_string(&state_path).expect("read state");
+ let truncated = &original[..original.len() / 2];
+ std::fs::write(&state_path, truncated).expect("write truncated");
+
+ let app = app_over(dir.path());
+ let (st, _) = get(&app, "/_index_template/logs").await;
+ assert_eq!(
+ st,
+ StatusCode::NOT_FOUND,
+ "a corrupt document cannot be restored — this is the honest outcome"
+ );
+
+ // The write that would have destroyed the evidence.
+ let (st, _) = put_json(
+ &app,
+ "/_index_template/fresh",
+ json!({ "index_patterns": ["fresh-*"] }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "PUT after corrupt load");
+
+ let salvage = dir.path().join("cluster_state.corrupt.json");
+ assert!(
+ salvage.is_file(),
+ "the corrupt document must be kept for recovery"
+ );
+ assert_eq!(
+ std::fs::read_to_string(&salvage).expect("read salvage"),
+ truncated,
+ "the preserved copy must be the bytes that were found, unmodified"
+ );
+ assert!(
+ std::fs::read_to_string(&state_path)
+ .expect("read state")
+ .contains("fresh-*"),
+ "the live document must have moved on"
+ );
+}
+
+/// Concurrent writers must not trip over each other.
+///
+/// The atomic rewrite stages through one fixed temp path, so two flushes
+/// running at once create, truncate and rename the *same* file: one loses
+/// its bytes to the other's truncate, or finds the temp file already renamed
+/// away. A provisioning script that fires its `PUT`s in parallel is all it
+/// takes — no stress needed. Measured without the engine's write lock, 32
+/// requests over a multi-threaded runtime fail immediately:
+///
+/// ```text
+/// PUT template t2: {"error":{"type":"store_exception",
+/// "reason":"storage error: IO error"},"status":500}
+/// ```
+#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+async fn concurrent_writes_do_not_corrupt_the_store() {
+ const N: usize = 32;
+ let dir = tempfile::tempdir().expect("tempdir");
+
+ {
+ let app = app_over(dir.path());
+ let mut writes = Vec::with_capacity(N);
+ for i in 0..N {
+ let app = app.clone();
+ writes.push(tokio::spawn(async move {
+ put_json(
+ &app,
+ &format!("/_index_template/t{i}"),
+ json!({ "index_patterns": [format!("t{i}-*")], "priority": i }),
+ )
+ .await
+ }));
+ }
+ for (i, w) in writes.into_iter().enumerate() {
+ let (st, body) = w.await.expect("join");
+ assert_eq!(st, StatusCode::OK, "PUT template t{i}: {body}");
+ }
+ }
+
+ let app = app_over(dir.path());
+ for i in 0..N {
+ let (st, body) = get(&app, &format!("/_index_template/t{i}")).await;
+ assert_eq!(
+ st,
+ StatusCode::OK,
+ "template t{i} did not survive concurrent writes + restart: {body}"
+ );
+ assert_eq!(
+ body["index_templates"][0]["index_template"]["priority"], i,
+ "template t{i} came back with the wrong contents: {body}"
+ );
+ }
+}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 89dc6a254..aa51406d6 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -138,6 +138,45 @@ pub struct IndexTemplate {
pub priority: i32,
}
+/// Version stamped into `cluster_state.json`. Bump only when the on-disk
+/// shape changes in a way an older build cannot read; new *optional* fields
+/// (`#[serde(default)]`) do not need one.
+const CLUSTER_STATE_VERSION: u32 = 1;
+
+/// Everything in `<data_dir>/cluster_state.json` — the cluster-level
+/// management state that used to be in-memory only (issue #203).
+///
+/// `BTreeMap` rather than `HashMap` so the file is byte-stable for a given
+/// state: an operator diffing two nodes' `cluster_state.json`, or a backup
+/// tool deduplicating it, sees a change only when the state really changed.
+/// Every field is `#[serde(default)]`, so a file written by an older build
+/// (or by one that gains another map later) still loads.
+#[derive(Debug, Default, Serialize, Deserialize)]
+struct PersistedClusterState {
+ #[serde(default)]
+ version: u32,
+ /// v2 index templates — `PUT /_index_template/{name}`.
+ #[serde(default)]
+ index_templates: std::collections::BTreeMap<String, IndexTemplate>,
+ /// v1 legacy templates — `PUT /_template/{name}`.
+ #[serde(default)]
+ legacy_templates: std::collections::BTreeMap<String, Value>,
+ /// Component templates — `PUT /_component_template/{name}`.
+ #[serde(default)]
+ component_templates: std::collections::BTreeMap<String, Value>,
+ /// Ingest pipeline definitions — `PUT /_ingest/pipeline/{id}`. Stored as
+ /// the document `GET` hands back, and recompiled into an executable
+ /// pipeline on boot.
+ #[serde(default)]
+ pipelines: std::collections::BTreeMap<String, Value>,
+ /// Data streams — `PUT /_data_stream/{name}` plus every rollover.
+ #[serde(default)]
+ data_streams: std::collections::BTreeMap<String, DataStream>,
+ /// ILM policies — `PUT /_ilm/policy/{name}`.
+ #[serde(default)]
+ ilm_policies: std::collections::BTreeMap<String, Value>,
+}
+
/// Active scroll context holding all matching hits.
///
/// Each context pins a fully-hydrated `Vec<Hit>` snapshot, so its lifetime
@@ -417,15 +456,24 @@ pub struct Engine {
data_dir: PathBuf,
/// alias_name → list of index names
pub aliases: Arc<DashMap<String, Vec<String>>>,
- /// template_name → IndexTemplate
+ /// template_name → IndexTemplate. Persisted in
+ /// `<data_dir>/cluster_state.json`; mutate only through
+ /// [`Engine::put_index_template`] / [`Engine::delete_index_template`],
+ /// or the change will not survive a restart (issue #203).
pub templates: Arc<DashMap<String, IndexTemplate>>,
/// scroll_id → ScrollContext
pub scrolls: Arc<DashMap<String, ScrollContext>>,
- /// pipeline_id → pipeline definition JSON
+ /// pipeline_id → pipeline definition JSON. Persisted in
+ /// `<data_dir>/cluster_state.json`; mutate only through
+ /// [`Engine::put_pipeline`] / [`Engine::delete_pipeline`], or the change
+ /// will not survive a restart (issue #203).
pub pipelines: Arc<DashMap<String, Value>>,
/// index_name → open/closed state (true = closed)
pub closed_indices: Arc<DashMap<String, bool>>,
- /// data stream name → DataStream
+ /// data stream name → DataStream. Persisted in
+ /// `<data_dir>/cluster_state.json`; mutate only through the
+ /// `*_data_stream` methods, or the change will not survive a restart
+ /// (issue #203).
pub data_streams: Arc<DashMap<String, DataStream>>,
/// ILM policy name → raw policy JSON, kept verbatim for `GET _ilm/policy`
/// round-trip fidelity. The REAL execution engine reads `ism_policies`
@@ -433,10 +481,19 @@ pub struct Engine {
/// `xerj_api::es_compat::put_ilm_policy`), translating into the shared
/// internal model. See `crate::lifecycle` for why one engine drives both
/// surfaces.
+ ///
+ /// This map — the verbatim ILM document — is persisted in
+ /// `<data_dir>/cluster_state.json`; mutate only through
+ /// [`Engine::put_ilm_policy`] / [`Engine::delete_ilm_policy`], or the
+ /// change will not survive a restart (issue #203). The translated
+ /// `ism_policies` model has its own file (`ism_policies.json`, issue
+ /// #199) — the two never write each other's file.
pub ilm_policies: Arc<DashMap<String, Value>>,
/// policy id → the internal ISM-shaped model every managed index is
/// actually driven by, regardless of whether the policy was created via
/// `_plugins/_ism/policies` (native) or `_ilm/policy` (translated).
+ /// Persisted separately in `<data_dir>/ism_policies.json` — see
+ /// [`Engine::put_ism_policy`].
pub ism_policies: Arc<DashMap<String, crate::lifecycle::LifecyclePolicy>>,
/// index name → lifecycle execution cursor (current state, pending
/// action, timestamps). Presence in this map is what "managed" means —
@@ -446,7 +503,11 @@ pub struct Engine {
/// `GET /{index}`'s `creation_date` (previously synthesized as
/// `Utc::now()` on every request — see `record_index_created_at`).
pub index_created_at: Arc<DashMap<String, i64>>,
- /// component template name → template JSON
+ /// component template name → template JSON. Persisted in
+ /// `<data_dir>/cluster_state.json`; mutate only through
+ /// [`Engine::put_component_template`] /
+ /// [`Engine::delete_component_template`], or the change will not survive
+ /// a restart (issue #203).
pub component_templates: Arc<DashMap<String, Value>>,
/// snapshot repository name → repo config JSON
pub snapshot_repos: Arc<DashMap<String, Value>>,
@@ -488,7 +549,10 @@ pub struct Engine {
/// subsequent poll, but nothing actually gates on them yet.
/// In-memory only (lost on restart).
pub application_privileges: Arc<DashMap<String, Value>>,
- /// legacy index template name (v1 /_template) → template JSON
+ /// legacy index template name (v1 /_template) → template JSON.
+ /// Persisted in `<data_dir>/cluster_state.json`; mutate only through
+ /// [`Engine::put_legacy_template`] / [`Engine::delete_legacy_template`],
+ /// or the change will not survive a restart (issue #203).
pub legacy_templates: Arc<DashMap<String, Value>>,
/// pipeline_name → compiled, executable Pipeline (typed transform pipeline)
pub transform_pipelines: Arc<DashMap<String, xerj_wasm::pipeline::Pipeline>>,
@@ -549,6 +613,20 @@ pub struct Engine {
/// the process: a `kill -9` releases it automatically and a stale
/// `node.lock` file never blocks the next boot.
_node_lock: Arc<std::fs::File>,
+
+ /// Serializes rewrites of `<data_dir>/cluster_state.json` (issue #203).
+ ///
+ /// The atomic rewrite stages through a single fixed path
+ /// (`cluster_state.tmp`), so two concurrent flushes — two provisioning
+ /// PUTs arriving together is all it takes — would create/truncate the
+ /// *same* temp file and rename each other's half-written bytes over the
+ /// real one. Holding this across snapshot-and-write makes the whole
+ /// rewrite one critical section; the last writer's snapshot necessarily
+ /// includes every mutation that had already been applied to the maps, so
+ /// serializing loses nothing.
+ ///
+ /// Never taken while holding a `DashMap` guard.
+ cluster_state_write: Arc<parking_lot::Mutex<()>>,
}
impl Engine {
@@ -660,6 +738,7 @@ impl Engine {
xerj_cluster::router::ShardRouter::new(1),
)),
_node_lock: node_lock,
+ cluster_state_write: Arc::new(parking_lot::Mutex::new(())),
};
// Scan data_dir for existing index directories.
@@ -730,9 +809,24 @@ impl Engine {
// migration by another instance).
engine.load_persisted_aliases();
+ // Restore the cluster-level management state — index templates,
+ // legacy templates, component templates, ingest pipelines, data
+ // streams, ILM policies (issue #203). Before this, "replace the
+ // binary and restart" silently reverted every one of them: the
+ // documents kept flowing, into indices that no longer had the shape
+ // the operator designed. Must run after the index scan above, so a
+ // restored data stream can report which of its backing indices are
+ // actually present, and before the server accepts requests.
+ engine.load_cluster_state();
+
// Restore ISM/ILM policies and managed-index execution state so a
// policy attached before a restart keeps running afterward instead
- // of silently going idle.
+ // of silently going idle. Separate files (`ism_policies.json`,
+ // `ism_managed_indices.json`) from `cluster_state.json` above, and
+ // deliberately so: that file holds the verbatim documents an
+ // operator PUT, this holds the executor's own cursor. Loaded after
+ // `load_cluster_state` because a restored data stream is what a
+ // rollover action operates on.
engine.load_persisted_ism_policies();
engine.load_persisted_managed_indices();
@@ -1626,6 +1720,448 @@ impl Engine {
self.record_index_created_at(name);
}
+ // ── Cluster metadata persistence (issue #203) ────────────────────────────
+ //
+ // Index templates, legacy (v1) templates, component templates, ingest
+ // pipelines, data streams and ILM policies used to live only in the
+ // in-memory maps above: `PUT /_index_template/logs` answered `200
+ // {"acknowledged": true}` and the template was gone after the next boot,
+ // with no error anywhere and the next matching index silently created
+ // unshaped. They are now snapshotted, as one consistent document, into
+ // `<data_dir>/cluster_state.json` on every mutation and reloaded in
+ // `Engine::new`.
+ //
+ // Durability contract: a full-file rewrite through
+ // `index::write_file_atomic` — write to `cluster_state.tmp`, `fsync` it,
+ // `rename` over the target, then `fsync` the parent directory so the
+ // rename itself survives power loss. The rename is the commit point, so
+ // a `kill -9` at any instant leaves either the previous complete
+ // document or the new one, never a half-written mix. The same shape peer
+ // engines use for their manifests (cf. `fjall/src/file.rs:17`
+ // `fsync_directory`, which fjall calls after every atomic rewrite;
+ // `sled/src/metadata_store.rs:696` discards leftover `*.tmp` files from
+ // an interrupted rewrite on the next boot, which `load_cluster_state`
+ // mirrors below).
+ //
+ // One file rather than six keeps the maps mutually consistent — a data
+ // stream and the template that shaped it can never be restored from
+ // different generations.
+
+ /// Path of the persisted cluster-metadata document
+ /// (`<data_dir>/cluster_state.json`).
+ fn cluster_state_path(&self) -> PathBuf {
+ self.data_dir.join("cluster_state.json")
+ }
+
+ /// Snapshot every persisted management map into one document.
+ fn cluster_state_snapshot(&self) -> PersistedClusterState {
+ fn dump<V: Clone>(map: &DashMap<String, V>) -> std::collections::BTreeMap<String, V> {
+ map.iter()
+ .map(|e| (e.key().clone(), e.value().clone()))
+ .collect()
+ }
+ PersistedClusterState {
+ version: CLUSTER_STATE_VERSION,
+ index_templates: dump(&self.templates),
+ legacy_templates: dump(&self.legacy_templates),
+ component_templates: dump(&self.component_templates),
+ pipelines: dump(&self.pipelines),
+ data_streams: dump(&self.data_streams),
+ ilm_policies: dump(&self.ilm_policies),
+ }
+ }
+
+ /// Durably write the current cluster metadata.
+ ///
+ /// Unlike `flush_aliases`, a failure here is **returned**, not swallowed:
+ /// every caller is an API write that would otherwise answer
+ /// `{"acknowledged": true}` for a change it cannot keep. Callers roll the
+ /// in-memory change back and surface a 500, so the operator finds out at
+ /// the moment of the write instead of at the next restart.
+ fn flush_cluster_state(&self) -> Result<()> {
+ // One writer at a time — see `cluster_state_write`. Snapshotting
+ // inside the lock also means the bytes that land are never older
+ // than a rewrite that has already returned.
+ let _writing = self.cluster_state_write.lock();
+ let snapshot = self.cluster_state_snapshot();
+ let bytes = serde_json::to_vec_pretty(&snapshot)?;
+ crate::index::write_file_atomic(&self.cluster_state_path(), &bytes)?;
+ Ok(())
+ }
+
+ /// Highest generation `N` for which a backing index `.ds-<stream>-<N>` is
+ /// actually open in this data dir; `0` when there is none.
+ ///
+ /// Read off the open indices rather than the stream document, because the
+ /// point is to catch the case where the two disagree. Suffix parsing keeps
+ /// neighbouring streams apart on its own: for stream `a`, `.ds-a-b-000001`
+ /// leaves `b-000001`, which is not a number.
+ fn highest_backing_generation(&self, stream: &str) -> u64 {
+ let prefix = format!(".ds-{stream}-");
+ self.indices
+ .iter()
+ .filter_map(|e| {
+ e.key()
+ .strip_prefix(&prefix)
+ .and_then(|gen| gen.parse::<u64>().ok())
+ })
+ .max()
+ .unwrap_or(0)
+ }
+
+ /// Load `<data_dir>/cluster_state.json` back into the management maps.
+ ///
+ /// A missing file is normal (fresh node, or a data dir written by a build
+ /// that predates this). A corrupt file is loud: it means the operator's
+ /// templates, pipelines and data streams are *not* coming back, which is
+ /// exactly the silence this whole mechanism exists to remove.
+ fn load_cluster_state(&self) {
+ // An interrupted rewrite can leave a partial `cluster_state.tmp`
+ // behind. It is never read (the rename is the commit point), but
+ // leaving it litters the data dir and invites a hand-edit that
+ // "restores" a torn document — sled sweeps the same leftovers on
+ // boot (`sled/src/metadata_store.rs:696`).
+ let tmp = self.cluster_state_path().with_extension("tmp");
+ if tmp.exists() {
+ warn!(
+ path = %tmp.display(),
+ "discarding an incomplete cluster_state rewrite left by an unclean shutdown"
+ );
+ let _ = std::fs::remove_file(&tmp);
+ }
+
+ let path = self.cluster_state_path();
+ let bytes = match std::fs::read(&path) {
+ Ok(b) => b,
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return,
+ Err(e) => {
+ error!(
+ path = %path.display(), error = %e,
+ "could not read cluster_state.json — index templates, ingest \
+ pipelines, data streams and ILM policies are NOT restored"
+ );
+ return;
+ }
+ };
+ let state: PersistedClusterState = match serde_json::from_slice(&bytes) {
+ Ok(s) => s,
+ Err(e) => {
+ error!(
+ path = %path.display(), error = %e,
+ "cluster_state.json is corrupt — index templates, ingest \
+ pipelines, data streams and ILM policies are NOT restored"
+ );
+ // Keep a copy before the node runs on. The maps are empty
+ // now, so the very next `PUT /_index_template/...` rewrites
+ // this file and takes whatever was still legible in it with
+ // it — and hand-recovering a template out of a damaged
+ // document is the operator's last option, not one to destroy
+ // silently. The original is deliberately left in place so
+ // every boot keeps logging the error above until someone
+ // deals with it, and an existing copy is never clobbered:
+ // the first corruption is the informative one.
+ let salvage = path.with_extension("corrupt.json");
+ if !salvage.exists() {
+ match std::fs::write(&salvage, &bytes) {
+ Ok(()) => warn!(
+ path = %salvage.display(),
+ "kept a copy of the corrupt cluster_state for recovery"
+ ),
+ Err(e) => warn!(
+ path = %salvage.display(), error = %e,
+ "could not preserve a copy of the corrupt cluster_state"
+ ),
+ }
+ }
+ return;
+ }
+ };
+ if state.version > CLUSTER_STATE_VERSION {
+ warn!(
+ found = state.version,
+ supported = CLUSTER_STATE_VERSION,
+ "cluster_state.json was written by a newer xerj; loading what \
+ this build understands (downgrade may drop metadata)"
+ );
+ }
+
+ for (name, tmpl) in state.index_templates {
+ self.templates.insert(name, tmpl);
+ }
+ for (name, body) in state.legacy_templates {
+ self.legacy_templates.insert(name, body);
+ }
+ for (name, body) in state.component_templates {
+ self.component_templates.insert(name, body);
+ }
+ for (name, body) in state.ilm_policies {
+ self.ilm_policies.insert(name, body);
+ }
+
+ // Data streams: restore the record, reconcile it against what is
+ // actually on disk, then say plainly when a backing index it names is
+ // not there. Reporting a stream whose data is gone as
+ // `"status": "GREEN"` without a word in the log is the same quiet lie
+ // this issue is about.
+ let mut reconciled_a_stream = false;
+ for (name, mut ds) in state.data_streams {
+ // `rollover_data_stream` creates the new backing index before it
+ // records the new generation, so `kill -9` in that window leaves
+ // `.ds-<name>-00000N` on disk while the document still says N-1.
+ // Restoring N-1 verbatim would wedge the stream permanently: the
+ // next rollover computes the same name, and `create_index`
+ // refuses it as `index_already_exists` — forever. Adopt the
+ // highest generation that actually exists instead, which is also
+ // what stops a second rollover writing into an index that already
+ // holds the first one's documents.
+ let on_disk = self.highest_backing_generation(&name);
+ if on_disk > ds.generation {
+ let adopted = format!(".ds-{name}-{on_disk:06}");
+ warn!(
+ data_stream = name.as_str(),
+ recorded = ds.generation,
+ found = on_disk,
+ adopted = adopted.as_str(),
+ "adopting a backing index left behind by a rollover that \
+ was interrupted before its generation reached disk"
+ );
+ if !ds.backing_indices.contains(&adopted) {
+ ds.backing_indices.push(adopted);
+ }
+ ds.generation = on_disk;
+ reconciled_a_stream = true;
+ }
+
+ let missing: Vec<&str> = ds
+ .backing_indices
+ .iter()
+ .filter(|b| !self.indices.contains_key(b.as_str()))
+ .map(|b| b.as_str())
+ .collect();
+ if !missing.is_empty() {
+ warn!(
+ data_stream = name.as_str(),
+ missing = missing.join(","),
+ "restored data stream references backing indices that are \
+ not present in the data dir"
+ );
+ }
+ self.data_streams.insert(name, ds);
+ }
+
+ // Pipelines carry behaviour, not just a document to hand back from
+ // `GET /_ingest/pipeline`: the executable form has to be rebuilt or
+ // `?pipeline=x` would be accepted after a restart and quietly do
+ // nothing. Keep the stored document either way so GET round-trips
+ // exactly as it did before the restart.
+ let mut pipelines_restored = 0usize;
+ for (name, config) in state.pipelines {
+ self.pipelines.insert(name.clone(), config.clone());
+ match self.compile_pipeline(&name, config) {
+ Ok(()) => pipelines_restored += 1,
+ Err(e) => error!(
+ pipeline = name.as_str(), error = %e,
+ "persisted ingest pipeline could not be recompiled — it is \
+ still visible to GET /_ingest/pipeline but will NOT \
+ transform anything; re-PUT the definition"
+ ),
+ }
+ }
+
+ // Make an adopted generation durable straight away, so the recovery
+ // happens once rather than on every boot from here on.
+ if reconciled_a_stream {
+ if let Err(e) = self.flush_cluster_state() {
+ warn!(
+ error = %e,
+ "could not persist the reconciled data-stream generation; \
+ it will be recomputed from disk on the next boot"
+ );
+ }
+ }
+
+ let n = self.templates.len()
+ + self.legacy_templates.len()
+ + self.component_templates.len()
+ + self.pipelines.len()
+ + self.data_streams.len()
+ + self.ilm_policies.len();
+ if n > 0 {
+ info!(
+ index_templates = self.templates.len(),
+ legacy_templates = self.legacy_templates.len(),
+ component_templates = self.component_templates.len(),
+ pipelines = pipelines_restored,
+ data_streams = self.data_streams.len(),
+ ilm_policies = self.ilm_policies.len(),
+ "restored persisted cluster metadata"
+ );
+ }
+ }
+
+ /// Insert into a persisted map and durably record the result, rolling the
+ /// in-memory change back if the write fails so a 500 never leaves a
+ /// change that only this process can see.
+ fn persisted_insert<V: Clone>(
+ &self,
+ map: &DashMap<String, V>,
+ name: String,
+ value: V,
+ ) -> Result<()> {
+ let previous = map.insert(name.clone(), value);
+ if let Err(e) = self.flush_cluster_state() {
+ match previous {
+ Some(old) => map.insert(name, old),
+ None => map.remove(&name).map(|(_, v)| v),
+ };
+ return Err(e);
+ }
+ Ok(())
+ }
+
+ /// Remove from a persisted map and durably record the result. Returns
+ /// `false` when there was nothing to remove (the caller answers 404);
+ /// an `Err` means the removal did not reach disk and was undone.
+ fn persisted_remove<V: Clone>(&self, map: &DashMap<String, V>, name: &str) -> Result<bool> {
+ let Some((key, removed)) = map.remove(name) else {
+ return Ok(false);
+ };
+ if let Err(e) = self.flush_cluster_state() {
+ map.insert(key, removed);
+ return Err(e);
+ }
+ Ok(true)
+ }
+
+ /// Create or replace a v2 index template (`PUT /_index_template/{name}`).
+ pub fn put_index_template(&self, name: String, template: IndexTemplate) -> Result<()> {
+ self.persisted_insert(&self.templates, name, template)
+ }
+
+ /// Delete a v2 index template; `false` when it did not exist.
+ pub fn delete_index_template(&self, name: &str) -> Result<bool> {
+ self.persisted_remove(&self.templates, name)
+ }
+
+ /// Create or replace a legacy v1 template (`PUT /_template/{name}`).
+ pub fn put_legacy_template(&self, name: String, body: Value) -> Result<()> {
+ self.persisted_insert(&self.legacy_templates, name, body)
+ }
+
+ /// Delete a legacy v1 template; `false` when it did not exist.
+ pub fn delete_legacy_template(&self, name: &str) -> Result<bool> {
+ self.persisted_remove(&self.legacy_templates, name)
+ }
+
+ /// Create or replace a component template.
+ pub fn put_component_template(&self, name: String, body: Value) -> Result<()> {
+ self.persisted_insert(&self.component_templates, name, body)
+ }
+
+ /// Delete a component template; `false` when it did not exist.
+ pub fn delete_component_template(&self, name: &str) -> Result<bool> {
+ self.persisted_remove(&self.component_templates, name)
+ }
+
+ /// Create or replace an ILM policy.
+ pub fn put_ilm_policy(&self, name: String, body: Value) -> Result<()> {
+ self.persisted_insert(&self.ilm_policies, name, body)
+ }
+
+ /// Delete an ILM policy; `false` when it did not exist.
+ pub fn delete_ilm_policy(&self, name: &str) -> Result<bool> {
+ self.persisted_remove(&self.ilm_policies, name)
+ }
+
+ /// Durably store an ingest pipeline definition and (re)build its
+ /// executable form. This is the only way a user-visible pipeline should
+ /// ever be created — `compile_pipeline` alone does not reach disk.
+ ///
+ /// `config` is the xerj-shaped pipeline config to compile, and is what
+ /// gets stored (and handed back by `GET /_ingest/pipeline/{id}`) when it
+ /// compiles — the shape this endpoint has always returned.
+ ///
+ /// `keep_if_uncompilable` says what to do when it does *not* compile, and
+ /// differs by surface:
+ ///
+ /// * `Some(doc)` — the ES-compat surface. It has always answered
+ /// `acknowledged` and left the definition readable, so `doc` (the
+ /// ES-shaped body as received) is stored and the compile error is
+ /// returned as `Ok(Some(err))` for the caller to log.
+ /// * `None` — the xerj-native surface, which rejects a definition it
+ /// cannot compile. Nothing is stored, nothing is flushed, and any
+ /// pipeline already registered under this id is left untouched.
+ ///
+ /// Either way a compile failure drops the *executable* form under this
+ /// id. Leaving the old one behind would mean `?pipeline={id}` kept
+ /// running a definition `GET` no longer shows — and stopped running it at
+ /// the next restart. A definition must behave the same before and after a
+ /// reboot; that is the whole point of persisting it.
+ ///
+ /// `Err` means the definition did not reach disk. The in-memory change is
+ /// rolled back first, so the caller can report the write as failed
+ /// without leaving behind a pipeline only this process can see.
+ pub fn put_pipeline(
+ &self,
+ id: &str,
+ config: Value,
+ keep_if_uncompilable: Option<Value>,
+ ) -> Result<Option<xerj_wasm::WasmError>> {
+ let previous_doc = self.pipelines.get(id).map(|v| v.value().clone());
+ // Taken out up front so a failed compile cannot leave a stale
+ // executable behind; put back verbatim on every abort path.
+ let previous_compiled = self.transform_pipelines.remove(id).map(|(_, p)| p);
+
+ // `compile_pipeline` mutates nothing when it fails, so an abort only
+ // has to undo the two lines above.
+ let compile_err = match self.compile_pipeline(id, config) {
+ Ok(()) => None,
+ Err(e) => match keep_if_uncompilable {
+ None => {
+ if let Some(p) = previous_compiled {
+ self.transform_pipelines.insert(id.to_string(), p);
+ }
+ return Ok(Some(e));
+ }
+ Some(doc) => {
+ self.pipelines.insert(id.to_string(), doc);
+ Some(e)
+ }
+ },
+ };
+
+ if let Err(e) = self.flush_cluster_state() {
+ match previous_doc {
+ Some(d) => self.pipelines.insert(id.to_string(), d),
+ None => self.pipelines.remove(id).map(|(_, v)| v),
+ };
+ self.transform_pipelines.remove(id);
+ if let Some(p) = previous_compiled {
+ self.transform_pipelines.insert(id.to_string(), p);
+ }
+ return Err(e);
+ }
+
+ if let Some(e) = &compile_err {
+ warn!(pipeline = id, error = %e, "pipeline stored but failed to compile");
+ } else {
+ info!(name = id, "transform pipeline created");
+ }
+ Ok(compile_err)
+ }
+
+ /// Delete an ingest pipeline — both the stored definition and the
+ /// compiled form, so `?pipeline={name}` stops working at the same moment
+ /// `GET /_ingest/pipeline/{name}` starts answering 404. `false` when it
+ /// did not exist.
+ pub fn delete_pipeline(&self, name: &str) -> Result<bool> {
+ let removed = self.persisted_remove(&self.pipelines, name)?;
+ if removed {
+ self.transform_pipelines.remove(name);
+ }
+ Ok(removed)
+ }
+
/// Add an alias pointing to an index.
pub fn add_alias(&self, alias: &str, index: &str) {
let mut entry = self.aliases.entry(alias.to_string()).or_default();
@@ -2147,7 +2683,11 @@ impl Engine {
timestamp_field: "@timestamp".to_string(),
generation: 1,
};
- self.data_streams.insert(name.to_string(), ds);
+ // Durable before the caller is told it worked (issue #203): the
+ // backing index is already on disk, so a stream record that lives
+ // only in memory would come back after a restart as an orphaned
+ // `.ds-*` index with no stream in front of it.
+ self.persisted_insert(&self.data_streams, name.to_string(), ds)?;
info!(name, "data stream created");
Ok(())
}
@@ -2179,6 +2719,10 @@ impl Engine {
if let Some(mut ds) = self.data_streams.get_mut(name) {
ds.backing_indices.push(new_backing.clone());
}
+ // The generation counter is the thing that must not be lost: an
+ // unpersisted rollover would hand `.ds-<name>-000002` out twice after
+ // a restart and write into the previous generation's index.
+ self.flush_cluster_state()?;
info!(
name,
@@ -2196,6 +2740,17 @@ impl Engine {
.map(|(_, v)| v)
.ok_or_else(|| EngineError::Common(xerj_common::XerjError::index_not_found(name)))?;
+ // Record the removal before any backing index is destroyed: if the
+ // process dies mid-delete, a boot that still believes in the stream
+ // is recoverable (it warns about the missing backing indices), while
+ // one that has forgotten it leaves orphaned `.ds-*` directories no
+ // API can reach. A write failure here aborts the delete outright —
+ // nothing has been destroyed yet, so the stream is put back intact.
+ if let Err(e) = self.flush_cluster_state() {
+ self.data_streams.insert(name.to_string(), ds);
+ return Err(e);
+ }
+
// Remove the alias.
self.aliases.remove(name);
self.flush_aliases();
@@ -2309,15 +2864,19 @@ impl Engine {
// ── Transform pipeline methods ────────────────────────────────────────────
- /// Compile and register a typed transform pipeline from a JSON config.
+ /// Compile and register a typed transform pipeline from a JSON config,
+ /// **in memory only**.
///
/// `config_json` must be a valid [`PipelineConfig`](xerj_wasm::pipeline::PipelineConfig)
- /// object. The compiled pipeline is stored in `transform_pipelines` and
- /// can be retrieved by name for use at ingest time.
+ /// object. The compiled pipeline is stored in `transform_pipelines` and
+ /// the raw JSON in `pipelines`, so the ES-compatible ingest API can hand
+ /// it back.
///
- /// The raw JSON is also stored in `pipelines` so it can be returned by the
- /// ES-compatible ingest pipeline API.
- pub fn create_pipeline(
+ /// This is the primitive underneath [`Engine::put_pipeline`] and the boot
+ /// path; it deliberately does not touch disk, so it is private. Anything
+ /// that serves a user request must go through `put_pipeline`, or the
+ /// pipeline is gone at the next restart (issue #203).
+ fn compile_pipeline(
&self,
name: &str,
config_json: Value,
@@ -2327,7 +2886,6 @@ impl Engine {
let pipeline = xerj_wasm::pipeline::Pipeline::from_config(name, &cfg)?;
self.pipelines.insert(name.to_string(), config_json);
self.transform_pipelines.insert(name.to_string(), pipeline);
- info!(name, "transform pipeline created");
Ok(())
}
diff --git a/engine/crates/xerj-engine/tests/es_compat_tests.rs b/engine/crates/xerj-engine/tests/es_compat_tests.rs
index f273e3cb1..2c8f617a7 100644
--- a/engine/crates/xerj-engine/tests/es_compat_tests.rs
+++ b/engine/crates/xerj-engine/tests/es_compat_tests.rs
@@ -865,8 +865,8 @@ async fn test_es_bulk_per_item_pipeline() {
engine.create_index("bulk_pipe", Schema::empty()).unwrap();
// Register a pipeline that stamps `ingested = "yes"` on every doc.
- engine
- .create_pipeline(
+ let compile_err = engine
+ .put_pipeline(
"set_pipe",
json!({
"description": "stamp ingested marker",
@@ -874,8 +874,13 @@ async fn test_es_bulk_per_item_pipeline() {
{ "type": "set", "config": { "field": "ingested", "value": "yes", "override": true } }
]
}),
+ None,
)
- .expect("create_pipeline");
+ .expect("put_pipeline");
+ assert!(
+ compile_err.is_none(),
+ "pipeline must compile: {compile_err:?}"
+ );
let ndjson = concat!(
// Item 0: no pipeline → stored verbatim.
diff --git a/engine/crates/xerj-server/tests/cluster_metadata_survives_sigkill.rs b/engine/crates/xerj-server/tests/cluster_metadata_survives_sigkill.rs
new file mode 100644
index 000000000..63546b2df
--- /dev/null
+++ b/engine/crates/xerj-server/tests/cluster_metadata_survives_sigkill.rs
@@ -0,0 +1,243 @@
+//! Issue #203, against the real binary and a real `SIGKILL`.
+//!
+//! The in-process test (`xerj-api/tests/cluster_metadata_survives_restart.rs`)
+//! proves the reload path. This one proves the *durability* half: the state
+//! has to be on disk at the moment the PUT is acknowledged, not at some later
+//! shutdown hook, because the restart that matters is the one nobody planned.
+//! `kill -9` gives the process no chance to flush anything, so anything that
+//! comes back was already committed by `write_file_atomic` (write → fsync →
+//! rename → fsync parent).
+//!
+//! The write also has to be *atomic*, not merely eventual: this test hard-kills
+//! the node while it is answering, so a rewrite that truncated the file in
+//! place could leave a half-document that takes every template with it.
+//!
+//! Linux/macOS only — `SIGKILL` has no Windows equivalent, and
+//! `Child::kill()` there is a `TerminateProcess` that Windows may let the
+//! runtime observe. The property under test is platform-independent, but the
+//! way to provoke it is not.
+
+#![cfg(unix)]
+
+use std::io::{BufRead, BufReader, Read, Write};
+use std::net::{TcpListener, TcpStream};
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+/// Three distinct free ports — `Config::validate` requires all three listener
+/// ports to differ.
+fn three_free_ports() -> (u16, u16, u16) {
+ let l1 = TcpListener::bind("127.0.0.1:0").unwrap();
+ let l2 = TcpListener::bind("127.0.0.1:0").unwrap();
+ let l3 = TcpListener::bind("127.0.0.1:0").unwrap();
+ (
+ l1.local_addr().unwrap().port(),
+ l2.local_addr().unwrap().port(),
+ l3.local_addr().unwrap().port(),
+ )
+}
+
+fn toml_path(p: &std::path::Path) -> String {
+ p.to_string_lossy().replace('\\', "/")
+}
+
+/// One HTTP/1.1 request over a fresh connection, returning `(status, body)`.
+///
+/// Hand-rolled rather than pulled from a client crate so the test has no
+/// runtime of its own: it must stay a plain `#[test]`, since the thing it
+/// kills is a separate process and any async machinery here would only be
+/// another thing that can hang.
+fn request(
+ port: u16,
+ method: &str,
+ path: &str,
+ body: Option<&str>,
+) -> std::io::Result<(u16, String)> {
+ let mut stream = TcpStream::connect(("127.0.0.1", port))?;
+ stream.set_read_timeout(Some(Duration::from_secs(30)))?;
+ stream.set_write_timeout(Some(Duration::from_secs(30)))?;
+
+ let payload = body.unwrap_or("");
+ let request = format!(
+ "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\
+ Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{payload}",
+ payload.len()
+ );
+ stream.write_all(request.as_bytes())?;
+ stream.flush()?;
+
+ let mut reader = BufReader::new(stream);
+ let mut status_line = String::new();
+ reader.read_line(&mut status_line)?;
+ let status: u16 = status_line
+ .split_whitespace()
+ .nth(1)
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(0);
+
+ // Drain headers, then everything else — `Connection: close` means the
+ // body ends at EOF, so no chunk/length parsing is needed.
+ loop {
+ let mut line = String::new();
+ if reader.read_line(&mut line)? == 0 || line == "\r\n" {
+ break;
+ }
+ }
+ let mut body = String::new();
+ reader.read_to_string(&mut body)?;
+ Ok((status, body))
+}
+
+/// Spawn the real binary on `data_dir` and wait until it answers.
+fn boot(
+ data_dir: &std::path::Path,
+ cfg_dir: &std::path::Path,
+ es_port: u16,
+ rest: u16,
+ grpc: u16,
+) -> Child {
+ let config = format!(
+ r#"
+[server]
+bind_address = "127.0.0.1"
+rest_port = {rest}
+grpc_port = {grpc}
+es_compat_port = {es_port}
+data_dir = "{data}"
+"#,
+ data = toml_path(data_dir),
+ );
+ let config_path = cfg_dir.join(format!("xerj-{es_port}.toml"));
+ std::fs::write(&config_path, config).unwrap();
+
+ let mut child = Command::new(env!("CARGO_BIN_EXE_xerj"))
+ .arg("--config")
+ .arg(&config_path)
+ .arg("--insecure")
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .spawn()
+ .expect("spawn xerj");
+
+ let deadline = Instant::now() + Duration::from_secs(120);
+ loop {
+ if let Ok((200, _)) = request(es_port, "GET", "/_cluster/health", None) {
+ return child;
+ }
+ if Instant::now() >= deadline {
+ break;
+ }
+ std::thread::sleep(Duration::from_millis(200));
+ }
+ // Never leave a half-booted node behind for the rest of the suite to
+ // trip over — reap it, then fail.
+ let _ = child.kill();
+ let _ = child.wait();
+ panic!("xerj did not become ready on port {es_port}");
+}
+
+/// `kill -9` and reap. No graceful shutdown, no flush hook, no chance to
+/// write anything the acknowledged PUTs did not already write.
+fn sigkill(mut child: Child) {
+ unsafe {
+ libc::kill(child.id() as libc::pid_t, libc::SIGKILL);
+ }
+ let status = child.wait().expect("reap child");
+ assert!(
+ !status.success(),
+ "the node was supposed to be hard-killed, but it exited cleanly ({status:?}) — \
+ a graceful exit could have flushed state that a real crash never would"
+ );
+}
+
+#[test]
+fn cluster_metadata_survives_kill_dash_nine() {
+ let dir = tempfile::tempdir().unwrap();
+ let data_dir = dir.path().join("data");
+ std::fs::create_dir_all(&data_dir).unwrap();
+
+ let (rest, grpc, es) = three_free_ports();
+ let child = boot(&data_dir, dir.path(), es, rest, grpc);
+
+ let (st, b) = request(
+ es,
+ "PUT",
+ "/_index_template/logs",
+ Some(r#"{"index_patterns":["logs-*"],"priority":200,"template":{"mappings":{"properties":{"host":{"type":"keyword"}}}}}"#),
+ )
+ .unwrap();
+ assert_eq!(st, 200, "PUT _index_template: {b}");
+
+ let (st, b) = request(
+ es,
+ "PUT",
+ "/_ingest/pipeline/tagger",
+ Some(r#"{"description":"tag","processors":[{"set":{"field":"tagged","value":"yes"}}]}"#),
+ )
+ .unwrap();
+ assert_eq!(st, 200, "PUT _ingest/pipeline: {b}");
+
+ let (st, b) = request(
+ es,
+ "PUT",
+ "/_ilm/policy/hot-warm",
+ Some(r#"{"policy":{"phases":{"hot":{"actions":{}}}}}"#),
+ )
+ .unwrap();
+ assert_eq!(st, 200, "PUT _ilm/policy: {b}");
+
+ let (st, b) = request(es, "PUT", "/_data_stream/metrics-app", Some("{}")).unwrap();
+ assert_eq!(st, 200, "PUT _data_stream: {b}");
+
+ // Roll the stream forward so the restored generation counter has to be
+ // 2, not the 1 a re-create would produce.
+ let (st, b) = request(es, "POST", "/metrics-app/_rollover", Some("{}")).unwrap();
+ assert_eq!(st, 200, "POST _rollover: {b}");
+ assert!(
+ b.contains(".ds-metrics-app-000002"),
+ "rollover should mint generation 2: {b}"
+ );
+
+ sigkill(child);
+
+ // Same data dir, brand-new process. Everything above was acknowledged,
+ // so everything above must still be here.
+ let child = boot(&data_dir, dir.path(), es, rest, grpc);
+
+ let (st, b) = request(es, "GET", "/_index_template/logs", None).unwrap();
+ assert_eq!(st, 200, "index template lost across kill -9: {b}");
+ assert!(
+ b.contains("logs-*") && b.contains("keyword"),
+ "partial template restore: {b}"
+ );
+
+ let (st, b) = request(es, "GET", "/_ingest/pipeline/tagger", None).unwrap();
+ assert_eq!(st, 200, "ingest pipeline lost across kill -9: {b}");
+
+ let (st, b) = request(es, "GET", "/_ilm/policy/hot-warm", None).unwrap();
+ assert_eq!(st, 200, "ILM policy lost across kill -9: {b}");
+
+ let (st, b) = request(es, "GET", "/_data_stream/metrics-app", None).unwrap();
+ assert_eq!(st, 200, "data stream lost across kill -9: {b}");
+ assert!(
+ b.contains("\"generation\":2") && b.contains(".ds-metrics-app-000002"),
+ "the rollover must survive too, or the next rollover reissues a \
+ backing index that already holds data: {b}"
+ );
+
+ // The restored pipeline must still *run*, not just be readable.
+ let (st, b) = request(
+ es,
+ "POST",
+ "/_ingest/pipeline/tagger/_simulate",
+ Some(r#"{"docs":[{"_source":{"host":"a"}}]}"#),
+ )
+ .unwrap();
+ assert_eq!(st, 200, "simulate after restart: {b}");
+ assert!(
+ b.contains("\"tagged\":\"yes\""),
+ "a restored pipeline that no longer transforms is accepted-and-ignored: {b}"
+ );
+
+ sigkill(child);
+}