AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): persist templates, pipelines, data streams and ILM policies (#203)
xerj-orge9798abf2cbd
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 3d84e0549..2d7cf9d92 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -17374,8 +17374,10 @@ pub async fn put_index_template(mappings,priority: body.priority.unwrap_or(0),};- state.engine.templates.insert(name, tmpl);- Json(json!({ "acknowledged": true })).into_response()+ match state.engine.put_index_template(name, tmpl) {+ Ok(()) => Json(json!({ "acknowledged": true })).into_response(),+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ }}pub async fn get_index_template(@@ -17435,11 +17437,14 @@ pub async fn delete_index_template(State(state): State<AppState>,Path(name): Path<String>,) -> impl IntoResponse {- if state.engine.templates.remove(&name).is_some() {- Json(json!({ "acknowledged": true })).into_response()- } else {- let e = xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));- ApiError::new(e).into_response()+ match state.engine.delete_index_template(&name) {+ Ok(true) => Json(json!({ "acknowledged": true })).into_response(),+ Ok(false) => {+ let e =+ xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));+ ApiError::new(e).into_response()+ }+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),}}@@ -22151,8 +22156,6 @@ pub async fn put_ingest_pipeline(Path(id): Path<String>,Json(body): Json<Value>,) -> impl IntoResponse {- // Store the raw JSON for GET /_ingest/pipeline.- state.engine.pipelines.insert(id.clone(), body.clone());// Convert ES processor format → xerj stage format, then compile.// ES: {"processors": [{"set": {"field":"x","value":"y"}}]}// xerj: {"stages": [{"type": "set", "config": {"field":"x","value":"y"}}]}@@ -22207,10 +22210,17 @@ pub async fn put_ingest_pipeline(} else {body.clone()};- if let Err(e) = state.engine.create_pipeline(&id, xerj_config) {- tracing::warn!(pipeline = %id, error = %e, "pipeline stored but failed to compile");+ // `Some(body)` keeps this surface's long-standing behaviour: a definition+ // that will not compile is still stored and still readable through+ // `GET /_ingest/pipeline/{id}` (the engine logs the compile error and+ // drops the executable form, so `?pipeline={id}` behaves identically+ // before and after a restart). An `Err` means it did not reach disk —+ // answering `acknowledged` there would promise a pipeline that is gone at+ // the next boot, which is the whole of issue #203.+ match state.engine.put_pipeline(&id, xerj_config, Some(body)) {+ Ok(_) => Json(json!({ "acknowledged": true })).into_response(),+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),}- Json(json!({ "acknowledged": true })).into_response()}pub async fn get_ingest_pipeline(@@ -22240,11 +22250,13 @@ pub async fn delete_ingest_pipeline(State(state): State<AppState>,Path(id): Path<String>,) -> impl IntoResponse {- if state.engine.pipelines.remove(&id).is_some() {- Json(json!({ "acknowledged": true })).into_response()- } else {- let e = xerj_common::XerjError::index_not_found(format!("pipeline [{id}] is missing"));- ApiError::new(e).into_response()+ match state.engine.delete_pipeline(&id) {+ Ok(true) => Json(json!({ "acknowledged": true })).into_response(),+ Ok(false) => {+ let e = xerj_common::XerjError::index_not_found(format!("pipeline [{id}] is missing"));+ ApiError::new(e).into_response()+ }+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),}}@@ -24062,8 +24074,15 @@ pub async fn put_ilm_policy() -> impl IntoResponse {// Persist the raw policy verbatim in the ILM store; `get_ilm_policy`// reads it back out of this same DashMap, so PUT then GET round-trips- // faithfully regardless of whether translation below succeeds.- state.engine.ilm_policies.insert(name.clone(), body.clone());+ // faithfully regardless of whether translation below succeeds — and the+ // store is written to `<data_dir>/cluster_state.json`, so it round-trips+ // after a restart too (issue #203). This write goes first because it is+ // the one that can fail loudly: if the document cannot be persisted the+ // request is a 500 and nothing has happened yet, rather than an+ // `acknowledged` that a reboot silently undoes.+ if let Err(e) = state.engine.put_ilm_policy(name.clone(), body.clone()) {+ return ApiError::new(xerj_common::XerjError::from(e)).into_response();+ }// Translate into the shared internal model (`xerj_engine::lifecycle`)// so the SAME background job that drives native ISM policies also@@ -24131,12 +24150,21 @@ pub async fn delete_ilm_policy(State(state): State<AppState>,Path(name): Path<String>,) -> impl IntoResponse {- if state.engine.ilm_policies.remove(&name).is_some() {- state.engine.remove_ism_policy(&name);- Json(json!({ "acknowledged": true })).into_response()- } else {- let e = xerj_common::XerjError::index_not_found(format!("policy [{name}] not found"));- ApiError::new(e).into_response()+ // Remove the verbatim document first: that is the guarded, persisted+ // path, so a failure there is a 500 with nothing removed anywhere. Only+ // once it is gone from `cluster_state.json` is the translated executor+ // policy dropped, so the two files can never disagree in the direction+ // that keeps running a policy `GET _ilm/policy` reports as deleted.+ match state.engine.delete_ilm_policy(&name) {+ Ok(true) => {+ state.engine.remove_ism_policy(&name);+ Json(json!({ "acknowledged": true })).into_response()+ }+ Ok(false) => {+ let e = xerj_common::XerjError::index_not_found(format!("policy [{name}] not found"));+ ApiError::new(e).into_response()+ }+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),}}@@ -24235,8 +24263,10 @@ pub async fn put_component_template(Path(name): Path<String>,Json(body): Json<Value>,) -> impl IntoResponse {- state.engine.component_templates.insert(name, body);- Json(json!({ "acknowledged": true })).into_response()+ match state.engine.put_component_template(name, body) {+ Ok(()) => Json(json!({ "acknowledged": true })).into_response(),+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ }}pub async fn get_component_template(@@ -24278,12 +24308,15 @@ pub async fn delete_component_template(State(state): State<AppState>,Path(name): Path<String>,) -> impl IntoResponse {- if state.engine.component_templates.remove(&name).is_some() {- Json(json!({ "acknowledged": true })).into_response()- } else {- let e =- xerj_common::XerjError::index_not_found(format!("component template [{name}] missing"));- ApiError::new(e).into_response()+ match state.engine.delete_component_template(&name) {+ Ok(true) => Json(json!({ "acknowledged": true })).into_response(),+ Ok(false) => {+ let e = xerj_common::XerjError::index_not_found(format!(+ "component template [{name}] missing"+ ));+ ApiError::new(e).into_response()+ }+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),}}@@ -33932,8 +33965,10 @@ pub async fn put_legacy_template(if let Some(settings) = body.get("settings").cloned() {body["settings"] = flatten_template_settings(&settings);}- state.engine.legacy_templates.insert(name, body);- Json(json!({ "acknowledged": true }))+ match state.engine.put_legacy_template(name, body) {+ Ok(()) => Json(json!({ "acknowledged": true })).into_response(),+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ }}/// Normalize a template's `settings` block to ES's storage form: every@@ -34030,11 +34065,14 @@ pub async fn delete_legacy_template(State(state): State<AppState>,Path(name): Path<String>,) -> impl IntoResponse {- if state.engine.legacy_templates.remove(&name).is_some() {- Json(json!({ "acknowledged": true })).into_response()- } else {- let e = xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));- ApiError::new(e).into_response()+ match state.engine.delete_legacy_template(&name) {+ Ok(true) => Json(json!({ "acknowledged": true })).into_response(),+ Ok(false) => {+ let e =+ xerj_common::XerjError::index_not_found(format!("index template [{name}] missing"));+ ApiError::new(e).into_response()+ }+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),}}diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rsindex c008077bc..3aa0f1534 100644--- a/engine/crates/xerj-api/src/native.rs+++ b/engine/crates/xerj-api/src/native.rs@@ -2486,8 +2486,13 @@ pub async fn put_pipeline(let started = Instant::now();let request_id = Uuid::new_v4().to_string();- match state.engine.create_pipeline(&name, body) {- Ok(()) => {+ // `None` — this surface rejects a definition it cannot compile, so+ // nothing is stored on a compile error. On success the pipeline is+ // written to `<data_dir>/cluster_state.json` before we acknowledge it+ // (issue #203): it used to live only in memory and vanish on restart,+ // exactly like the ES-compat one.+ match state.engine.put_pipeline(&name, body, None) {+ Ok(None) => {let took_ms = started.elapsed().as_millis() as u64;let resp = NativeResponse::new(serde_json::json!({ "pipeline": name, "acknowledged": true }),@@ -2496,6 +2501,13 @@ pub async fn put_pipeline();(StatusCode::OK, Json(resp)).into_response()}+ // The definition does not compile — same 500 this surface has always+ // returned, and nothing was stored.+ Ok(Some(compile_err)) => {+ let ze = xerj_common::XerjError::internal(compile_err.to_string());+ native_error(ze, Some(&request_id), started.elapsed().as_millis() as u64)+ .into_response()+ }Err(e) => {let ze = xerj_common::XerjError::internal(e.to_string());native_error(ze, Some(&request_id), started.elapsed().as_millis() as u64)diff --git a/engine/crates/xerj-api/tests/cluster_metadata_survives_restart.rs b/engine/crates/xerj-api/tests/cluster_metadata_survives_restart.rsnew file mode 100644index 000000000..19a5f83f0--- /dev/null+++ b/engine/crates/xerj-api/tests/cluster_metadata_survives_restart.rs@@ -0,0 +1,447 @@+//! Issue #203: index templates, legacy templates, component templates,+//! ingest pipelines, data streams and ILM policies must survive a restart.+//!+//! Before the fix each of these lived only in an in-memory `DashMap` on+//! `Engine`. `Engine::new` restored index directories, `es_mapping.json`,+//! API keys and aliases — nothing else — so a `PUT _index_template/logs`+//! answered `200 {"acknowledged": true}` and then evaporated on the next+//! boot. The operator got no error, and the next index that should have+//! matched the template was created without it.+//!+//! Elasticsearch is referenced for wire semantics only; no ES code is+//! reproduced here.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++fn config_for(dir: &std::path::Path) -> xerj_common::config::Config {+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ config+}++/// Build a fresh app over `dir`. Calling this twice (with the first engine+/// dropped in between) is a restart: nothing is handed over in memory, the+/// second boot sees only what reached the disk.+fn app_over(dir: &std::path::Path) -> axum::Router {+ let config = config_for(dir);+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ xerj_api::router::build_es_compat_router(state)+}++async fn send(app: &axum::Router, req: Request<Body>) -> (StatusCode, Value) {+ let response = app.clone().oneshot(req).await.expect("response");+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .expect("body");+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+}++async fn put_json(app: &axum::Router, path: &str, body: Value) -> (StatusCode, Value) {+ send(+ app,+ Request::put(path)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .expect("request"),+ )+ .await+}++async fn get(app: &axum::Router, path: &str) -> (StatusCode, Value) {+ send(+ app,+ Request::get(path).body(Body::empty()).expect("request"),+ )+ .await+}++async fn delete(app: &axum::Router, path: &str) -> (StatusCode, Value) {+ send(+ app,+ Request::delete(path).body(Body::empty()).expect("request"),+ )+ .await+}++/// Write one of each managed object. Kept separate from the assertions so+/// the same fixture can be replayed after a restart.+async fn write_all_the_config(app: &axum::Router) {+ let (st, _) = put_json(+ app,+ "/_index_template/logs",+ json!({+ "index_patterns": ["logs-*"],+ "priority": 200,+ "template": {+ "settings": { "number_of_replicas": 0 },+ "mappings": { "properties": { "host": { "type": "keyword" } } }+ }+ }),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "PUT _index_template");++ let (st, _) = put_json(+ app,+ "/_template/legacy-logs",+ json!({+ "index_patterns": ["legacy-*"],+ "settings": { "number_of_shards": 1 },+ "mappings": { "properties": { "msg": { "type": "text" } } }+ }),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "PUT _template");++ let (st, _) = put_json(+ app,+ "/_component_template/base-settings",+ json!({ "template": { "settings": { "number_of_replicas": 1 } } }),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "PUT _component_template");++ let (st, _) = put_json(+ app,+ "/_ingest/pipeline/tagger",+ json!({+ "description": "tag every doc",+ "processors": [ { "set": { "field": "tagged", "value": "yes" } } ]+ }),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "PUT _ingest/pipeline");++ let (st, _) = put_json(+ app,+ "/_ilm/policy/hot-warm",+ json!({ "policy": { "phases": { "hot": { "actions": {} } } } }),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "PUT _ilm/policy");++ let (st, _) = put_json(app, "/_data_stream/metrics-app", json!({})).await;+ assert_eq!(st, StatusCode::OK, "PUT _data_stream");+}++/// The read side of each managed object.+const READ_BACK: [&str; 6] = [+ "/_index_template/logs",+ "/_template/legacy-logs",+ "/_component_template/base-settings",+ "/_ingest/pipeline/tagger",+ "/_ilm/policy/hot-warm",+ "/_data_stream/metrics-app",+];++/// Read every managed object back, asserting each is present, and return the+/// exact responses so a later boot can be compared against them verbatim.+///+/// Comparing whole bodies is deliberate: it catches a partial restore (a+/// template that comes back without its mappings, a data stream that comes+/// back at generation 1) that a spot-check on one field would wave through.+async fn read_back_everything(app: &axum::Router, when: &str) -> Vec<Value> {+ let mut out = Vec::new();+ for path in READ_BACK {+ let (st, body) = get(app, path).await;+ assert_eq!(st, StatusCode::OK, "GET {path} {when}: {body}");+ out.push(body);+ }++ // Enough content assertions to prove the fixture really was written, so+ // an all-empty "equal" comparison cannot pass for a restore.+ let tmpl = &out[0]["index_templates"][0]["index_template"];+ assert_eq!(tmpl["index_patterns"][0], "logs-*", "{when}: {}", out[0]);+ assert_eq!(tmpl["priority"], 200, "{when}: {}", out[0]);+ assert_eq!(+ tmpl["template"]["mappings"]["properties"]["host"]["type"], "keyword",+ "{when}: {}",+ out[0]+ );+ assert_eq!(+ out[1]["legacy-logs"]["index_patterns"][0], "legacy-*",+ "{when}: {}",+ out[1]+ );+ assert_eq!(+ out[2]["component_templates"][0]["component_template"]["template"]["settings"]+ ["number_of_replicas"],+ 1,+ "{when}: {}",+ out[2]+ );+ assert_eq!(+ out[5]["data_streams"][0]["name"], "metrics-app",+ "{when}: {}",+ out[5]+ );+ out+}++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn templates_pipelines_streams_and_policies_survive_a_restart() {+ let dir = tempfile::tempdir().expect("tempdir");++ let before = {+ let app = app_over(dir.path());+ write_all_the_config(&app).await;+ // Sanity: it is all readable *before* the restart, so a failure+ // below is about durability and not about the write path.+ read_back_everything(&app, "before restart").await+ }; // engine dropped — the node.lock is released, nothing is handed over++ let app = app_over(dir.path());+ let after = read_back_everything(&app, "after restart").await;++ for (path, (b, a)) in READ_BACK.iter().zip(before.iter().zip(after.iter())) {+ assert_eq!(+ b, a,+ "GET {path} answered differently after a restart\nbefore: {b}\nafter: {a}"+ );+ }+}++/// A restored ingest pipeline must actually *run*, not merely round-trip+/// through `GET /_ingest/pipeline`. Storing the raw JSON without+/// recompiling it would leave `?pipeline=tagger` accepted and ignored.+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn a_restored_pipeline_still_transforms_documents() {+ let dir = tempfile::tempdir().expect("tempdir");++ {+ let app = app_over(dir.path());+ write_all_the_config(&app).await;+ }++ let app = app_over(dir.path());+ let (st, body) = send(+ &app,+ Request::post("/_ingest/pipeline/tagger/_simulate")+ .header("content-type", "application/json")+ .body(Body::from(+ json!({ "docs": [ { "_source": { "host": "a" } } ] }).to_string(),+ ))+ .expect("request"),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "simulate after restart: {body}");+ assert_eq!(+ body["docs"][0]["doc"]["_source"]["tagged"], "yes",+ "the restored pipeline must still execute its processors: {body}"+ );+}++/// Deletes must be durable too — a deleted template that comes back on the+/// next boot is the same class of surprise as one that disappears.+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn a_deleted_template_stays_deleted_across_a_restart() {+ let dir = tempfile::tempdir().expect("tempdir");++ {+ let app = app_over(dir.path());+ write_all_the_config(&app).await;+ let (st, _) = delete(&app, "/_index_template/logs").await;+ assert_eq!(st, StatusCode::OK, "DELETE _index_template");+ let (st, _) = delete(&app, "/_ingest/pipeline/tagger").await;+ assert_eq!(st, StatusCode::OK, "DELETE _ingest/pipeline");+ }++ let app = app_over(dir.path());+ let (st, _) = get(&app, "/_index_template/logs").await;+ assert_eq!(+ st,+ StatusCode::NOT_FOUND,+ "a deleted index template must not resurrect on restart"+ );+ let (st, _) = get(&app, "/_ingest/pipeline/tagger").await;+ assert_eq!(+ st,+ StatusCode::NOT_FOUND,+ "a deleted pipeline must not resurrect on restart"+ );+}++async fn post(app: &axum::Router, path: &str, body: Value) -> (StatusCode, Value) {+ send(+ app,+ Request::post(path)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .expect("request"),+ )+ .await+}++/// A rollover interrupted between "backing index created" and "generation+/// written" must not wedge the stream.+///+/// `rollover_data_stream` creates `.ds-<name>-00000N` on disk first and only+/// then flushes the new generation, so a `kill -9` in that window leaves the+/// index present and the document still saying N-1. Restoring N-1 verbatim+/// makes the next rollover recompute the *same* name, which `create_index`+/// refuses as `index_already_exists` — and it does so on every subsequent+/// attempt, so the stream can never roll again.+///+/// The crash window is reproduced exactly rather than approximated: the+/// backing index is left on disk and the persisted generation is rewound by+/// hand, which is byte-for-byte the state a crash at that instant leaves.+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn a_rollover_interrupted_before_its_flush_does_not_wedge_the_stream() {+ let dir = tempfile::tempdir().expect("tempdir");++ {+ let app = app_over(dir.path());+ let (st, _) = put_json(&app, "/_data_stream/metrics-app", json!({})).await;+ assert_eq!(st, StatusCode::OK, "PUT _data_stream");+ let (st, body) = post(&app, "/metrics-app/_rollover", json!({})).await;+ assert_eq!(st, StatusCode::OK, "rollover: {body}");+ }++ // Rewind only the generation, exactly as a crash between `create_index`+ // and `flush_cluster_state` would have left it: `.ds-metrics-app-000002`+ // is on disk, the document still reads generation 1.+ let state_path = dir.path().join("cluster_state.json");+ let mut state: Value =+ serde_json::from_slice(&std::fs::read(&state_path).expect("read state")).expect("parse");+ let ds = &mut state["data_streams"]["metrics-app"];+ ds["generation"] = json!(1);+ ds["backing_indices"] = json!([".ds-metrics-app-000001"]);+ std::fs::write(&state_path, serde_json::to_vec(&state).expect("serialize")).expect("write");+ assert!(+ dir.path().join(".ds-metrics-app-000002").is_dir(),+ "fixture is wrong: the generation-2 backing index should still be on disk"+ );++ let app = app_over(dir.path());+ let (st, body) = post(&app, "/metrics-app/_rollover", json!({})).await;+ assert_eq!(+ st,+ StatusCode::OK,+ "a stream recovered from an interrupted rollover must still roll: {body}"+ );+ assert!(+ body.to_string().contains(".ds-metrics-app-000003"),+ "the next rollover must skip past the backing index the crash left \+ behind, not reissue it: {body}"+ );+}++/// A corrupt `cluster_state.json` must not be destroyed by the next write.+///+/// The node boots with empty maps and logs the failure, so the first+/// `PUT /_index_template/...` rewrites the file — taking whatever was still+/// legible in it along. Hand-recovery out of a damaged document is the+/// operator's last option; it should not be silently removed.+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn a_corrupt_cluster_state_is_preserved_before_it_is_overwritten() {+ let dir = tempfile::tempdir().expect("tempdir");++ {+ let app = app_over(dir.path());+ write_all_the_config(&app).await;+ }++ // Truncate mid-document — the shape a torn write would have had, if the+ // rewrite were not atomic, and the shape media damage leaves.+ let state_path = dir.path().join("cluster_state.json");+ let original = std::fs::read_to_string(&state_path).expect("read state");+ let truncated = &original[..original.len() / 2];+ std::fs::write(&state_path, truncated).expect("write truncated");++ let app = app_over(dir.path());+ let (st, _) = get(&app, "/_index_template/logs").await;+ assert_eq!(+ st,+ StatusCode::NOT_FOUND,+ "a corrupt document cannot be restored — this is the honest outcome"+ );++ // The write that would have destroyed the evidence.+ let (st, _) = put_json(+ &app,+ "/_index_template/fresh",+ json!({ "index_patterns": ["fresh-*"] }),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "PUT after corrupt load");++ let salvage = dir.path().join("cluster_state.corrupt.json");+ assert!(+ salvage.is_file(),+ "the corrupt document must be kept for recovery"+ );+ assert_eq!(+ std::fs::read_to_string(&salvage).expect("read salvage"),+ truncated,+ "the preserved copy must be the bytes that were found, unmodified"+ );+ assert!(+ std::fs::read_to_string(&state_path)+ .expect("read state")+ .contains("fresh-*"),+ "the live document must have moved on"+ );+}++/// Concurrent writers must not trip over each other.+///+/// The atomic rewrite stages through one fixed temp path, so two flushes+/// running at once create, truncate and rename the *same* file: one loses+/// its bytes to the other's truncate, or finds the temp file already renamed+/// away. A provisioning script that fires its `PUT`s in parallel is all it+/// takes — no stress needed. Measured without the engine's write lock, 32+/// requests over a multi-threaded runtime fail immediately:+///+/// ```text+/// PUT template t2: {"error":{"type":"store_exception",+/// "reason":"storage error: IO error"},"status":500}+/// ```+#[tokio::test(flavor = "multi_thread", worker_threads = 8)]+async fn concurrent_writes_do_not_corrupt_the_store() {+ const N: usize = 32;+ let dir = tempfile::tempdir().expect("tempdir");++ {+ let app = app_over(dir.path());+ let mut writes = Vec::with_capacity(N);+ for i in 0..N {+ let app = app.clone();+ writes.push(tokio::spawn(async move {+ put_json(+ &app,+ &format!("/_index_template/t{i}"),+ json!({ "index_patterns": [format!("t{i}-*")], "priority": i }),+ )+ .await+ }));+ }+ for (i, w) in writes.into_iter().enumerate() {+ let (st, body) = w.await.expect("join");+ assert_eq!(st, StatusCode::OK, "PUT template t{i}: {body}");+ }+ }++ let app = app_over(dir.path());+ for i in 0..N {+ let (st, body) = get(&app, &format!("/_index_template/t{i}")).await;+ assert_eq!(+ st,+ StatusCode::OK,+ "template t{i} did not survive concurrent writes + restart: {body}"+ );+ assert_eq!(+ body["index_templates"][0]["index_template"]["priority"], i,+ "template t{i} came back with the wrong contents: {body}"+ );+ }+}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 89dc6a254..aa51406d6 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -138,6 +138,45 @@ pub struct IndexTemplate {pub priority: i32,}+/// Version stamped into `cluster_state.json`. Bump only when the on-disk+/// shape changes in a way an older build cannot read; new *optional* fields+/// (`#[serde(default)]`) do not need one.+const CLUSTER_STATE_VERSION: u32 = 1;++/// Everything in `<data_dir>/cluster_state.json` — the cluster-level+/// management state that used to be in-memory only (issue #203).+///+/// `BTreeMap` rather than `HashMap` so the file is byte-stable for a given+/// state: an operator diffing two nodes' `cluster_state.json`, or a backup+/// tool deduplicating it, sees a change only when the state really changed.+/// Every field is `#[serde(default)]`, so a file written by an older build+/// (or by one that gains another map later) still loads.+#[derive(Debug, Default, Serialize, Deserialize)]+struct PersistedClusterState {+ #[serde(default)]+ version: u32,+ /// v2 index templates — `PUT /_index_template/{name}`.+ #[serde(default)]+ index_templates: std::collections::BTreeMap<String, IndexTemplate>,+ /// v1 legacy templates — `PUT /_template/{name}`.+ #[serde(default)]+ legacy_templates: std::collections::BTreeMap<String, Value>,+ /// Component templates — `PUT /_component_template/{name}`.+ #[serde(default)]+ component_templates: std::collections::BTreeMap<String, Value>,+ /// Ingest pipeline definitions — `PUT /_ingest/pipeline/{id}`. Stored as+ /// the document `GET` hands back, and recompiled into an executable+ /// pipeline on boot.+ #[serde(default)]+ pipelines: std::collections::BTreeMap<String, Value>,+ /// Data streams — `PUT /_data_stream/{name}` plus every rollover.+ #[serde(default)]+ data_streams: std::collections::BTreeMap<String, DataStream>,+ /// ILM policies — `PUT /_ilm/policy/{name}`.+ #[serde(default)]+ ilm_policies: std::collections::BTreeMap<String, Value>,+}+/// Active scroll context holding all matching hits.////// Each context pins a fully-hydrated `Vec<Hit>` snapshot, so its lifetime@@ -417,15 +456,24 @@ pub struct Engine {data_dir: PathBuf,/// alias_name → list of index namespub aliases: Arc<DashMap<String, Vec<String>>>,- /// template_name → IndexTemplate+ /// template_name → IndexTemplate. Persisted in+ /// `<data_dir>/cluster_state.json`; mutate only through+ /// [`Engine::put_index_template`] / [`Engine::delete_index_template`],+ /// or the change will not survive a restart (issue #203).pub templates: Arc<DashMap<String, IndexTemplate>>,/// scroll_id → ScrollContextpub scrolls: Arc<DashMap<String, ScrollContext>>,- /// pipeline_id → pipeline definition JSON+ /// pipeline_id → pipeline definition JSON. Persisted in+ /// `<data_dir>/cluster_state.json`; mutate only through+ /// [`Engine::put_pipeline`] / [`Engine::delete_pipeline`], or the change+ /// will not survive a restart (issue #203).pub pipelines: Arc<DashMap<String, Value>>,/// index_name → open/closed state (true = closed)pub closed_indices: Arc<DashMap<String, bool>>,- /// data stream name → DataStream+ /// data stream name → DataStream. Persisted in+ /// `<data_dir>/cluster_state.json`; mutate only through the+ /// `*_data_stream` methods, or the change will not survive a restart+ /// (issue #203).pub data_streams: Arc<DashMap<String, DataStream>>,/// ILM policy name → raw policy JSON, kept verbatim for `GET _ilm/policy`/// round-trip fidelity. The REAL execution engine reads `ism_policies`@@ -433,10 +481,19 @@ pub struct Engine {/// `xerj_api::es_compat::put_ilm_policy`), translating into the shared/// internal model. See `crate::lifecycle` for why one engine drives both/// surfaces.+ ///+ /// This map — the verbatim ILM document — is persisted in+ /// `<data_dir>/cluster_state.json`; mutate only through+ /// [`Engine::put_ilm_policy`] / [`Engine::delete_ilm_policy`], or the+ /// change will not survive a restart (issue #203). The translated+ /// `ism_policies` model has its own file (`ism_policies.json`, issue+ /// #199) — the two never write each other's file.pub ilm_policies: Arc<DashMap<String, Value>>,/// policy id → the internal ISM-shaped model every managed index is/// actually driven by, regardless of whether the policy was created via/// `_plugins/_ism/policies` (native) or `_ilm/policy` (translated).+ /// Persisted separately in `<data_dir>/ism_policies.json` — see+ /// [`Engine::put_ism_policy`].pub ism_policies: Arc<DashMap<String, crate::lifecycle::LifecyclePolicy>>,/// index name → lifecycle execution cursor (current state, pending/// action, timestamps). Presence in this map is what "managed" means —@@ -446,7 +503,11 @@ pub struct Engine {/// `GET /{index}`'s `creation_date` (previously synthesized as/// `Utc::now()` on every request — see `record_index_created_at`).pub index_created_at: Arc<DashMap<String, i64>>,- /// component template name → template JSON+ /// component template name → template JSON. Persisted in+ /// `<data_dir>/cluster_state.json`; mutate only through+ /// [`Engine::put_component_template`] /+ /// [`Engine::delete_component_template`], or the change will not survive+ /// a restart (issue #203).pub component_templates: Arc<DashMap<String, Value>>,/// snapshot repository name → repo config JSONpub snapshot_repos: Arc<DashMap<String, Value>>,@@ -488,7 +549,10 @@ pub struct Engine {/// subsequent poll, but nothing actually gates on them yet./// In-memory only (lost on restart).pub application_privileges: Arc<DashMap<String, Value>>,- /// legacy index template name (v1 /_template) → template JSON+ /// legacy index template name (v1 /_template) → template JSON.+ /// Persisted in `<data_dir>/cluster_state.json`; mutate only through+ /// [`Engine::put_legacy_template`] / [`Engine::delete_legacy_template`],+ /// or the change will not survive a restart (issue #203).pub legacy_templates: Arc<DashMap<String, Value>>,/// pipeline_name → compiled, executable Pipeline (typed transform pipeline)pub transform_pipelines: Arc<DashMap<String, xerj_wasm::pipeline::Pipeline>>,@@ -549,6 +613,20 @@ pub struct Engine {/// the process: a `kill -9` releases it automatically and a stale/// `node.lock` file never blocks the next boot._node_lock: Arc<std::fs::File>,++ /// Serializes rewrites of `<data_dir>/cluster_state.json` (issue #203).+ ///+ /// The atomic rewrite stages through a single fixed path+ /// (`cluster_state.tmp`), so two concurrent flushes — two provisioning+ /// PUTs arriving together is all it takes — would create/truncate the+ /// *same* temp file and rename each other's half-written bytes over the+ /// real one. Holding this across snapshot-and-write makes the whole+ /// rewrite one critical section; the last writer's snapshot necessarily+ /// includes every mutation that had already been applied to the maps, so+ /// serializing loses nothing.+ ///+ /// Never taken while holding a `DashMap` guard.+ cluster_state_write: Arc<parking_lot::Mutex<()>>,}impl Engine {@@ -660,6 +738,7 @@ impl Engine {xerj_cluster::router::ShardRouter::new(1),)),_node_lock: node_lock,+ cluster_state_write: Arc::new(parking_lot::Mutex::new(())),};// Scan data_dir for existing index directories.@@ -730,9 +809,24 @@ impl Engine {// migration by another instance).engine.load_persisted_aliases();+ // Restore the cluster-level management state — index templates,+ // legacy templates, component templates, ingest pipelines, data+ // streams, ILM policies (issue #203). Before this, "replace the+ // binary and restart" silently reverted every one of them: the+ // documents kept flowing, into indices that no longer had the shape+ // the operator designed. Must run after the index scan above, so a+ // restored data stream can report which of its backing indices are+ // actually present, and before the server accepts requests.+ engine.load_cluster_state();+// Restore ISM/ILM policies and managed-index execution state so a// policy attached before a restart keeps running afterward instead- // of silently going idle.+ // of silently going idle. Separate files (`ism_policies.json`,+ // `ism_managed_indices.json`) from `cluster_state.json` above, and+ // deliberately so: that file holds the verbatim documents an+ // operator PUT, this holds the executor's own cursor. Loaded after+ // `load_cluster_state` because a restored data stream is what a+ // rollover action operates on.engine.load_persisted_ism_policies();engine.load_persisted_managed_indices();@@ -1626,6 +1720,448 @@ impl Engine {self.record_index_created_at(name);}+ // ── Cluster metadata persistence (issue #203) ────────────────────────────+ //+ // Index templates, legacy (v1) templates, component templates, ingest+ // pipelines, data streams and ILM policies used to live only in the+ // in-memory maps above: `PUT /_index_template/logs` answered `200+ // {"acknowledged": true}` and the template was gone after the next boot,+ // with no error anywhere and the next matching index silently created+ // unshaped. They are now snapshotted, as one consistent document, into+ // `<data_dir>/cluster_state.json` on every mutation and reloaded in+ // `Engine::new`.+ //+ // Durability contract: a full-file rewrite through+ // `index::write_file_atomic` — write to `cluster_state.tmp`, `fsync` it,+ // `rename` over the target, then `fsync` the parent directory so the+ // rename itself survives power loss. The rename is the commit point, so+ // a `kill -9` at any instant leaves either the previous complete+ // document or the new one, never a half-written mix. The same shape peer+ // engines use for their manifests (cf. `fjall/src/file.rs:17`+ // `fsync_directory`, which fjall calls after every atomic rewrite;+ // `sled/src/metadata_store.rs:696` discards leftover `*.tmp` files from+ // an interrupted rewrite on the next boot, which `load_cluster_state`+ // mirrors below).+ //+ // One file rather than six keeps the maps mutually consistent — a data+ // stream and the template that shaped it can never be restored from+ // different generations.++ /// Path of the persisted cluster-metadata document+ /// (`<data_dir>/cluster_state.json`).+ fn cluster_state_path(&self) -> PathBuf {+ self.data_dir.join("cluster_state.json")+ }++ /// Snapshot every persisted management map into one document.+ fn cluster_state_snapshot(&self) -> PersistedClusterState {+ fn dump<V: Clone>(map: &DashMap<String, V>) -> std::collections::BTreeMap<String, V> {+ map.iter()+ .map(|e| (e.key().clone(), e.value().clone()))+ .collect()+ }+ PersistedClusterState {+ version: CLUSTER_STATE_VERSION,+ index_templates: dump(&self.templates),+ legacy_templates: dump(&self.legacy_templates),+ component_templates: dump(&self.component_templates),+ pipelines: dump(&self.pipelines),+ data_streams: dump(&self.data_streams),+ ilm_policies: dump(&self.ilm_policies),+ }+ }++ /// Durably write the current cluster metadata.+ ///+ /// Unlike `flush_aliases`, a failure here is **returned**, not swallowed:+ /// every caller is an API write that would otherwise answer+ /// `{"acknowledged": true}` for a change it cannot keep. Callers roll the+ /// in-memory change back and surface a 500, so the operator finds out at+ /// the moment of the write instead of at the next restart.+ fn flush_cluster_state(&self) -> Result<()> {+ // One writer at a time — see `cluster_state_write`. Snapshotting+ // inside the lock also means the bytes that land are never older+ // than a rewrite that has already returned.+ let _writing = self.cluster_state_write.lock();+ let snapshot = self.cluster_state_snapshot();+ let bytes = serde_json::to_vec_pretty(&snapshot)?;+ crate::index::write_file_atomic(&self.cluster_state_path(), &bytes)?;+ Ok(())+ }++ /// Highest generation `N` for which a backing index `.ds-<stream>-<N>` is+ /// actually open in this data dir; `0` when there is none.+ ///+ /// Read off the open indices rather than the stream document, because the+ /// point is to catch the case where the two disagree. Suffix parsing keeps+ /// neighbouring streams apart on its own: for stream `a`, `.ds-a-b-000001`+ /// leaves `b-000001`, which is not a number.+ fn highest_backing_generation(&self, stream: &str) -> u64 {+ let prefix = format!(".ds-{stream}-");+ self.indices+ .iter()+ .filter_map(|e| {+ e.key()+ .strip_prefix(&prefix)+ .and_then(|gen| gen.parse::<u64>().ok())+ })+ .max()+ .unwrap_or(0)+ }++ /// Load `<data_dir>/cluster_state.json` back into the management maps.+ ///+ /// A missing file is normal (fresh node, or a data dir written by a build+ /// that predates this). A corrupt file is loud: it means the operator's+ /// templates, pipelines and data streams are *not* coming back, which is+ /// exactly the silence this whole mechanism exists to remove.+ fn load_cluster_state(&self) {+ // An interrupted rewrite can leave a partial `cluster_state.tmp`+ // behind. It is never read (the rename is the commit point), but+ // leaving it litters the data dir and invites a hand-edit that+ // "restores" a torn document — sled sweeps the same leftovers on+ // boot (`sled/src/metadata_store.rs:696`).+ let tmp = self.cluster_state_path().with_extension("tmp");+ if tmp.exists() {+ warn!(+ path = %tmp.display(),+ "discarding an incomplete cluster_state rewrite left by an unclean shutdown"+ );+ let _ = std::fs::remove_file(&tmp);+ }++ let path = self.cluster_state_path();+ let bytes = match std::fs::read(&path) {+ Ok(b) => b,+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return,+ Err(e) => {+ error!(+ path = %path.display(), error = %e,+ "could not read cluster_state.json — index templates, ingest \+ pipelines, data streams and ILM policies are NOT restored"+ );+ return;+ }+ };+ let state: PersistedClusterState = match serde_json::from_slice(&bytes) {+ Ok(s) => s,+ Err(e) => {+ error!(+ path = %path.display(), error = %e,+ "cluster_state.json is corrupt — index templates, ingest \+ pipelines, data streams and ILM policies are NOT restored"+ );+ // Keep a copy before the node runs on. The maps are empty+ // now, so the very next `PUT /_index_template/...` rewrites+ // this file and takes whatever was still legible in it with+ // it — and hand-recovering a template out of a damaged+ // document is the operator's last option, not one to destroy+ // silently. The original is deliberately left in place so+ // every boot keeps logging the error above until someone+ // deals with it, and an existing copy is never clobbered:+ // the first corruption is the informative one.+ let salvage = path.with_extension("corrupt.json");+ if !salvage.exists() {+ match std::fs::write(&salvage, &bytes) {+ Ok(()) => warn!(+ path = %salvage.display(),+ "kept a copy of the corrupt cluster_state for recovery"+ ),+ Err(e) => warn!(+ path = %salvage.display(), error = %e,+ "could not preserve a copy of the corrupt cluster_state"+ ),+ }+ }+ return;+ }+ };+ if state.version > CLUSTER_STATE_VERSION {+ warn!(+ found = state.version,+ supported = CLUSTER_STATE_VERSION,+ "cluster_state.json was written by a newer xerj; loading what \+ this build understands (downgrade may drop metadata)"+ );+ }++ for (name, tmpl) in state.index_templates {+ self.templates.insert(name, tmpl);+ }+ for (name, body) in state.legacy_templates {+ self.legacy_templates.insert(name, body);+ }+ for (name, body) in state.component_templates {+ self.component_templates.insert(name, body);+ }+ for (name, body) in state.ilm_policies {+ self.ilm_policies.insert(name, body);+ }++ // Data streams: restore the record, reconcile it against what is+ // actually on disk, then say plainly when a backing index it names is+ // not there. Reporting a stream whose data is gone as+ // `"status": "GREEN"` without a word in the log is the same quiet lie+ // this issue is about.+ let mut reconciled_a_stream = false;+ for (name, mut ds) in state.data_streams {+ // `rollover_data_stream` creates the new backing index before it+ // records the new generation, so `kill -9` in that window leaves+ // `.ds-<name>-00000N` on disk while the document still says N-1.+ // Restoring N-1 verbatim would wedge the stream permanently: the+ // next rollover computes the same name, and `create_index`+ // refuses it as `index_already_exists` — forever. Adopt the+ // highest generation that actually exists instead, which is also+ // what stops a second rollover writing into an index that already+ // holds the first one's documents.+ let on_disk = self.highest_backing_generation(&name);+ if on_disk > ds.generation {+ let adopted = format!(".ds-{name}-{on_disk:06}");+ warn!(+ data_stream = name.as_str(),+ recorded = ds.generation,+ found = on_disk,+ adopted = adopted.as_str(),+ "adopting a backing index left behind by a rollover that \+ was interrupted before its generation reached disk"+ );+ if !ds.backing_indices.contains(&adopted) {+ ds.backing_indices.push(adopted);+ }+ ds.generation = on_disk;+ reconciled_a_stream = true;+ }++ let missing: Vec<&str> = ds+ .backing_indices+ .iter()+ .filter(|b| !self.indices.contains_key(b.as_str()))+ .map(|b| b.as_str())+ .collect();+ if !missing.is_empty() {+ warn!(+ data_stream = name.as_str(),+ missing = missing.join(","),+ "restored data stream references backing indices that are \+ not present in the data dir"+ );+ }+ self.data_streams.insert(name, ds);+ }++ // Pipelines carry behaviour, not just a document to hand back from+ // `GET /_ingest/pipeline`: the executable form has to be rebuilt or+ // `?pipeline=x` would be accepted after a restart and quietly do+ // nothing. Keep the stored document either way so GET round-trips+ // exactly as it did before the restart.+ let mut pipelines_restored = 0usize;+ for (name, config) in state.pipelines {+ self.pipelines.insert(name.clone(), config.clone());+ match self.compile_pipeline(&name, config) {+ Ok(()) => pipelines_restored += 1,+ Err(e) => error!(+ pipeline = name.as_str(), error = %e,+ "persisted ingest pipeline could not be recompiled — it is \+ still visible to GET /_ingest/pipeline but will NOT \+ transform anything; re-PUT the definition"+ ),+ }+ }++ // Make an adopted generation durable straight away, so the recovery+ // happens once rather than on every boot from here on.+ if reconciled_a_stream {+ if let Err(e) = self.flush_cluster_state() {+ warn!(+ error = %e,+ "could not persist the reconciled data-stream generation; \+ it will be recomputed from disk on the next boot"+ );+ }+ }++ let n = self.templates.len()+ + self.legacy_templates.len()+ + self.component_templates.len()+ + self.pipelines.len()+ + self.data_streams.len()+ + self.ilm_policies.len();+ if n > 0 {+ info!(+ index_templates = self.templates.len(),+ legacy_templates = self.legacy_templates.len(),+ component_templates = self.component_templates.len(),+ pipelines = pipelines_restored,+ data_streams = self.data_streams.len(),+ ilm_policies = self.ilm_policies.len(),+ "restored persisted cluster metadata"+ );+ }+ }++ /// Insert into a persisted map and durably record the result, rolling the+ /// in-memory change back if the write fails so a 500 never leaves a+ /// change that only this process can see.+ fn persisted_insert<V: Clone>(+ &self,+ map: &DashMap<String, V>,+ name: String,+ value: V,+ ) -> Result<()> {+ let previous = map.insert(name.clone(), value);+ if let Err(e) = self.flush_cluster_state() {+ match previous {+ Some(old) => map.insert(name, old),+ None => map.remove(&name).map(|(_, v)| v),+ };+ return Err(e);+ }+ Ok(())+ }++ /// Remove from a persisted map and durably record the result. Returns+ /// `false` when there was nothing to remove (the caller answers 404);+ /// an `Err` means the removal did not reach disk and was undone.+ fn persisted_remove<V: Clone>(&self, map: &DashMap<String, V>, name: &str) -> Result<bool> {+ let Some((key, removed)) = map.remove(name) else {+ return Ok(false);+ };+ if let Err(e) = self.flush_cluster_state() {+ map.insert(key, removed);+ return Err(e);+ }+ Ok(true)+ }++ /// Create or replace a v2 index template (`PUT /_index_template/{name}`).+ pub fn put_index_template(&self, name: String, template: IndexTemplate) -> Result<()> {+ self.persisted_insert(&self.templates, name, template)+ }++ /// Delete a v2 index template; `false` when it did not exist.+ pub fn delete_index_template(&self, name: &str) -> Result<bool> {+ self.persisted_remove(&self.templates, name)+ }++ /// Create or replace a legacy v1 template (`PUT /_template/{name}`).+ pub fn put_legacy_template(&self, name: String, body: Value) -> Result<()> {+ self.persisted_insert(&self.legacy_templates, name, body)+ }++ /// Delete a legacy v1 template; `false` when it did not exist.+ pub fn delete_legacy_template(&self, name: &str) -> Result<bool> {+ self.persisted_remove(&self.legacy_templates, name)+ }++ /// Create or replace a component template.+ pub fn put_component_template(&self, name: String, body: Value) -> Result<()> {+ self.persisted_insert(&self.component_templates, name, body)+ }++ /// Delete a component template; `false` when it did not exist.+ pub fn delete_component_template(&self, name: &str) -> Result<bool> {+ self.persisted_remove(&self.component_templates, name)+ }++ /// Create or replace an ILM policy.+ pub fn put_ilm_policy(&self, name: String, body: Value) -> Result<()> {+ self.persisted_insert(&self.ilm_policies, name, body)+ }++ /// Delete an ILM policy; `false` when it did not exist.+ pub fn delete_ilm_policy(&self, name: &str) -> Result<bool> {+ self.persisted_remove(&self.ilm_policies, name)+ }++ /// Durably store an ingest pipeline definition and (re)build its+ /// executable form. This is the only way a user-visible pipeline should+ /// ever be created — `compile_pipeline` alone does not reach disk.+ ///+ /// `config` is the xerj-shaped pipeline config to compile, and is what+ /// gets stored (and handed back by `GET /_ingest/pipeline/{id}`) when it+ /// compiles — the shape this endpoint has always returned.+ ///+ /// `keep_if_uncompilable` says what to do when it does *not* compile, and+ /// differs by surface:+ ///+ /// * `Some(doc)` — the ES-compat surface. It has always answered+ /// `acknowledged` and left the definition readable, so `doc` (the+ /// ES-shaped body as received) is stored and the compile error is+ /// returned as `Ok(Some(err))` for the caller to log.+ /// * `None` — the xerj-native surface, which rejects a definition it+ /// cannot compile. Nothing is stored, nothing is flushed, and any+ /// pipeline already registered under this id is left untouched.+ ///+ /// Either way a compile failure drops the *executable* form under this+ /// id. Leaving the old one behind would mean `?pipeline={id}` kept+ /// running a definition `GET` no longer shows — and stopped running it at+ /// the next restart. A definition must behave the same before and after a+ /// reboot; that is the whole point of persisting it.+ ///+ /// `Err` means the definition did not reach disk. The in-memory change is+ /// rolled back first, so the caller can report the write as failed+ /// without leaving behind a pipeline only this process can see.+ pub fn put_pipeline(+ &self,+ id: &str,+ config: Value,+ keep_if_uncompilable: Option<Value>,+ ) -> Result<Option<xerj_wasm::WasmError>> {+ let previous_doc = self.pipelines.get(id).map(|v| v.value().clone());+ // Taken out up front so a failed compile cannot leave a stale+ // executable behind; put back verbatim on every abort path.+ let previous_compiled = self.transform_pipelines.remove(id).map(|(_, p)| p);++ // `compile_pipeline` mutates nothing when it fails, so an abort only+ // has to undo the two lines above.+ let compile_err = match self.compile_pipeline(id, config) {+ Ok(()) => None,+ Err(e) => match keep_if_uncompilable {+ None => {+ if let Some(p) = previous_compiled {+ self.transform_pipelines.insert(id.to_string(), p);+ }+ return Ok(Some(e));+ }+ Some(doc) => {+ self.pipelines.insert(id.to_string(), doc);+ Some(e)+ }+ },+ };++ if let Err(e) = self.flush_cluster_state() {+ match previous_doc {+ Some(d) => self.pipelines.insert(id.to_string(), d),+ None => self.pipelines.remove(id).map(|(_, v)| v),+ };+ self.transform_pipelines.remove(id);+ if let Some(p) = previous_compiled {+ self.transform_pipelines.insert(id.to_string(), p);+ }+ return Err(e);+ }++ if let Some(e) = &compile_err {+ warn!(pipeline = id, error = %e, "pipeline stored but failed to compile");+ } else {+ info!(name = id, "transform pipeline created");+ }+ Ok(compile_err)+ }++ /// Delete an ingest pipeline — both the stored definition and the+ /// compiled form, so `?pipeline={name}` stops working at the same moment+ /// `GET /_ingest/pipeline/{name}` starts answering 404. `false` when it+ /// did not exist.+ pub fn delete_pipeline(&self, name: &str) -> Result<bool> {+ let removed = self.persisted_remove(&self.pipelines, name)?;+ if removed {+ self.transform_pipelines.remove(name);+ }+ Ok(removed)+ }+/// Add an alias pointing to an index.pub fn add_alias(&self, alias: &str, index: &str) {let mut entry = self.aliases.entry(alias.to_string()).or_default();@@ -2147,7 +2683,11 @@ impl Engine {timestamp_field: "@timestamp".to_string(),generation: 1,};- self.data_streams.insert(name.to_string(), ds);+ // Durable before the caller is told it worked (issue #203): the+ // backing index is already on disk, so a stream record that lives+ // only in memory would come back after a restart as an orphaned+ // `.ds-*` index with no stream in front of it.+ self.persisted_insert(&self.data_streams, name.to_string(), ds)?;info!(name, "data stream created");Ok(())}@@ -2179,6 +2719,10 @@ impl Engine {if let Some(mut ds) = self.data_streams.get_mut(name) {ds.backing_indices.push(new_backing.clone());}+ // The generation counter is the thing that must not be lost: an+ // unpersisted rollover would hand `.ds-<name>-000002` out twice after+ // a restart and write into the previous generation's index.+ self.flush_cluster_state()?;info!(name,@@ -2196,6 +2740,17 @@ impl Engine {.map(|(_, v)| v).ok_or_else(|| EngineError::Common(xerj_common::XerjError::index_not_found(name)))?;+ // Record the removal before any backing index is destroyed: if the+ // process dies mid-delete, a boot that still believes in the stream+ // is recoverable (it warns about the missing backing indices), while+ // one that has forgotten it leaves orphaned `.ds-*` directories no+ // API can reach. A write failure here aborts the delete outright —+ // nothing has been destroyed yet, so the stream is put back intact.+ if let Err(e) = self.flush_cluster_state() {+ self.data_streams.insert(name.to_string(), ds);+ return Err(e);+ }+// Remove the alias.self.aliases.remove(name);self.flush_aliases();@@ -2309,15 +2864,19 @@ impl Engine {// ── Transform pipeline methods ────────────────────────────────────────────- /// Compile and register a typed transform pipeline from a JSON config.+ /// Compile and register a typed transform pipeline from a JSON config,+ /// **in memory only**.////// `config_json` must be a valid [`PipelineConfig`](xerj_wasm::pipeline::PipelineConfig)- /// object. The compiled pipeline is stored in `transform_pipelines` and- /// can be retrieved by name for use at ingest time.+ /// object. The compiled pipeline is stored in `transform_pipelines` and+ /// the raw JSON in `pipelines`, so the ES-compatible ingest API can hand+ /// it back.///- /// The raw JSON is also stored in `pipelines` so it can be returned by the- /// ES-compatible ingest pipeline API.- pub fn create_pipeline(+ /// This is the primitive underneath [`Engine::put_pipeline`] and the boot+ /// path; it deliberately does not touch disk, so it is private. Anything+ /// that serves a user request must go through `put_pipeline`, or the+ /// pipeline is gone at the next restart (issue #203).+ fn compile_pipeline(&self,name: &str,config_json: Value,@@ -2327,7 +2886,6 @@ impl Engine {let pipeline = xerj_wasm::pipeline::Pipeline::from_config(name, &cfg)?;self.pipelines.insert(name.to_string(), config_json);self.transform_pipelines.insert(name.to_string(), pipeline);- info!(name, "transform pipeline created");Ok(())}diff --git a/engine/crates/xerj-engine/tests/es_compat_tests.rs b/engine/crates/xerj-engine/tests/es_compat_tests.rsindex f273e3cb1..2c8f617a7 100644--- a/engine/crates/xerj-engine/tests/es_compat_tests.rs+++ b/engine/crates/xerj-engine/tests/es_compat_tests.rs@@ -865,8 +865,8 @@ async fn test_es_bulk_per_item_pipeline() {engine.create_index("bulk_pipe", Schema::empty()).unwrap();// Register a pipeline that stamps `ingested = "yes"` on every doc.- engine- .create_pipeline(+ let compile_err = engine+ .put_pipeline("set_pipe",json!({"description": "stamp ingested marker",@@ -874,8 +874,13 @@ async fn test_es_bulk_per_item_pipeline() {{ "type": "set", "config": { "field": "ingested", "value": "yes", "override": true } }]}),+ None,)- .expect("create_pipeline");+ .expect("put_pipeline");+ assert!(+ compile_err.is_none(),+ "pipeline must compile: {compile_err:?}"+ );let ndjson = concat!(// Item 0: no pipeline → stored verbatim.diff --git a/engine/crates/xerj-server/tests/cluster_metadata_survives_sigkill.rs b/engine/crates/xerj-server/tests/cluster_metadata_survives_sigkill.rsnew file mode 100644index 000000000..63546b2df--- /dev/null+++ b/engine/crates/xerj-server/tests/cluster_metadata_survives_sigkill.rs@@ -0,0 +1,243 @@+//! Issue #203, against the real binary and a real `SIGKILL`.+//!+//! The in-process test (`xerj-api/tests/cluster_metadata_survives_restart.rs`)+//! proves the reload path. This one proves the *durability* half: the state+//! has to be on disk at the moment the PUT is acknowledged, not at some later+//! shutdown hook, because the restart that matters is the one nobody planned.+//! `kill -9` gives the process no chance to flush anything, so anything that+//! comes back was already committed by `write_file_atomic` (write → fsync →+//! rename → fsync parent).+//!+//! The write also has to be *atomic*, not merely eventual: this test hard-kills+//! the node while it is answering, so a rewrite that truncated the file in+//! place could leave a half-document that takes every template with it.+//!+//! Linux/macOS only — `SIGKILL` has no Windows equivalent, and+//! `Child::kill()` there is a `TerminateProcess` that Windows may let the+//! runtime observe. The property under test is platform-independent, but the+//! way to provoke it is not.++#![cfg(unix)]++use std::io::{BufRead, BufReader, Read, Write};+use std::net::{TcpListener, TcpStream};+use std::process::{Child, Command, Stdio};+use std::time::{Duration, Instant};++/// Three distinct free ports — `Config::validate` requires all three listener+/// ports to differ.+fn three_free_ports() -> (u16, u16, u16) {+ let l1 = TcpListener::bind("127.0.0.1:0").unwrap();+ let l2 = TcpListener::bind("127.0.0.1:0").unwrap();+ let l3 = TcpListener::bind("127.0.0.1:0").unwrap();+ (+ l1.local_addr().unwrap().port(),+ l2.local_addr().unwrap().port(),+ l3.local_addr().unwrap().port(),+ )+}++fn toml_path(p: &std::path::Path) -> String {+ p.to_string_lossy().replace('\\', "/")+}++/// One HTTP/1.1 request over a fresh connection, returning `(status, body)`.+///+/// Hand-rolled rather than pulled from a client crate so the test has no+/// runtime of its own: it must stay a plain `#[test]`, since the thing it+/// kills is a separate process and any async machinery here would only be+/// another thing that can hang.+fn request(+ port: u16,+ method: &str,+ path: &str,+ body: Option<&str>,+) -> std::io::Result<(u16, String)> {+ let mut stream = TcpStream::connect(("127.0.0.1", port))?;+ stream.set_read_timeout(Some(Duration::from_secs(30)))?;+ stream.set_write_timeout(Some(Duration::from_secs(30)))?;++ let payload = body.unwrap_or("");+ let request = format!(+ "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\+ Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{payload}",+ payload.len()+ );+ stream.write_all(request.as_bytes())?;+ stream.flush()?;++ let mut reader = BufReader::new(stream);+ let mut status_line = String::new();+ reader.read_line(&mut status_line)?;+ let status: u16 = status_line+ .split_whitespace()+ .nth(1)+ .and_then(|s| s.parse().ok())+ .unwrap_or(0);++ // Drain headers, then everything else — `Connection: close` means the+ // body ends at EOF, so no chunk/length parsing is needed.+ loop {+ let mut line = String::new();+ if reader.read_line(&mut line)? == 0 || line == "\r\n" {+ break;+ }+ }+ let mut body = String::new();+ reader.read_to_string(&mut body)?;+ Ok((status, body))+}++/// Spawn the real binary on `data_dir` and wait until it answers.+fn boot(+ data_dir: &std::path::Path,+ cfg_dir: &std::path::Path,+ es_port: u16,+ rest: u16,+ grpc: u16,+) -> Child {+ let config = format!(+ r#"+[server]+bind_address = "127.0.0.1"+rest_port = {rest}+grpc_port = {grpc}+es_compat_port = {es_port}+data_dir = "{data}"+"#,+ data = toml_path(data_dir),+ );+ let config_path = cfg_dir.join(format!("xerj-{es_port}.toml"));+ std::fs::write(&config_path, config).unwrap();++ let mut child = Command::new(env!("CARGO_BIN_EXE_xerj"))+ .arg("--config")+ .arg(&config_path)+ .arg("--insecure")+ .stdout(Stdio::null())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn xerj");++ let deadline = Instant::now() + Duration::from_secs(120);+ loop {+ if let Ok((200, _)) = request(es_port, "GET", "/_cluster/health", None) {+ return child;+ }+ if Instant::now() >= deadline {+ break;+ }+ std::thread::sleep(Duration::from_millis(200));+ }+ // Never leave a half-booted node behind for the rest of the suite to+ // trip over — reap it, then fail.+ let _ = child.kill();+ let _ = child.wait();+ panic!("xerj did not become ready on port {es_port}");+}++/// `kill -9` and reap. No graceful shutdown, no flush hook, no chance to+/// write anything the acknowledged PUTs did not already write.+fn sigkill(mut child: Child) {+ unsafe {+ libc::kill(child.id() as libc::pid_t, libc::SIGKILL);+ }+ let status = child.wait().expect("reap child");+ assert!(+ !status.success(),+ "the node was supposed to be hard-killed, but it exited cleanly ({status:?}) — \+ a graceful exit could have flushed state that a real crash never would"+ );+}++#[test]+fn cluster_metadata_survives_kill_dash_nine() {+ let dir = tempfile::tempdir().unwrap();+ let data_dir = dir.path().join("data");+ std::fs::create_dir_all(&data_dir).unwrap();++ let (rest, grpc, es) = three_free_ports();+ let child = boot(&data_dir, dir.path(), es, rest, grpc);++ let (st, b) = request(+ es,+ "PUT",+ "/_index_template/logs",+ Some(r#"{"index_patterns":["logs-*"],"priority":200,"template":{"mappings":{"properties":{"host":{"type":"keyword"}}}}}"#),+ )+ .unwrap();+ assert_eq!(st, 200, "PUT _index_template: {b}");++ let (st, b) = request(+ es,+ "PUT",+ "/_ingest/pipeline/tagger",+ Some(r#"{"description":"tag","processors":[{"set":{"field":"tagged","value":"yes"}}]}"#),+ )+ .unwrap();+ assert_eq!(st, 200, "PUT _ingest/pipeline: {b}");++ let (st, b) = request(+ es,+ "PUT",+ "/_ilm/policy/hot-warm",+ Some(r#"{"policy":{"phases":{"hot":{"actions":{}}}}}"#),+ )+ .unwrap();+ assert_eq!(st, 200, "PUT _ilm/policy: {b}");++ let (st, b) = request(es, "PUT", "/_data_stream/metrics-app", Some("{}")).unwrap();+ assert_eq!(st, 200, "PUT _data_stream: {b}");++ // Roll the stream forward so the restored generation counter has to be+ // 2, not the 1 a re-create would produce.+ let (st, b) = request(es, "POST", "/metrics-app/_rollover", Some("{}")).unwrap();+ assert_eq!(st, 200, "POST _rollover: {b}");+ assert!(+ b.contains(".ds-metrics-app-000002"),+ "rollover should mint generation 2: {b}"+ );++ sigkill(child);++ // Same data dir, brand-new process. Everything above was acknowledged,+ // so everything above must still be here.+ let child = boot(&data_dir, dir.path(), es, rest, grpc);++ let (st, b) = request(es, "GET", "/_index_template/logs", None).unwrap();+ assert_eq!(st, 200, "index template lost across kill -9: {b}");+ assert!(+ b.contains("logs-*") && b.contains("keyword"),+ "partial template restore: {b}"+ );++ let (st, b) = request(es, "GET", "/_ingest/pipeline/tagger", None).unwrap();+ assert_eq!(st, 200, "ingest pipeline lost across kill -9: {b}");++ let (st, b) = request(es, "GET", "/_ilm/policy/hot-warm", None).unwrap();+ assert_eq!(st, 200, "ILM policy lost across kill -9: {b}");++ let (st, b) = request(es, "GET", "/_data_stream/metrics-app", None).unwrap();+ assert_eq!(st, 200, "data stream lost across kill -9: {b}");+ assert!(+ b.contains("\"generation\":2") && b.contains(".ds-metrics-app-000002"),+ "the rollover must survive too, or the next rollover reissues a \+ backing index that already holds data: {b}"+ );++ // The restored pipeline must still *run*, not just be readable.+ let (st, b) = request(+ es,+ "POST",+ "/_ingest/pipeline/tagger/_simulate",+ Some(r#"{"docs":[{"_source":{"host":"a"}}]}"#),+ )+ .unwrap();+ assert_eq!(st, 200, "simulate after restart: {b}");+ assert!(+ b.contains("\"tagged\":\"yes\""),+ "a restored pipeline that no longer transforms is accepted-and-ignored: {b}"+ );++ sigkill(child);+}