AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(engine): real ISM/ILM index lifecycle management — one engine, two REST surfaces
Vincenzo Lombardod34803ea5a97
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 1ac656576..29c8a1f74 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -832,6 +832,17 @@ pub async fn create_index(.insert(index.clone(), Value::Object(resolved_aliases));}+ // ES-shape lifecycle attach: `index.lifecycle.name` in create-time+ // settings is how ILM associates an index with a policy (there is+ // no dedicated "attach" endpoint on the ES surface, unlike ISM's+ // explicit `POST _plugins/_ism/add/{index}`). Best-effort and+ // silent on a missing/untranslatable policy — real ILM is+ // equally lenient here: it discovers the policy by name lazily+ // and simply doesn't start managing until one exists.+ if let Some(policy_name) = ilm_policy_name_from_settings(&body) {+ let _ = crate::ism_api::attach_policy(&state, &index, &policy_name).await;+ }+let resp = EsIndexResponse::ok(&index);(StatusCode::OK, Json(resp)).into_response()}@@ -20910,9 +20921,33 @@ pub async fn put_settings(state.engine.index_settings.insert(idx.clone(), existing);}+ // Same lazy, best-effort `index.lifecycle.name` attach as create_index —+ // `PUT {index}/_settings {"index.lifecycle.name": "policy"}` is the+ // other real ILM entry point (attaching to an index that already+ // exists, not just at creation).+ if let Some(policy_name) = ilm_policy_name_from_settings(&body) {+ for idx in &targets {+ let _ = crate::ism_api::attach_policy(&state, idx, &policy_name).await;+ }+ }+Json(json!({ "acknowledged": true })).into_response()}+/// Extract `index.lifecycle.name` from an index-settings body, accepting+/// both the nested (`{"index":{"lifecycle":{"name":"..."}}}`,+/// `{"settings":{...}}`-wrapped at create time) and flat dotted-key+/// (`{"index.lifecycle.name":"..."}`) forms ES accepts for every index+/// setting.+fn ilm_policy_name_from_settings(body: &Value) -> Option<String> {+ let settings = body.get("settings").unwrap_or(body);+ settings+ .pointer("/index/lifecycle/name")+ .or_else(|| settings.get("index.lifecycle.name"))+ .and_then(Value::as_str)+ .map(String::from)+}+// ─────────────────────────────────────────────────────────────────────────────// Ingest Pipelines// PUT /_ingest/pipeline/{id}@@ -22722,10 +22757,34 @@ pub async fn put_ilm_policy(Path(name): Path<String>,Json(body): Json<Value>,) -> impl IntoResponse {- // Persist the policy in the real ILM store; `get_ilm_policy` reads it back- // out of this same DashMap, so PUT then GET round-trips faithfully.- state.engine.ilm_policies.insert(name, body);- Json(json!({ "acknowledged": true })).into_response()+ // Persist the raw policy verbatim in the ILM store; `get_ilm_policy`+ // reads it back out of this same DashMap, so PUT then GET round-trips+ // faithfully regardless of whether translation below succeeds.+ state.engine.ilm_policies.insert(name.clone(), body.clone());++ // Translate into the shared internal model (`xerj_engine::lifecycle`)+ // so the SAME background job that drives native ISM policies also+ // drives this one — see that module's doc comment for why. Real ILM+ // is lenient about phase actions it can't reproduce internally too+ // (e.g. `searchable_snapshot` is best-effort); mirroring that, a+ // translation failure here doesn't fail the PUT — the policy is+ // still stored and GET-able, it just won't actually execute (no entry+ // lands in `ism_policies`) until it's fixed. Surfaced back in the+ // response so this isn't silent.+ match xerj_engine::lifecycle::translate_ilm_policy(&body) {+ Ok(policy) => {+ state.engine.put_ism_policy(name, policy);+ Json(json!({ "acknowledged": true })).into_response()+ }+ Err(reason) => Json(json!({+ "acknowledged": true,+ "xerj_lifecycle_execution": {+ "status": "unsupported",+ "reason": reason,+ }+ }))+ .into_response(),+ }}pub async fn get_ilm_policy(@@ -22756,6 +22815,7 @@ pub async fn delete_ilm_policy(Path(name): Path<String>,) -> impl IntoResponse {if state.engine.ilm_policies.remove(&name).is_some() {+ state.engine.remove_ism_policy(&name);Json(json!({ "acknowledged": true })).into_response()} else {let e = xerj_common::XerjError::index_not_found(format!("policy [{name}] not found"));@@ -22763,6 +22823,39 @@ pub async fn delete_ilm_policy(}}+// ─────────────────────────────────────────────────────────────────────────────+// GET /{index}/_ilm/explain+// ─────────────────────────────────────────────────────────────────────────────++pub async fn ilm_explain(+ State(state): State<AppState>,+ Path(index): Path<String>,+) -> impl IntoResponse {+ let mut indices = serde_json::Map::new();+ match state.engine.managed_indices.get(&index) {+ Some(managed) => {+ indices.insert(+ index.clone(),+ json!({+ "index": index,+ "managed": true,+ "policy": managed.policy_id,+ "phase": managed.current_state,+ "phase_time_millis": managed.state_entered_at_ms,+ "action": managed.info_message,+ "step": managed.info_message,+ "step_info": { "message": managed.info_message },+ "failed_step": if managed.failed { Some(managed.info_message.clone()) } else { None },+ }),+ );+ }+ None => {+ indices.insert(index.clone(), json!({ "index": index, "managed": false }));+ }+ }+ Json(json!({ "indices": Value::Object(indices) })).into_response()+}+// ─────────────────────────────────────────────────────────────────────────────// Component Templates// PUT /_component_template/{name}diff --git a/engine/crates/xerj-api/src/ism_api.rs b/engine/crates/xerj-api/src/ism_api.rsnew file mode 100644index 000000000..f7062e2fa--- /dev/null+++ b/engine/crates/xerj-api/src/ism_api.rs@@ -0,0 +1,272 @@+//! OpenSearch Index State Management (ISM) — the native REST surface over+//! `xerj_engine::lifecycle`'s execution engine. See that module's doc+//! comment for why ISM's state machine (not ES ILM's fixed phases) is the+//! internal model both `_plugins/_ism/*` (here) and `_ilm/*`+//! (`es_compat::put_ilm_policy` and friends) are built on.+//!+//! Endpoints:+//! PUT/GET/DELETE /_plugins/_ism/policies/{policy_id}+//! GET /_plugins/_ism/policies (list/search — added+//! after live OSD UI verification showed the "State+//! management policies" table 500ing without it)+//! POST /_plugins/_ism/add/{index}+//! GET /_plugins/_ism/explain/{index}+//! GET /_plugins/_ism/explain (list-all — same+//! reason, for the "Policy managed indexes" table)++use axum::{+ extract::{Path, Query, State},+ response::{IntoResponse, Response},+ Json,+};+use serde::Deserialize;+use serde_json::{json, Value};++use xerj_engine::lifecycle::{self, LifecyclePolicy, ManagedIndexState};++use crate::{error::ApiError, state::AppState};++// ─────────────────────────────────────────────────────────────────────────────+// PUT/GET/DELETE /_plugins/_ism/policies/{policy_id}+// ─────────────────────────────────────────────────────────────────────────────++/// Real ISM wraps the policy body as `{"policy": {...}}`; accept that shape+/// but also the bare policy object, since it's an easy, harmless mistake to+/// make by hand (and this parses unambiguously either way).+fn unwrap_policy_body(body: &Value) -> &Value {+ body.get("policy").unwrap_or(body)+}++pub async fn put_ism_policy(+ State(state): State<AppState>,+ Path(policy_id): Path<String>,+ Json(body): Json<Value>,+) -> impl IntoResponse {+ let policy_value = unwrap_policy_body(&body).clone();+ let policy: LifecyclePolicy = match serde_json::from_value(policy_value) {+ Ok(p) => p,+ Err(e) => {+ let err =+ xerj_common::XerjError::invalid_query(format!("invalid ISM policy body: {e}"));+ return ApiError::new(err).into_response();+ }+ };+ if let Err(reason) = policy.validate() {+ let err = xerj_common::XerjError::invalid_query(format!("invalid ISM policy: {reason}"));+ return ApiError::new(err).into_response();+ }+ state+ .engine+ .put_ism_policy(policy_id.clone(), policy.clone());+ Json(json!({+ "_id": policy_id,+ "_version": 1,+ "_seq_no": 0,+ "_primary_term": 1,+ "policy": policy,+ }))+ .into_response()+}++pub async fn get_ism_policy(+ State(state): State<AppState>,+ Path(policy_id): Path<String>,+) -> impl IntoResponse {+ match state.engine.ism_policies.get(&policy_id) {+ Some(policy) => Json(json!({+ "_id": policy_id,+ "_version": 1,+ "_seq_no": 0,+ "_primary_term": 1,+ "policy": policy.value(),+ }))+ .into_response(),+ None => {+ let e =+ xerj_common::XerjError::index_not_found(format!("policy [{policy_id}] not found"));+ ApiError::new(e).into_response()+ }+ }+}++#[derive(Debug, Deserialize, Default)]+pub struct ListParams {+ #[serde(default)]+ pub from: usize,+ #[serde(default = "default_size")]+ pub size: usize,+ #[serde(default)]+ pub search: String,+ #[serde(default)]+ pub sort_direction: Option<String>,+}+fn default_size() -> usize {+ 20+}++/// `GET /_plugins/_ism/policies` — the list/search form OSD's own Index+/// Management UI uses to populate the "State management policies" table+/// (its backend proxies `/api/ism/policies` straight through to this+/// endpoint). Found missing while verifying the real UI: only the+/// single-policy `GET .../policies/{id}` was implemented, so the list page+/// rendered "no existing policies" and a raw Internal Server Error toast+/// even immediately after a successful create.+pub async fn list_ism_policies(+ State(state): State<AppState>,+ Query(params): Query<ListParams>,+) -> impl IntoResponse {+ let mut all: Vec<(String, LifecyclePolicy)> = state+ .engine+ .ism_policies+ .iter()+ .map(|e| (e.key().clone(), e.value().clone()))+ .filter(|(id, _)| params.search.is_empty() || id.contains(¶ms.search))+ .collect();+ all.sort_by(|a, b| a.0.cmp(&b.0));+ if params.sort_direction.as_deref() == Some("desc") {+ all.reverse();+ }+ let total = all.len();+ let page: Vec<Value> = all+ .into_iter()+ .skip(params.from)+ .take(params.size.max(1))+ .map(|(id, policy)| {+ json!({+ "_id": id,+ "_version": 1,+ "_seq_no": 0,+ "_primary_term": 1,+ "policy": policy,+ })+ })+ .collect();+ Json(json!({ "totalPolicies": total, "policies": page })).into_response()+}++pub async fn delete_ism_policy(+ State(state): State<AppState>,+ Path(policy_id): Path<String>,+) -> impl IntoResponse {+ if state.engine.remove_ism_policy(&policy_id) {+ Json(json!({ "_id": policy_id, "result": "deleted" })).into_response()+ } else {+ let e = xerj_common::XerjError::index_not_found(format!("policy [{policy_id}] not found"));+ ApiError::new(e).into_response()+ }+}++// ─────────────────────────────────────────────────────────────────────────────+// POST /_plugins/_ism/add/{index}+// ─────────────────────────────────────────────────────────────────────────────++#[derive(Debug, Deserialize)]+pub struct AddPolicyBody {+ pub policy_id: String,+}++pub async fn add_ism_policy(+ State(state): State<AppState>,+ Path(index): Path<String>,+ Json(body): Json<AddPolicyBody>,+) -> impl IntoResponse {+ attach_policy(&state, &index, &body.policy_id).await+}++/// Shared attach implementation: validates the target index and policy both+/// exist, then creates (or resets) that index's managed-index execution+/// cursor at the policy's `default_state`. Used by the native ISM endpoint+/// above and by the ES-shape `index.lifecycle.name` auto-attach in+/// `es_compat` (a single index↔policy association model, per the design —+/// not two).+pub async fn attach_policy(state: &AppState, index: &str, policy_id: &str) -> Response {+ if state.engine.get_index(index).is_err() {+ let e = xerj_common::XerjError::index_not_found(index);+ return ApiError::new(e).into_response();+ }+ let Some(policy) = state+ .engine+ .ism_policies+ .get(policy_id)+ .map(|e| e.value().clone())+ else {+ let e = xerj_common::XerjError::index_not_found(format!("policy [{policy_id}] not found"));+ return ApiError::new(e).into_response();+ };+ let managed = ManagedIndexState::new(+ policy_id.to_string(),+ policy.default_state.clone(),+ lifecycle::now_ms(),+ );+ state+ .engine+ .managed_indices+ .insert(index.to_string(), managed);+ state.engine.persist_managed_indices();++ Json(json!({+ "failures": false,+ "updated_indices": 1,+ "failed_indices": [],+ }))+ .into_response()+}++// ─────────────────────────────────────────────────────────────────────────────+// GET /_plugins/_ism/explain/{index}+// ─────────────────────────────────────────────────────────────────────────────++/// `GET /_plugins/_ism/explain` — the list-all form OSD's "Policy managed+/// indexes" table uses (its backend's `/api/ism/managedIndices` proxies+/// straight through to this). Same gap as `list_ism_policies`: only the+/// single-index `.../explain/{index}` was implemented at first.+pub async fn list_managed_indices(+ State(state): State<AppState>,+ Query(params): Query<ListParams>,+) -> impl IntoResponse {+ let mut all: Vec<(String, ManagedIndexState)> = state+ .engine+ .managed_indices+ .iter()+ .map(|e| (e.key().clone(), e.value().clone()))+ .filter(|(name, _)| params.search.is_empty() || name.contains(¶ms.search))+ .collect();+ all.sort_by(|a, b| a.0.cmp(&b.0));+ if params.sort_direction.as_deref() == Some("desc") {+ all.reverse();+ }+ let total = all.len();+ let mut body = serde_json::Map::new();+ for (name, managed) in all.into_iter().skip(params.from).take(params.size.max(1)) {+ body.insert(name.clone(), lifecycle::explain_json(&name, &managed));+ }+ body.insert("total_managed_indices".to_string(), json!(total));+ Json(Value::Object(body)).into_response()+}++pub async fn explain_ism_index(+ State(state): State<AppState>,+ Path(index): Path<String>,+) -> impl IntoResponse {+ match state.engine.managed_indices.get(&index) {+ Some(managed) => {+ let mut body = serde_json::Map::new();+ body.insert(+ index.clone(),+ lifecycle::explain_json(&index, managed.value()),+ );+ body.insert("total_managed_indices".to_string(), json!(1));+ Json(Value::Object(body)).into_response()+ }+ None => {+ // Real ISM answers 200 with a "not managed" note rather than+ // 404 — `explain` on an unmanaged (or nonexistent) index is a+ // normal, expected call from Dashboards' Managed Indexes view.+ Json(json!({+ index.clone(): { "index": index, "enabled": false },+ "total_managed_indices": 0,+ }))+ .into_response()+ }+ }+}diff --git a/engine/crates/xerj-api/src/lib.rs b/engine/crates/xerj-api/src/lib.rsindex 876ba358e..0e63816d3 100644--- a/engine/crates/xerj-api/src/lib.rs+++ b/engine/crates/xerj-api/src/lib.rs@@ -63,6 +63,7 @@ pub mod error;pub mod es_compat;pub mod extract;pub mod graph_api;+pub mod ism_api;pub mod memory_api;pub mod native;pub mod responses;diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rsindex a91c4b7bf..f26ac4b1d 100644--- a/engine/crates/xerj-api/src/router.rs+++ b/engine/crates/xerj-api/src/router.rs@@ -28,7 +28,8 @@ use uuid::Uuid;use xerj_common::config::CorsConfig;use crate::{- auth::auth_middleware, authz, es_compat, graph_api, memory_api, native, state::AppState,+ auth::auth_middleware, authz, es_compat, graph_api, ism_api, memory_api, native,+ state::AppState,};// ─────────────────────────────────────────────────────────────────────────────@@ -454,6 +455,22 @@ pub fn build_es_compat_router(state: AppState) -> Router {.get(es_compat::get_ilm_policy).delete(es_compat::delete_ilm_policy),)+ .route("/:index/_ilm/explain", get(es_compat::ilm_explain))+ // ── ISM (OpenSearch Index State Management) ─────────────────────────+ // Same execution engine as ILM above — see `xerj_engine::lifecycle`.+ .route(+ "/_plugins/_ism/policies/:policy_id",+ put(ism_api::put_ism_policy)+ .get(ism_api::get_ism_policy)+ .delete(ism_api::delete_ism_policy),+ )+ .route("/_plugins/_ism/policies", get(ism_api::list_ism_policies))+ .route("/_plugins/_ism/add/:index", post(ism_api::add_ism_policy))+ .route(+ "/_plugins/_ism/explain/:index",+ get(ism_api::explain_ism_index),+ )+ .route("/_plugins/_ism/explain", get(ism_api::list_managed_indices))// ── Component templates ────────────────────────────────────────────.route("/_component_template/:name",diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rsindex 06f8616e7..418940a30 100644--- a/engine/crates/xerj-common/src/config.rs+++ b/engine/crates/xerj-common/src/config.rs@@ -84,9 +84,11 @@ pub struct Config {pub logging: LoggingConfig,/// Elasticsearch/OpenSearch wire-compatibility identity — 2 settings.pub compat: CompatConfig,+ /// ISM/ILM index-lifecycle-management background execution — 1 setting.+ pub lifecycle: LifecycleConfig,}-// Total: 5+3+2+3+10+5+3+1+6+2+4+3+4+3+2 = 56 fields (incl. cors: 2, auth: 3,+// Total: 5+3+2+3+10+5+3+1+6+2+4+3+4+3+2+1 = 57 fields (incl. cors: 2, auth: 3,// logging: 2). `Default` is derived — every field is a sub-config that// implements `Default`, so the derive produces exactly the same all-defaults// value the manual impl used to build by hand.@@ -1315,6 +1317,28 @@ impl Default for PitConfig {}}+/// ISM/ILM index-lifecycle-management background execution.+///+/// Drives `xerj_engine::lifecycle`'s tick: for every managed index, run the+/// current state's pending actions, then evaluate transitions in order and+/// move to the first one whose conditions are met.+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(default, deny_unknown_fields)]+pub struct LifecycleConfig {+ /// How often the background job ticks (default: 300 seconds = 5+ /// minutes — the same default OpenSearch ISM itself uses for+ /// `plugins.index_state_management.job_interval`).+ pub tick_interval_secs: u64,+}++impl Default for LifecycleConfig {+ fn default() -> Self {+ Self {+ tick_interval_secs: 300,+ }+ }+}+// ─────────────────────────────────────────────────────────────────────────────/// Scroll + async-search context lifecycle settings (RC4 blocker 11).diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex fd51d2346..29ef3de29 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -272,8 +272,25 @@ pub struct Engine {pub closed_indices: Arc<DashMap<String, bool>>,/// data stream name → DataStreampub data_streams: Arc<DashMap<String, DataStream>>,- /// ILM policy name → policy JSON+ /// ILM policy name → raw policy JSON, kept verbatim for `GET _ilm/policy`+ /// round-trip fidelity. The REAL execution engine reads `ism_policies`+ /// instead — `PUT _ilm/policy/{name}` writes to BOTH maps (see+ /// `xerj_api::es_compat::put_ilm_policy`), translating into the shared+ /// internal model. See `crate::lifecycle` for why one engine drives both+ /// surfaces.pub ilm_policies: Arc<DashMap<String, Value>>,+ /// policy id → the internal ISM-shaped model every managed index is+ /// actually driven by, regardless of whether the policy was created via+ /// `_plugins/_ism/policies` (native) or `_ilm/policy` (translated).+ pub ism_policies: Arc<DashMap<String, crate::lifecycle::LifecyclePolicy>>,+ /// index name → lifecycle execution cursor (current state, pending+ /// action, timestamps). Presence in this map is what "managed" means —+ /// an index with no entry here is not touched by the background job.+ pub managed_indices: Arc<DashMap<String, crate::lifecycle::ManagedIndexState>>,+ /// index name → creation time (epoch ms), for `min_index_age` and for+ /// `GET /{index}`'s `creation_date` (previously synthesized as+ /// `Utc::now()` on every request — see `record_index_created_at`).+ pub index_created_at: Arc<DashMap<String, i64>>,/// component template name → template JSONpub component_templates: Arc<DashMap<String, Value>>,/// snapshot repository name → repo config JSON@@ -430,6 +447,9 @@ impl Engine {closed_indices: Arc::new(DashMap::new()),data_streams: Arc::new(DashMap::new()),ilm_policies: Arc::new(DashMap::new()),+ ism_policies: Arc::new(DashMap::new()),+ managed_indices: Arc::new(DashMap::new()),+ index_created_at: Arc::new(DashMap::new()),component_templates: Arc::new(DashMap::new()),snapshot_repos: Arc::new(DashMap::new()),snapshots: Arc::new(DashMap::new()),@@ -506,6 +526,7 @@ impl Engine {if let Some(m) = engine.index_mappings.get(name_str.as_str()) {Engine::apply_date_mapping_flags(&idx, m.value());}+ engine.load_or_backfill_index_created_at(&name_str);engine.indices.insert(name_str, idx);}Err(e) => {@@ -529,6 +550,12 @@ impl Engine {// migration by another instance).engine.load_persisted_aliases();+ // Restore ISM/ILM policies and managed-index execution state so a+ // policy attached before a restart keeps running afterward instead+ // of silently going idle.+ engine.load_persisted_ism_policies();+ engine.load_persisted_managed_indices();+// Spawn the PIT sweeper. Pre-v0.6.2 PITs accumulated forever;// every open without close was a memory leak. The sweeper// walks `engine.pits` every `pit.sweep_interval_secs` and@@ -834,6 +861,7 @@ impl Engine {&self.data_dir,)?;self.indices.insert(name.to_string(), idx);+ self.record_index_created_at(name);info!(name, "index created with custom settings");Ok(())}@@ -1098,6 +1126,184 @@ impl Engine {}}+ // ── Lifecycle management (ISM/ILM) persistence ──────────────────────────++ fn ism_policies_path(&self) -> PathBuf {+ self.data_dir.join("ism_policies.json")+ }++ fn managed_indices_path(&self) -> PathBuf {+ self.data_dir.join("ism_managed_indices.json")+ }++ /// Insert (or overwrite) an ISM-shaped policy and persist the whole+ /// store, mirroring `flush_aliases`. Both `_plugins/_ism/policies` (as+ /// entered) and `_ilm/policy` (after translation) write through here —+ /// see the `ism_policies` field doc.+ pub fn put_ism_policy(&self, id: String, policy: crate::lifecycle::LifecyclePolicy) {+ self.ism_policies.insert(id, policy);+ self.flush_ism_policies();+ }++ pub fn remove_ism_policy(&self, id: &str) -> bool {+ let removed = self.ism_policies.remove(id).is_some();+ if removed {+ self.flush_ism_policies();+ }+ removed+ }++ fn flush_ism_policies(&self) {+ let snapshot: std::collections::HashMap<String, crate::lifecycle::LifecyclePolicy> = self+ .ism_policies+ .iter()+ .map(|e| (e.key().clone(), e.value().clone()))+ .collect();+ let bytes = match serde_json::to_vec_pretty(&snapshot) {+ Ok(b) => b,+ Err(e) => {+ warn!(error = %e, "failed to serialize ism_policies for persistence");+ return;+ }+ };+ if let Err(e) = crate::index::write_file_atomic(&self.ism_policies_path(), &bytes) {+ warn!(error = %e, "failed to persist ism_policies.json (policies work until restart)");+ }+ }++ fn load_persisted_ism_policies(&self) {+ let Ok(bytes) = std::fs::read(self.ism_policies_path()) else {+ return;+ };+ match serde_json::from_slice::<+ std::collections::HashMap<String, crate::lifecycle::LifecyclePolicy>,+ >(&bytes)+ {+ Ok(map) => {+ let n = map.len();+ for (id, policy) in map {+ self.ism_policies.insert(id, policy);+ }+ if n > 0 {+ info!(count = n, "restored persisted ISM/ILM policies");+ }+ }+ Err(e) => warn!(error = %e, "ignoring corrupt ism_policies.json"),+ }+ }++ /// Persist the whole `managed_indices` map. Called by `lifecycle::tick`+ /// after any change, and by the attach/detach handlers.+ pub fn persist_managed_indices(&self) {+ let snapshot: std::collections::HashMap<String, crate::lifecycle::ManagedIndexState> = self+ .managed_indices+ .iter()+ .map(|e| (e.key().clone(), e.value().clone()))+ .collect();+ let bytes = match serde_json::to_vec_pretty(&snapshot) {+ Ok(b) => b,+ Err(e) => {+ warn!(error = %e, "failed to serialize managed_indices for persistence");+ return;+ }+ };+ if let Err(e) = crate::index::write_file_atomic(&self.managed_indices_path(), &bytes) {+ warn!(error = %e, "failed to persist ism_managed_indices.json (state works until restart)");+ }+ }++ fn load_persisted_managed_indices(&self) {+ let Ok(bytes) = std::fs::read(self.managed_indices_path()) else {+ return;+ };+ match serde_json::from_slice::<+ std::collections::HashMap<String, crate::lifecycle::ManagedIndexState>,+ >(&bytes)+ {+ Ok(map) => {+ let n = map.len();+ for (index_name, state) in map {+ self.managed_indices.insert(index_name, state);+ }+ if n > 0 {+ info!(count = n, "restored persisted ISM managed-index state");+ }+ }+ Err(e) => warn!(error = %e, "ignoring corrupt ism_managed_indices.json"),+ }+ }++ /// Spawn the background lifecycle-execution job: every+ /// `config.lifecycle.tick_interval_secs` (default 300s = 5 minutes,+ /// matching OpenSearch ISM's own default job interval), runs+ /// `lifecycle::tick` over every managed index. Not latency-critical+ /// (unlike the resource governor's sampler), so a plain `tokio::spawn`+ /// on an interval is fine — uses a `Weak` self-pointer so it exits+ /// cleanly when the engine is dropped (matters for tests, which create+ /// and drop many `Engine`s).+ pub fn spawn_lifecycle_manager(self: &Arc<Self>) {+ let interval =+ std::time::Duration::from_secs(self.config.lifecycle.tick_interval_secs.max(1));+ let weak = Arc::downgrade(self);+ tokio::spawn(async move {+ let mut tick = tokio::time::interval(interval);+ tick.tick().await; // skip the immediate first tick+ loop {+ tick.tick().await;+ let Some(engine) = weak.upgrade() else {+ return; // engine dropped+ };+ crate::lifecycle::tick(&engine).await;+ }+ });+ }++ // ── Index creation-date tracking ─────────────────────────────────────────++ fn index_created_at_path(&self, name: &str) -> PathBuf {+ self.data_dir.join(name).join("created_at.json")+ }++ /// Record `name`'s creation time (now) and persist it — called exactly+ /// once, at the end of `create_index_with_settings`. Without this,+ /// `min_index_age` has no ground truth, and `GET /{index}`'s+ /// `creation_date` was synthesized fresh on every single request+ /// (always reporting "now"), which is what this replaces.+ fn record_index_created_at(&self, name: &str) {+ let now = crate::lifecycle::now_ms();+ self.index_created_at.insert(name.to_string(), now);+ match serde_json::to_vec_pretty(&serde_json::json!({ "creation_date_ms": now })) {+ Ok(bytes) => {+ if let Err(e) =+ crate::index::write_file_atomic(&self.index_created_at_path(name), &bytes)+ {+ warn!(index = name, error = %e, "failed to persist created_at.json");+ }+ }+ Err(e) => warn!(index = name, error = %e, "failed to serialize created_at.json"),+ }+ }++ /// Load `name`'s persisted creation time on startup. An index directory+ /// that predates this feature (no `created_at.json`) gets a synthetic+ /// baseline of "now" recorded and persisted on first post-upgrade boot —+ /// its true history is unknown, so `min_index_age` starts counting from+ /// the upgrade rather than from the index's real (unrecorded) creation.+ /// Documented tradeoff, not silently wrong: every index that existed+ /// before this shipped gets exactly one synthetic reset, once.+ fn load_or_backfill_index_created_at(&self, name: &str) {+ let path = self.index_created_at_path(name);+ if let Ok(bytes) = std::fs::read(&path) {+ if let Ok(v) = serde_json::from_slice::<Value>(&bytes) {+ if let Some(ms) = v.get("creation_date_ms").and_then(Value::as_i64) {+ self.index_created_at.insert(name.to_string(), ms);+ return;+ }+ }+ }+ self.record_index_created_at(name);+ }+/// Add an alias pointing to an index.pub fn add_alias(&self, alias: &str, index: &str) {let mut entry = self.aliases.entry(alias.to_string()).or_default();diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rsindex 1e07ea989..60fadf50f 100644--- a/engine/crates/xerj-engine/src/lib.rs+++ b/engine/crates/xerj-engine/src/lib.rs@@ -25,6 +25,7 @@ pub mod governor;pub mod index;pub mod index_guard;pub mod ingest_memory;+pub mod lifecycle;pub mod memtable;pub mod painless;pub mod rbac;diff --git a/engine/crates/xerj-engine/src/lifecycle.rs b/engine/crates/xerj-engine/src/lifecycle.rsnew file mode 100644index 000000000..d1a75de31--- /dev/null+++ b/engine/crates/xerj-engine/src/lifecycle.rs@@ -0,0 +1,848 @@+//! Index lifecycle management — one execution engine, two REST surfaces.+//!+//! Modeled on OpenSearch's Index State Management (ISM), not Elasticsearch's+//! fixed hot/warm/cold/delete phases: ISM's state machine (named states,+//! each with its own ordered actions and ordered transitions with+//! conditions) is strictly more general, so it's the internal model.+//! Elasticsearch's `_ilm/*` surface is a translator into this same model+//! (see [`translate_ilm_policy`]), not a second engine — an ILM policy with+//! hot/warm/cold/delete phases becomes an ISM policy whose states are named+//! `"hot"`/`"warm"`/`"cold"`/`"delete"` in that fixed sequence.+//!+//! ## Known simplification vs. real ILM+//!+//! Real Elasticsearch measures a phase's `min_age` from the index's+//! rollover time (not its creation time) once the policy's hot phase has a+//! `rollover` action — the age clock effectively restarts at rollover. This+//! engine measures every state's age from when the index *entered that+//! state* (`state_entered_at_ms`), which is index-attach time for the+//! initial state, not rollover time. For a policy without `rollover`, or+//! for reasoning about relative phase durations, this is equivalent; for a+//! policy that rolls over AND has a downstream phase whose `min_age` is+//! meant to be measured strictly from rollover, this engine will advance+//! phases slightly earlier than real ILM would. Documented rather than+//! silently diverging — see PR discussion for whether this needs closing.+//!+//! ## What's NOT supported, and errors honestly instead of faking success+//!+//! `replica_count`: XERJ has no per-index configurable replica concept+//! (single-shard, single-node engine; `number_of_replicas` is accepted in+//! index settings and echoed back for wire compatibility but never+//! consumed). The action fails with an explicit error rather than silently+//! no-op'ing, so a policy relying on it is visibly stuck, not silently+//! wrong.++use std::time::{SystemTime, UNIX_EPOCH};++use serde::{Deserialize, Serialize};+use serde_json::{json, Value};++use crate::engine::Engine;+use crate::{EngineError, Result};++// ─────────────────────────────────────────────────────────────────────────────+// Data model — faithful to OpenSearch ISM's policy JSON shape.+// ─────────────────────────────────────────────────────────────────────────────++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct LifecyclePolicy {+ #[serde(default)]+ pub description: Option<String>,+ pub default_state: String,+ pub states: Vec<LifecycleState>,+}++impl LifecyclePolicy {+ /// Structural validation beyond what serde already enforces: every+ /// transition target and `default_state` must name a real state, and+ /// state names must be unique. Real ISM rejects a policy with a+ /// dangling transition at PUT time rather than discovering it mid-run.+ pub fn validate(&self) -> std::result::Result<(), String> {+ if self.states.is_empty() {+ return Err("policy must declare at least one state".to_string());+ }+ let names: std::collections::HashSet<&str> =+ self.states.iter().map(|s| s.name.as_str()).collect();+ if names.len() != self.states.len() {+ return Err("state names must be unique".to_string());+ }+ if !names.contains(self.default_state.as_str()) {+ return Err(format!(+ "default_state '{}' does not name a declared state",+ self.default_state+ ));+ }+ for state in &self.states {+ for t in &state.transitions {+ if !names.contains(t.state_name.as_str()) {+ return Err(format!(+ "state '{}' transitions to undeclared state '{}'",+ state.name, t.state_name+ ));+ }+ }+ }+ Ok(())+ }++ pub fn state(&self, name: &str) -> Option<&LifecycleState> {+ self.states.iter().find(|s| s.name == name)+ }+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct LifecycleState {+ pub name: String,+ #[serde(default)]+ pub actions: Vec<LifecycleAction>,+ #[serde(default)]+ pub transitions: Vec<LifecycleTransition>,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct LifecycleTransition {+ pub state_name: String,+ #[serde(default)]+ pub conditions: Option<LifecycleConditions>,+}++/// Conditions shared by transitions (when to leave a state) and the+/// `rollover` action (when it's actually worth rolling over) — same shape,+/// same evaluator, two different call sites. A transition/action with no+/// conditions at all is always eligible the moment it's checked (an+/// unconditional transition; an unconditional rollover).+#[derive(Debug, Clone, Default, Serialize, Deserialize)]+pub struct LifecycleConditions {+ /// ES/ISM duration string: digits followed by `ms|s|m|h|d`. E.g. `"7d"`.+ #[serde(default)]+ pub min_index_age: Option<String>,+ /// ES/ISM size string: digits followed by `b|kb|mb|gb|tb`. E.g. `"50gb"`.+ #[serde(default)]+ pub min_size: Option<String>,+ #[serde(default)]+ pub min_doc_count: Option<u64>,+}++#[derive(Debug, Clone, Default, Serialize, Deserialize)]+pub struct EmptyParams {}++#[derive(Debug, Clone, Default, Serialize, Deserialize)]+pub struct RolloverAction {+ #[serde(flatten)]+ pub conditions: LifecycleConditions,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct ReplicaCountAction {+ pub number_of_replicas: u64,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum LifecycleAction {+ Rollover(RolloverAction),+ Delete(EmptyParams),+ ReadOnly(EmptyParams),+ ReplicaCount(ReplicaCountAction),+}++impl LifecycleAction {+ pub fn name(&self) -> &'static str {+ match self {+ LifecycleAction::Rollover(_) => "rollover",+ LifecycleAction::Delete(_) => "delete",+ LifecycleAction::ReadOnly(_) => "read_only",+ LifecycleAction::ReplicaCount(_) => "replica_count",+ }+ }+}++// ─────────────────────────────────────────────────────────────────────────────+// Managed-index state — the per-index execution cursor.+// ─────────────────────────────────────────────────────────────────────────────++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct ManagedIndexState {+ pub policy_id: String,+ pub current_state: String,+ pub state_entered_at_ms: i64,+ /// Index into the current state's `actions[]` — which one runs next.+ /// Equal to `actions.len()` once every action in this state has+ /// completed and only transition-evaluation remains.+ #[serde(default)]+ pub next_action_index: usize,+ pub last_updated_ms: i64,+ /// Mirrors ISM explain's free-form `info.message`.+ #[serde(default)]+ pub info_message: String,+ /// Set when a step errors, so `explain` surfaces it instead of the+ /// index silently retrying the same failing action forever unseen.+ /// Retries still happen (a transient failure — e.g. disk contention —+ /// should self-heal); this is visibility, not a halt.+ #[serde(default)]+ pub failed: bool,+}++impl ManagedIndexState {+ pub fn new(policy_id: String, default_state: String, now_ms: i64) -> Self {+ Self {+ policy_id,+ current_state: default_state,+ state_entered_at_ms: now_ms,+ next_action_index: 0,+ last_updated_ms: now_ms,+ info_message: "attached, awaiting first execution".to_string(),+ failed: false,+ }+ }+}++pub fn now_ms() -> i64 {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map(|d| d.as_millis() as i64)+ .unwrap_or(0)+}++// ─────────────────────────────────────────────────────────────────────────────+// Duration / size string parsing — ES/ISM shorthand.+// ─────────────────────────────────────────────────────────────────────────────++/// Parse an ES/ISM duration string (`"7d"`, `"30m"`, `"500ms"`) into+/// milliseconds. Unlike ES's full duration grammar this accepts exactly one+/// integer + one unit, which is all `min_index_age` ever uses in practice.+pub fn parse_duration_ms(s: &str) -> std::result::Result<i64, String> {+ let s = s.trim();+ let split_at = s+ .find(|c: char| !c.is_ascii_digit())+ .ok_or_else(|| format!("duration '{s}' has no unit"))?;+ let (digits, unit) = s.split_at(split_at);+ let n: i64 = digits+ .parse()+ .map_err(|_| format!("duration '{s}' has no numeric prefix"))?;+ let ms = match unit {+ "ms" => n,+ "s" => n * 1_000,+ "m" => n * 60_000,+ "h" => n * 3_600_000,+ "d" => n * 86_400_000,+ other => return Err(format!("unknown duration unit '{other}' in '{s}'")),+ };+ Ok(ms)+}++/// Parse an ES/ISM size string (`"50gb"`, `"100mb"`) into bytes. Binary+/// (1024-based) units, matching ES's own `ByteSizeValue`.+pub fn parse_size_bytes(s: &str) -> std::result::Result<u64, String> {+ let s = s.trim().to_ascii_lowercase();+ let split_at = s+ .find(|c: char| !c.is_ascii_digit())+ .ok_or_else(|| format!("size '{s}' has no unit"))?;+ let (digits, unit) = s.split_at(split_at);+ let n: u64 = digits+ .parse()+ .map_err(|_| format!("size '{s}' has no numeric prefix"))?;+ let bytes = match unit {+ "b" => n,+ "kb" => n * 1024,+ "mb" => n * 1024 * 1024,+ "gb" => n * 1024 * 1024 * 1024,+ "tb" => n * 1024 * 1024 * 1024 * 1024,+ other => return Err(format!("unknown size unit '{other}' in '{s}'")),+ };+ Ok(bytes)+}++// ─────────────────────────────────────────────────────────────────────────────+// ES ILM → internal model translator.+// ─────────────────────────────────────────────────────────────────────────────++/// Fixed ILM phase order — ISM states are named exactly this, one per phase+/// actually present in the policy. A phase absent from the ILM policy+/// produces no state at all (matching ILM, where an unconfigured phase is+/// simply skipped).+const ILM_PHASE_ORDER: [&str; 4] = ["hot", "warm", "cold", "delete"];++/// Translate an ES ILM policy body (`{"policy": {"phases": {...}}}`, the+/// same shape `PUT _ilm/policy/{name}` accepts) into a [`LifecyclePolicy`].+///+/// Each present phase becomes a state named after the phase. A phase's own+/// `actions` (minus `min_age`, which isn't an action) become that state's+/// actions. A phase's `min_age` becomes the CONDITION on the transition+/// INTO that phase from the previous one — i.e. "warm: min_age 7d" means+/// the state before warm gets a transition to warm gated on+/// `min_index_age: 7d`. The final present phase gets no transitions+/// (terminal), matching a real ISM policy's last state.+pub fn translate_ilm_policy(ilm_body: &Value) -> std::result::Result<LifecyclePolicy, String> {+ let phases = ilm_body+ .pointer("/policy/phases")+ .and_then(Value::as_object)+ .ok_or_else(|| "ILM policy body missing policy.phases".to_string())?;++ let present: Vec<&str> = ILM_PHASE_ORDER+ .iter()+ .copied()+ .filter(|p| phases.contains_key(*p))+ .collect();+ if present.is_empty() {+ return Err("ILM policy declares no known phase (hot/warm/cold/delete)".to_string());+ }++ let mut states = Vec::with_capacity(present.len());+ for (i, phase_name) in present.iter().enumerate() {+ let phase = &phases[*phase_name];+ let actions = translate_ilm_phase_actions(phase_name, phase)?;++ let transitions = if let Some(&next_phase) = present.get(i + 1) {+ let next_min_age = phases[next_phase]+ .get("min_age")+ .and_then(Value::as_str)+ .unwrap_or("0ms");+ vec![LifecycleTransition {+ state_name: next_phase.to_string(),+ conditions: Some(LifecycleConditions {+ min_index_age: Some(next_min_age.to_string()),+ min_size: None,+ min_doc_count: None,+ }),+ }]+ } else {+ Vec::new()+ };++ states.push(LifecycleState {+ name: phase_name.to_string(),+ actions,+ transitions,+ });+ }++ let policy = LifecyclePolicy {+ description: ilm_body+ .pointer("/policy/_meta/description")+ .and_then(Value::as_str)+ .map(String::from),+ default_state: present[0].to_string(),+ states,+ };+ policy.validate()?;+ Ok(policy)+}++fn translate_ilm_phase_actions(+ phase_name: &str,+ phase: &Value,+) -> std::result::Result<Vec<LifecycleAction>, String> {+ let Some(actions_obj) = phase.get("actions").and_then(Value::as_object) else {+ return Ok(Vec::new());+ };+ let mut out = Vec::new();+ for (action_name, params) in actions_obj {+ let action = match action_name.as_str() {+ "rollover" => {+ let conditions = LifecycleConditions {+ min_index_age: params+ .get("max_age")+ .and_then(Value::as_str)+ .map(String::from),+ min_size: params+ .get("max_size")+ .and_then(Value::as_str)+ .map(String::from),+ min_doc_count: params.get("max_docs").and_then(Value::as_u64),+ };+ LifecycleAction::Rollover(RolloverAction { conditions })+ }+ "delete" => LifecycleAction::Delete(EmptyParams {}),+ "readonly" => LifecycleAction::ReadOnly(EmptyParams {}),+ "set_priority" | "allocate" | "unfollow" | "shrink" | "forcemerge" | "freeze" => {+ // Real ILM actions with no XERJ equivalent (no shard+ // allocation awareness, no shrink/forcemerge-as-a-lifecycle-+ // step wiring, no CCR). Skipped rather than erroring the+ // whole policy translation — matches this engine's existing+ // "explicit error only where behavior would be silently+ // wrong" policy: skipping a no-op-shaped action is honest+ // (nothing was promised and not delivered), unlike+ // replica_count which IS user-visible if silently ignored.+ continue;+ }+ other => {+ return Err(format!(+ "phase '{phase_name}' action '{other}' has no ISM/XERJ equivalent"+ ))+ }+ };+ out.push(action);+ }+ Ok(out)+}++// ─────────────────────────────────────────────────────────────────────────────+// Condition evaluation.+// ─────────────────────────────────────────────────────────────────────────────++struct IndexFacts {+ age_ms: i64,+ size_bytes: u64,+ doc_count: u64,+}++async fn gather_index_facts(+ engine: &Engine,+ index_name: &str,+ state_age_ms: i64,+) -> Option<IndexFacts> {+ let idx = engine.get_index(index_name).ok()?;+ let stats = idx.stats().await;+ let segment_bytes: u64 = idx+ .store_snapshot()+ .segments+ .iter()+ .map(|s| s.size_bytes)+ .sum();+ let size_bytes = segment_bytes + stats.memtable_size_bytes as u64;+ Some(IndexFacts {+ age_ms: state_age_ms,+ size_bytes,+ doc_count: stats.doc_count,+ })+}++/// A `LifecycleConditions` with every field `None` matches immediately —+/// this is what makes an unconditional transition / unconditional rollover+/// always-eligible the first time it's checked.+fn conditions_met(+ conditions: &LifecycleConditions,+ facts: &IndexFacts,+) -> std::result::Result<bool, String> {+ if let Some(age_str) = &conditions.min_index_age {+ let threshold_ms = parse_duration_ms(age_str)?;+ if facts.age_ms < threshold_ms {+ return Ok(false);+ }+ }+ if let Some(size_str) = &conditions.min_size {+ let threshold_bytes = parse_size_bytes(size_str)?;+ if facts.size_bytes < threshold_bytes {+ return Ok(false);+ }+ }+ if let Some(min_docs) = conditions.min_doc_count {+ if facts.doc_count < min_docs {+ return Ok(false);+ }+ }+ Ok(true)+}++// ─────────────────────────────────────────────────────────────────────────────+// Action execution.+// ─────────────────────────────────────────────────────────────────────────────++pub enum ActionOutcome {+ /// The action completed; advance past it.+ Done,+ /// The action's own conditions aren't met yet (rollover only); leave+ /// `next_action_index` where it is and retry next tick.+ Waiting,+ /// The managed index itself is gone (delete action succeeded) — the+ /// caller must drop this index from `managed_indices` entirely rather+ /// than continue processing it.+ IndexDeleted,+}++async fn execute_action(+ engine: &Engine,+ index_name: &str,+ action: &LifecycleAction,+ state_age_ms: i64,+) -> Result<ActionOutcome> {+ match action {+ LifecycleAction::Rollover(rollover) => {+ let has_conditions = rollover.conditions.min_index_age.is_some()+ || rollover.conditions.min_size.is_some()+ || rollover.conditions.min_doc_count.is_some();+ if has_conditions {+ let Some(facts) = gather_index_facts(engine, index_name, state_age_ms).await else {+ return Err(EngineError::Common(+ xerj_common::XerjError::index_not_found(index_name),+ ));+ };+ let met = conditions_met(&rollover.conditions, &facts)+ .map_err(|e| EngineError::Common(xerj_common::XerjError::invalid_query(e)))?;+ if !met {+ return Ok(ActionOutcome::Waiting);+ }+ }+ // Reuses the existing, already-real rollover implementation —+ // requires `index_name` to be a registered data stream (alias ++ // generation-numbered backing indices). A plain index that+ // isn't a data stream fails here with a clear not-found error+ // rather than silently doing nothing.+ engine.rollover_data_stream(index_name)?;+ Ok(ActionOutcome::Done)+ }+ LifecycleAction::Delete(_) => {+ engine.delete_index(index_name).await?;+ engine.managed_indices.remove(index_name);+ Ok(ActionOutcome::IndexDeleted)+ }+ LifecycleAction::ReadOnly(_) => {+ let idx = engine.get_index(index_name)?;+ idx.set_block("write").await?;+ Ok(ActionOutcome::Done)+ }+ LifecycleAction::ReplicaCount(_) => Err(EngineError::Common(+ xerj_common::XerjError::internal(format!(+ "the 'replica_count' lifecycle action is not supported: XERJ is a \+ single-shard, single-node engine with no per-index configurable \+ replica count to change (number_of_replicas is accepted in index \+ settings for wire compatibility but never consumed) — policy for \+ index '{index_name}' cannot proceed past this action"+ )),+ )),+ }+}++// ─────────────────────────────────────────────────────────────────────────────+// Tick — the background job's one unit of work.+// ─────────────────────────────────────────────────────────────────────────────++/// Run one lifecycle pass over every managed index: execute the current+/// state's next pending action (if any), or — once all of a state's+/// actions have completed — evaluate its transitions in order and move to+/// the first one whose conditions are met.+pub async fn tick(engine: &Engine) {+ let snapshot: Vec<(String, ManagedIndexState)> = engine+ .managed_indices+ .iter()+ .map(|e| (e.key().clone(), e.value().clone()))+ .collect();++ let mut any_change = false;+ for (index_name, mut managed) in snapshot {+ let Some(policy) = engine+ .ism_policies+ .get(&managed.policy_id)+ .map(|e| e.value().clone())+ else {+ if !managed.failed {+ managed.failed = true;+ managed.info_message = format!("policy '{}' no longer exists", managed.policy_id);+ managed.last_updated_ms = now_ms();+ engine.managed_indices.insert(index_name, managed);+ any_change = true;+ }+ continue;+ };+ let Some(state_def) = policy.state(&managed.current_state).cloned() else {+ managed.failed = true;+ managed.info_message = format!(+ "state '{}' no longer exists in policy '{}'",+ managed.current_state, managed.policy_id+ );+ managed.last_updated_ms = now_ms();+ engine.managed_indices.insert(index_name, managed);+ any_change = true;+ continue;+ };++ let state_age_ms = now_ms() - managed.state_entered_at_ms;++ if managed.next_action_index < state_def.actions.len() {+ let action = &state_def.actions[managed.next_action_index];+ match execute_action(engine, &index_name, action, state_age_ms).await {+ Ok(ActionOutcome::Done) => {+ managed.next_action_index += 1;+ managed.failed = false;+ managed.info_message = format!("completed action '{}'", action.name());+ managed.last_updated_ms = now_ms();+ engine.managed_indices.insert(index_name, managed);+ any_change = true;+ }+ Ok(ActionOutcome::Waiting) => {+ managed.info_message =+ format!("waiting on '{}' action conditions", action.name());+ managed.last_updated_ms = now_ms();+ engine.managed_indices.insert(index_name, managed);+ any_change = true;+ }+ Ok(ActionOutcome::IndexDeleted) => {+ // Already removed from managed_indices by execute_action.+ any_change = true;+ }+ Err(e) => {+ managed.failed = true;+ managed.info_message = format!("action '{}' failed: {e}", action.name());+ managed.last_updated_ms = now_ms();+ engine.managed_indices.insert(index_name, managed);+ any_change = true;+ }+ }+ continue;+ }++ // All actions in this state are done — evaluate transitions in+ // order; first met condition wins. No transitions = terminal state.+ let Some(facts) = gather_index_facts(engine, &index_name, state_age_ms).await else {+ managed.failed = true;+ managed.info_message = "index no longer exists".to_string();+ managed.last_updated_ms = now_ms();+ engine.managed_indices.insert(index_name, managed);+ any_change = true;+ continue;+ };+ let mut transitioned = false;+ for t in &state_def.transitions {+ let met = match &t.conditions {+ None => true,+ Some(c) => conditions_met(c, &facts).unwrap_or(false),+ };+ if met {+ managed.current_state = t.state_name.clone();+ managed.state_entered_at_ms = now_ms();+ managed.next_action_index = 0;+ managed.failed = false;+ managed.info_message = format!("transitioned to state '{}'", t.state_name);+ managed.last_updated_ms = now_ms();+ engine.managed_indices.insert(index_name, managed);+ transitioned = true;+ any_change = true;+ break;+ }+ }+ if !transitioned {+ // Nothing to do this tick; leave last_updated_ms untouched so+ // explain doesn't churn on every quiet tick.+ }+ }++ if any_change {+ engine.persist_managed_indices();+ }+}++// ─────────────────────────────────────────────────────────────────────────────+// Explain — the read model both REST surfaces render from.+// ─────────────────────────────────────────────────────────────────────────────++/// Build the `explain` JSON for one managed index, matching the fields real+/// ISM's `GET _plugins/_ism/explain/{index}` reports (a practical subset —+/// `policy_seq_no`/`policy_primary_term` are omitted since this engine has+/// no policy versioning).+pub fn explain_json(index_name: &str, managed: &ManagedIndexState) -> Value {+ json!({+ "index": index_name,+ "index.plugins.index_state_management.policy_id": managed.policy_id,+ "policy_id": managed.policy_id,+ "state": {+ "name": managed.current_state,+ "start_time": managed.state_entered_at_ms,+ },+ "action": {+ "name": managed.info_message,+ "failed": managed.failed,+ },+ "info": {+ "message": managed.info_message,+ },+ })+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn parse_duration_handles_all_units() {+ assert_eq!(parse_duration_ms("500ms").unwrap(), 500);+ assert_eq!(parse_duration_ms("7s").unwrap(), 7_000);+ assert_eq!(parse_duration_ms("5m").unwrap(), 300_000);+ assert_eq!(parse_duration_ms("2h").unwrap(), 7_200_000);+ assert_eq!(parse_duration_ms("30d").unwrap(), 30 * 86_400_000);+ assert!(parse_duration_ms("30x").is_err());+ assert!(parse_duration_ms("abc").is_err());+ }++ #[test]+ fn parse_size_handles_all_units() {+ assert_eq!(parse_size_bytes("100b").unwrap(), 100);+ assert_eq!(parse_size_bytes("1kb").unwrap(), 1024);+ assert_eq!(parse_size_bytes("50mb").unwrap(), 50 * 1024 * 1024);+ assert_eq!(parse_size_bytes("2gb").unwrap(), 2 * 1024 * 1024 * 1024);+ assert!(parse_size_bytes("1zz").is_err());+ }++ #[test]+ fn conditions_met_empty_conditions_always_true() {+ let facts = IndexFacts {+ age_ms: 0,+ size_bytes: 0,+ doc_count: 0,+ };+ assert!(conditions_met(&LifecycleConditions::default(), &facts).unwrap());+ }++ #[test]+ fn conditions_met_requires_every_declared_condition() {+ let facts = IndexFacts {+ age_ms: 10_000,+ size_bytes: 100,+ doc_count: 5,+ };+ let conditions = LifecycleConditions {+ min_index_age: Some("1s".to_string()),+ min_size: None,+ min_doc_count: Some(10),+ };+ // age condition met (10s >= 1s) but doc_count condition not met (5 < 10)+ assert!(!conditions_met(&conditions, &facts).unwrap());++ let conditions2 = LifecycleConditions {+ min_index_age: Some("1s".to_string()),+ min_size: None,+ min_doc_count: Some(5),+ };+ assert!(conditions_met(&conditions2, &facts).unwrap());+ }++ #[test]+ fn policy_validate_rejects_dangling_transition() {+ let policy = LifecyclePolicy {+ description: None,+ default_state: "hot".to_string(),+ states: vec![LifecycleState {+ name: "hot".to_string(),+ actions: vec![],+ transitions: vec![LifecycleTransition {+ state_name: "warm".to_string(),+ conditions: None,+ }],+ }],+ };+ assert!(policy.validate().is_err());+ }++ #[test]+ fn policy_validate_rejects_unknown_default_state() {+ let policy = LifecyclePolicy {+ description: None,+ default_state: "nope".to_string(),+ states: vec![LifecycleState {+ name: "hot".to_string(),+ actions: vec![],+ transitions: vec![],+ }],+ };+ assert!(policy.validate().is_err());+ }++ #[test]+ fn policy_validate_accepts_well_formed_policy() {+ let policy = LifecyclePolicy {+ description: Some("test".to_string()),+ default_state: "hot".to_string(),+ states: vec![+ LifecycleState {+ name: "hot".to_string(),+ actions: vec![LifecycleAction::Rollover(RolloverAction::default())],+ transitions: vec![LifecycleTransition {+ state_name: "delete".to_string(),+ conditions: Some(LifecycleConditions {+ min_index_age: Some("30d".to_string()),+ min_size: None,+ min_doc_count: None,+ }),+ }],+ },+ LifecycleState {+ name: "delete".to_string(),+ actions: vec![LifecycleAction::Delete(EmptyParams {})],+ transitions: vec![],+ },+ ],+ };+ assert!(policy.validate().is_ok());+ }++ #[test]+ fn action_json_round_trips_expected_shape() {+ let action = LifecycleAction::Rollover(RolloverAction {+ conditions: LifecycleConditions {+ min_doc_count: Some(100),+ ..Default::default()+ },+ });+ let v = serde_json::to_value(&action).unwrap();+ assert_eq!(v["rollover"]["min_doc_count"], 100);++ let delete = LifecycleAction::Delete(EmptyParams {});+ let v = serde_json::to_value(&delete).unwrap();+ assert_eq!(v, json!({"delete": {}}));++ let ro = LifecycleAction::ReadOnly(EmptyParams {});+ let v = serde_json::to_value(&ro).unwrap();+ assert_eq!(v, json!({"read_only": {}}));+ }++ #[test]+ fn translate_ilm_maps_phases_to_states_in_order() {+ let ilm = json!({+ "policy": {+ "phases": {+ "hot": {+ "min_age": "0ms",+ "actions": { "rollover": { "max_size": "50gb", "max_docs": 100 } }+ },+ "delete": {+ "min_age": "30d",+ "actions": { "delete": {} }+ }+ }+ }+ });+ let policy = translate_ilm_policy(&ilm).unwrap();+ assert_eq!(policy.default_state, "hot");+ assert_eq!(policy.states.len(), 2);+ assert_eq!(policy.states[0].name, "hot");+ assert_eq!(policy.states[0].transitions.len(), 1);+ assert_eq!(policy.states[0].transitions[0].state_name, "delete");+ assert_eq!(+ policy.states[0].transitions[0]+ .conditions+ .as_ref()+ .unwrap()+ .min_index_age,+ Some("30d".to_string())+ );+ assert_eq!(policy.states[1].name, "delete");+ assert!(policy.states[1].transitions.is_empty());+ match &policy.states[0].actions[0] {+ LifecycleAction::Rollover(r) => {+ assert_eq!(r.conditions.min_size, Some("50gb".to_string()));+ assert_eq!(r.conditions.min_doc_count, Some(100));+ }+ other => panic!("expected rollover action, got {other:?}"),+ }+ }++ #[test]+ fn translate_ilm_skips_unmapped_actions_errors_on_unknown() {+ let ilm = json!({+ "policy": {+ "phases": {+ "hot": {+ "actions": { "set_priority": { "priority": 100 } }+ }+ }+ }+ });+ let policy = translate_ilm_policy(&ilm).unwrap();+ assert!(policy.states[0].actions.is_empty());++ let ilm_bad = json!({+ "policy": { "phases": { "hot": { "actions": { "totally_made_up": {} } } } }+ });+ assert!(translate_ilm_policy(&ilm_bad).is_err());+ }+}diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex baf98e0ab..22bac8301 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -1801,6 +1801,11 @@ async fn async_main() -> Result<()> {// a 429 circuit_breaking_exception before the kernel OOM-kills us.state.engine.spawn_resource_sampler();+ // 12c. ISM/ILM lifecycle-management background job: drives every+ // managed index's state machine (actions, then transitions) on+ // `lifecycle.tick_interval_secs` (default 5 minutes).+ state.engine.spawn_lifecycle_manager();+let _ingest_memory_trace = ingest_memory_trace::spawn(&state.engine);// 13. Start servers concurrentlylet rest_tls = tls_config.clone();