Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(engine): real ISM/ILM index lifecycle management — one engine, two REST surfaces

Vincenzo Lombardod34803ea5a97
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 1ac656576..29c8a1f74 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -832,6 +832,17 @@ pub async fn create_index(
.insert(index.clone(), Value::Object(resolved_aliases));
}
+ // ES-shape lifecycle attach: `index.lifecycle.name` in create-time
+ // settings is how ILM associates an index with a policy (there is
+ // no dedicated "attach" endpoint on the ES surface, unlike ISM's
+ // explicit `POST _plugins/_ism/add/{index}`). Best-effort and
+ // silent on a missing/untranslatable policy — real ILM is
+ // equally lenient here: it discovers the policy by name lazily
+ // and simply doesn't start managing until one exists.
+ if let Some(policy_name) = ilm_policy_name_from_settings(&body) {
+ let _ = crate::ism_api::attach_policy(&state, &index, &policy_name).await;
+ }
+
let resp = EsIndexResponse::ok(&index);
(StatusCode::OK, Json(resp)).into_response()
}
@@ -20910,9 +20921,33 @@ pub async fn put_settings(
state.engine.index_settings.insert(idx.clone(), existing);
}
+ // Same lazy, best-effort `index.lifecycle.name` attach as create_index —
+ // `PUT {index}/_settings {"index.lifecycle.name": "policy"}` is the
+ // other real ILM entry point (attaching to an index that already
+ // exists, not just at creation).
+ if let Some(policy_name) = ilm_policy_name_from_settings(&body) {
+ for idx in &targets {
+ let _ = crate::ism_api::attach_policy(&state, idx, &policy_name).await;
+ }
+ }
+
Json(json!({ "acknowledged": true })).into_response()
}
+/// Extract `index.lifecycle.name` from an index-settings body, accepting
+/// both the nested (`{"index":{"lifecycle":{"name":"..."}}}`,
+/// `{"settings":{...}}`-wrapped at create time) and flat dotted-key
+/// (`{"index.lifecycle.name":"..."}`) forms ES accepts for every index
+/// setting.
+fn ilm_policy_name_from_settings(body: &Value) -> Option<String> {
+ let settings = body.get("settings").unwrap_or(body);
+ settings
+ .pointer("/index/lifecycle/name")
+ .or_else(|| settings.get("index.lifecycle.name"))
+ .and_then(Value::as_str)
+ .map(String::from)
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Ingest Pipelines
// PUT /_ingest/pipeline/{id}
@@ -22722,10 +22757,34 @@ pub async fn put_ilm_policy(
Path(name): Path<String>,
Json(body): Json<Value>,
) -> impl IntoResponse {
- // Persist the policy in the real ILM store; `get_ilm_policy` reads it back
- // out of this same DashMap, so PUT then GET round-trips faithfully.
- state.engine.ilm_policies.insert(name, body);
- Json(json!({ "acknowledged": true })).into_response()
+ // Persist the raw policy verbatim in the ILM store; `get_ilm_policy`
+ // reads it back out of this same DashMap, so PUT then GET round-trips
+ // faithfully regardless of whether translation below succeeds.
+ state.engine.ilm_policies.insert(name.clone(), body.clone());
+
+ // Translate into the shared internal model (`xerj_engine::lifecycle`)
+ // so the SAME background job that drives native ISM policies also
+ // drives this one — see that module's doc comment for why. Real ILM
+ // is lenient about phase actions it can't reproduce internally too
+ // (e.g. `searchable_snapshot` is best-effort); mirroring that, a
+ // translation failure here doesn't fail the PUT — the policy is
+ // still stored and GET-able, it just won't actually execute (no entry
+ // lands in `ism_policies`) until it's fixed. Surfaced back in the
+ // response so this isn't silent.
+ match xerj_engine::lifecycle::translate_ilm_policy(&body) {
+ Ok(policy) => {
+ state.engine.put_ism_policy(name, policy);
+ Json(json!({ "acknowledged": true })).into_response()
+ }
+ Err(reason) => Json(json!({
+ "acknowledged": true,
+ "xerj_lifecycle_execution": {
+ "status": "unsupported",
+ "reason": reason,
+ }
+ }))
+ .into_response(),
+ }
}
pub async fn get_ilm_policy(
@@ -22756,6 +22815,7 @@ pub async fn delete_ilm_policy(
Path(name): Path<String>,
) -> impl IntoResponse {
if state.engine.ilm_policies.remove(&name).is_some() {
+ state.engine.remove_ism_policy(&name);
Json(json!({ "acknowledged": true })).into_response()
} else {
let e = xerj_common::XerjError::index_not_found(format!("policy [{name}] not found"));
@@ -22763,6 +22823,39 @@ pub async fn delete_ilm_policy(
}
}
+// ─────────────────────────────────────────────────────────────────────────────
+// GET /{index}/_ilm/explain
+// ─────────────────────────────────────────────────────────────────────────────
+
+pub async fn ilm_explain(
+ State(state): State<AppState>,
+ Path(index): Path<String>,
+) -> impl IntoResponse {
+ let mut indices = serde_json::Map::new();
+ match state.engine.managed_indices.get(&index) {
+ Some(managed) => {
+ indices.insert(
+ index.clone(),
+ json!({
+ "index": index,
+ "managed": true,
+ "policy": managed.policy_id,
+ "phase": managed.current_state,
+ "phase_time_millis": managed.state_entered_at_ms,
+ "action": managed.info_message,
+ "step": managed.info_message,
+ "step_info": { "message": managed.info_message },
+ "failed_step": if managed.failed { Some(managed.info_message.clone()) } else { None },
+ }),
+ );
+ }
+ None => {
+ indices.insert(index.clone(), json!({ "index": index, "managed": false }));
+ }
+ }
+ Json(json!({ "indices": Value::Object(indices) })).into_response()
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Component Templates
// PUT /_component_template/{name}
diff --git a/engine/crates/xerj-api/src/ism_api.rs b/engine/crates/xerj-api/src/ism_api.rs
new file mode 100644
index 000000000..f7062e2fa
--- /dev/null
+++ b/engine/crates/xerj-api/src/ism_api.rs
@@ -0,0 +1,272 @@
+//! OpenSearch Index State Management (ISM) — the native REST surface over
+//! `xerj_engine::lifecycle`'s execution engine. See that module's doc
+//! comment for why ISM's state machine (not ES ILM's fixed phases) is the
+//! internal model both `_plugins/_ism/*` (here) and `_ilm/*`
+//! (`es_compat::put_ilm_policy` and friends) are built on.
+//!
+//! Endpoints:
+//! PUT/GET/DELETE /_plugins/_ism/policies/{policy_id}
+//! GET /_plugins/_ism/policies (list/search — added
+//! after live OSD UI verification showed the "State
+//! management policies" table 500ing without it)
+//! POST /_plugins/_ism/add/{index}
+//! GET /_plugins/_ism/explain/{index}
+//! GET /_plugins/_ism/explain (list-all — same
+//! reason, for the "Policy managed indexes" table)
+
+use axum::{
+ extract::{Path, Query, State},
+ response::{IntoResponse, Response},
+ Json,
+};
+use serde::Deserialize;
+use serde_json::{json, Value};
+
+use xerj_engine::lifecycle::{self, LifecyclePolicy, ManagedIndexState};
+
+use crate::{error::ApiError, state::AppState};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// PUT/GET/DELETE /_plugins/_ism/policies/{policy_id}
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Real ISM wraps the policy body as `{"policy": {...}}`; accept that shape
+/// but also the bare policy object, since it's an easy, harmless mistake to
+/// make by hand (and this parses unambiguously either way).
+fn unwrap_policy_body(body: &Value) -> &Value {
+ body.get("policy").unwrap_or(body)
+}
+
+pub async fn put_ism_policy(
+ State(state): State<AppState>,
+ Path(policy_id): Path<String>,
+ Json(body): Json<Value>,
+) -> impl IntoResponse {
+ let policy_value = unwrap_policy_body(&body).clone();
+ let policy: LifecyclePolicy = match serde_json::from_value(policy_value) {
+ Ok(p) => p,
+ Err(e) => {
+ let err =
+ xerj_common::XerjError::invalid_query(format!("invalid ISM policy body: {e}"));
+ return ApiError::new(err).into_response();
+ }
+ };
+ if let Err(reason) = policy.validate() {
+ let err = xerj_common::XerjError::invalid_query(format!("invalid ISM policy: {reason}"));
+ return ApiError::new(err).into_response();
+ }
+ state
+ .engine
+ .put_ism_policy(policy_id.clone(), policy.clone());
+ Json(json!({
+ "_id": policy_id,
+ "_version": 1,
+ "_seq_no": 0,
+ "_primary_term": 1,
+ "policy": policy,
+ }))
+ .into_response()
+}
+
+pub async fn get_ism_policy(
+ State(state): State<AppState>,
+ Path(policy_id): Path<String>,
+) -> impl IntoResponse {
+ match state.engine.ism_policies.get(&policy_id) {
+ Some(policy) => Json(json!({
+ "_id": policy_id,
+ "_version": 1,
+ "_seq_no": 0,
+ "_primary_term": 1,
+ "policy": policy.value(),
+ }))
+ .into_response(),
+ None => {
+ let e =
+ xerj_common::XerjError::index_not_found(format!("policy [{policy_id}] not found"));
+ ApiError::new(e).into_response()
+ }
+ }
+}
+
+#[derive(Debug, Deserialize, Default)]
+pub struct ListParams {
+ #[serde(default)]
+ pub from: usize,
+ #[serde(default = "default_size")]
+ pub size: usize,
+ #[serde(default)]
+ pub search: String,
+ #[serde(default)]
+ pub sort_direction: Option<String>,
+}
+fn default_size() -> usize {
+ 20
+}
+
+/// `GET /_plugins/_ism/policies` — the list/search form OSD's own Index
+/// Management UI uses to populate the "State management policies" table
+/// (its backend proxies `/api/ism/policies` straight through to this
+/// endpoint). Found missing while verifying the real UI: only the
+/// single-policy `GET .../policies/{id}` was implemented, so the list page
+/// rendered "no existing policies" and a raw Internal Server Error toast
+/// even immediately after a successful create.
+pub async fn list_ism_policies(
+ State(state): State<AppState>,
+ Query(params): Query<ListParams>,
+) -> impl IntoResponse {
+ let mut all: Vec<(String, LifecyclePolicy)> = state
+ .engine
+ .ism_policies
+ .iter()
+ .map(|e| (e.key().clone(), e.value().clone()))
+ .filter(|(id, _)| params.search.is_empty() || id.contains(&params.search))
+ .collect();
+ all.sort_by(|a, b| a.0.cmp(&b.0));
+ if params.sort_direction.as_deref() == Some("desc") {
+ all.reverse();
+ }
+ let total = all.len();
+ let page: Vec<Value> = all
+ .into_iter()
+ .skip(params.from)
+ .take(params.size.max(1))
+ .map(|(id, policy)| {
+ json!({
+ "_id": id,
+ "_version": 1,
+ "_seq_no": 0,
+ "_primary_term": 1,
+ "policy": policy,
+ })
+ })
+ .collect();
+ Json(json!({ "totalPolicies": total, "policies": page })).into_response()
+}
+
+pub async fn delete_ism_policy(
+ State(state): State<AppState>,
+ Path(policy_id): Path<String>,
+) -> impl IntoResponse {
+ if state.engine.remove_ism_policy(&policy_id) {
+ Json(json!({ "_id": policy_id, "result": "deleted" })).into_response()
+ } else {
+ let e = xerj_common::XerjError::index_not_found(format!("policy [{policy_id}] not found"));
+ ApiError::new(e).into_response()
+ }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// POST /_plugins/_ism/add/{index}
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[derive(Debug, Deserialize)]
+pub struct AddPolicyBody {
+ pub policy_id: String,
+}
+
+pub async fn add_ism_policy(
+ State(state): State<AppState>,
+ Path(index): Path<String>,
+ Json(body): Json<AddPolicyBody>,
+) -> impl IntoResponse {
+ attach_policy(&state, &index, &body.policy_id).await
+}
+
+/// Shared attach implementation: validates the target index and policy both
+/// exist, then creates (or resets) that index's managed-index execution
+/// cursor at the policy's `default_state`. Used by the native ISM endpoint
+/// above and by the ES-shape `index.lifecycle.name` auto-attach in
+/// `es_compat` (a single index↔policy association model, per the design —
+/// not two).
+pub async fn attach_policy(state: &AppState, index: &str, policy_id: &str) -> Response {
+ if state.engine.get_index(index).is_err() {
+ let e = xerj_common::XerjError::index_not_found(index);
+ return ApiError::new(e).into_response();
+ }
+ let Some(policy) = state
+ .engine
+ .ism_policies
+ .get(policy_id)
+ .map(|e| e.value().clone())
+ else {
+ let e = xerj_common::XerjError::index_not_found(format!("policy [{policy_id}] not found"));
+ return ApiError::new(e).into_response();
+ };
+ let managed = ManagedIndexState::new(
+ policy_id.to_string(),
+ policy.default_state.clone(),
+ lifecycle::now_ms(),
+ );
+ state
+ .engine
+ .managed_indices
+ .insert(index.to_string(), managed);
+ state.engine.persist_managed_indices();
+
+ Json(json!({
+ "failures": false,
+ "updated_indices": 1,
+ "failed_indices": [],
+ }))
+ .into_response()
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// GET /_plugins/_ism/explain/{index}
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// `GET /_plugins/_ism/explain` — the list-all form OSD's "Policy managed
+/// indexes" table uses (its backend's `/api/ism/managedIndices` proxies
+/// straight through to this). Same gap as `list_ism_policies`: only the
+/// single-index `.../explain/{index}` was implemented at first.
+pub async fn list_managed_indices(
+ State(state): State<AppState>,
+ Query(params): Query<ListParams>,
+) -> impl IntoResponse {
+ let mut all: Vec<(String, ManagedIndexState)> = state
+ .engine
+ .managed_indices
+ .iter()
+ .map(|e| (e.key().clone(), e.value().clone()))
+ .filter(|(name, _)| params.search.is_empty() || name.contains(&params.search))
+ .collect();
+ all.sort_by(|a, b| a.0.cmp(&b.0));
+ if params.sort_direction.as_deref() == Some("desc") {
+ all.reverse();
+ }
+ let total = all.len();
+ let mut body = serde_json::Map::new();
+ for (name, managed) in all.into_iter().skip(params.from).take(params.size.max(1)) {
+ body.insert(name.clone(), lifecycle::explain_json(&name, &managed));
+ }
+ body.insert("total_managed_indices".to_string(), json!(total));
+ Json(Value::Object(body)).into_response()
+}
+
+pub async fn explain_ism_index(
+ State(state): State<AppState>,
+ Path(index): Path<String>,
+) -> impl IntoResponse {
+ match state.engine.managed_indices.get(&index) {
+ Some(managed) => {
+ let mut body = serde_json::Map::new();
+ body.insert(
+ index.clone(),
+ lifecycle::explain_json(&index, managed.value()),
+ );
+ body.insert("total_managed_indices".to_string(), json!(1));
+ Json(Value::Object(body)).into_response()
+ }
+ None => {
+ // Real ISM answers 200 with a "not managed" note rather than
+ // 404 — `explain` on an unmanaged (or nonexistent) index is a
+ // normal, expected call from Dashboards' Managed Indexes view.
+ Json(json!({
+ index.clone(): { "index": index, "enabled": false },
+ "total_managed_indices": 0,
+ }))
+ .into_response()
+ }
+ }
+}
diff --git a/engine/crates/xerj-api/src/lib.rs b/engine/crates/xerj-api/src/lib.rs
index 876ba358e..0e63816d3 100644
--- a/engine/crates/xerj-api/src/lib.rs
+++ b/engine/crates/xerj-api/src/lib.rs
@@ -63,6 +63,7 @@ pub mod error;
pub mod es_compat;
pub mod extract;
pub mod graph_api;
+pub mod ism_api;
pub mod memory_api;
pub mod native;
pub mod responses;
diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rs
index a91c4b7bf..f26ac4b1d 100644
--- a/engine/crates/xerj-api/src/router.rs
+++ b/engine/crates/xerj-api/src/router.rs
@@ -28,7 +28,8 @@ use uuid::Uuid;
use xerj_common::config::CorsConfig;
use crate::{
- auth::auth_middleware, authz, es_compat, graph_api, memory_api, native, state::AppState,
+ auth::auth_middleware, authz, es_compat, graph_api, ism_api, memory_api, native,
+ state::AppState,
};
// ─────────────────────────────────────────────────────────────────────────────
@@ -454,6 +455,22 @@ pub fn build_es_compat_router(state: AppState) -> Router {
.get(es_compat::get_ilm_policy)
.delete(es_compat::delete_ilm_policy),
)
+ .route("/:index/_ilm/explain", get(es_compat::ilm_explain))
+ // ── ISM (OpenSearch Index State Management) ─────────────────────────
+ // Same execution engine as ILM above — see `xerj_engine::lifecycle`.
+ .route(
+ "/_plugins/_ism/policies/:policy_id",
+ put(ism_api::put_ism_policy)
+ .get(ism_api::get_ism_policy)
+ .delete(ism_api::delete_ism_policy),
+ )
+ .route("/_plugins/_ism/policies", get(ism_api::list_ism_policies))
+ .route("/_plugins/_ism/add/:index", post(ism_api::add_ism_policy))
+ .route(
+ "/_plugins/_ism/explain/:index",
+ get(ism_api::explain_ism_index),
+ )
+ .route("/_plugins/_ism/explain", get(ism_api::list_managed_indices))
// ── Component templates ────────────────────────────────────────────
.route(
"/_component_template/:name",
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index 06f8616e7..418940a30 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -84,9 +84,11 @@ pub struct Config {
pub logging: LoggingConfig,
/// Elasticsearch/OpenSearch wire-compatibility identity — 2 settings.
pub compat: CompatConfig,
+ /// ISM/ILM index-lifecycle-management background execution — 1 setting.
+ pub lifecycle: LifecycleConfig,
}
-// Total: 5+3+2+3+10+5+3+1+6+2+4+3+4+3+2 = 56 fields (incl. cors: 2, auth: 3,
+// Total: 5+3+2+3+10+5+3+1+6+2+4+3+4+3+2+1 = 57 fields (incl. cors: 2, auth: 3,
// logging: 2). `Default` is derived — every field is a sub-config that
// implements `Default`, so the derive produces exactly the same all-defaults
// value the manual impl used to build by hand.
@@ -1315,6 +1317,28 @@ impl Default for PitConfig {
}
}
+/// ISM/ILM index-lifecycle-management background execution.
+///
+/// Drives `xerj_engine::lifecycle`'s tick: for every managed index, run the
+/// current state's pending actions, then evaluate transitions in order and
+/// move to the first one whose conditions are met.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct LifecycleConfig {
+ /// How often the background job ticks (default: 300 seconds = 5
+ /// minutes — the same default OpenSearch ISM itself uses for
+ /// `plugins.index_state_management.job_interval`).
+ pub tick_interval_secs: u64,
+}
+
+impl Default for LifecycleConfig {
+ fn default() -> Self {
+ Self {
+ tick_interval_secs: 300,
+ }
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
/// Scroll + async-search context lifecycle settings (RC4 blocker 11).
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index fd51d2346..29ef3de29 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -272,8 +272,25 @@ pub struct Engine {
pub closed_indices: Arc<DashMap<String, bool>>,
/// data stream name → DataStream
pub data_streams: Arc<DashMap<String, DataStream>>,
- /// ILM policy name → policy JSON
+ /// ILM policy name → raw policy JSON, kept verbatim for `GET _ilm/policy`
+ /// round-trip fidelity. The REAL execution engine reads `ism_policies`
+ /// instead — `PUT _ilm/policy/{name}` writes to BOTH maps (see
+ /// `xerj_api::es_compat::put_ilm_policy`), translating into the shared
+ /// internal model. See `crate::lifecycle` for why one engine drives both
+ /// surfaces.
pub ilm_policies: Arc<DashMap<String, Value>>,
+ /// policy id → the internal ISM-shaped model every managed index is
+ /// actually driven by, regardless of whether the policy was created via
+ /// `_plugins/_ism/policies` (native) or `_ilm/policy` (translated).
+ pub ism_policies: Arc<DashMap<String, crate::lifecycle::LifecyclePolicy>>,
+ /// index name → lifecycle execution cursor (current state, pending
+ /// action, timestamps). Presence in this map is what "managed" means —
+ /// an index with no entry here is not touched by the background job.
+ pub managed_indices: Arc<DashMap<String, crate::lifecycle::ManagedIndexState>>,
+ /// index name → creation time (epoch ms), for `min_index_age` and for
+ /// `GET /{index}`'s `creation_date` (previously synthesized as
+ /// `Utc::now()` on every request — see `record_index_created_at`).
+ pub index_created_at: Arc<DashMap<String, i64>>,
/// component template name → template JSON
pub component_templates: Arc<DashMap<String, Value>>,
/// snapshot repository name → repo config JSON
@@ -430,6 +447,9 @@ impl Engine {
closed_indices: Arc::new(DashMap::new()),
data_streams: Arc::new(DashMap::new()),
ilm_policies: Arc::new(DashMap::new()),
+ ism_policies: Arc::new(DashMap::new()),
+ managed_indices: Arc::new(DashMap::new()),
+ index_created_at: Arc::new(DashMap::new()),
component_templates: Arc::new(DashMap::new()),
snapshot_repos: Arc::new(DashMap::new()),
snapshots: Arc::new(DashMap::new()),
@@ -506,6 +526,7 @@ impl Engine {
if let Some(m) = engine.index_mappings.get(name_str.as_str()) {
Engine::apply_date_mapping_flags(&idx, m.value());
}
+ engine.load_or_backfill_index_created_at(&name_str);
engine.indices.insert(name_str, idx);
}
Err(e) => {
@@ -529,6 +550,12 @@ impl Engine {
// migration by another instance).
engine.load_persisted_aliases();
+ // Restore ISM/ILM policies and managed-index execution state so a
+ // policy attached before a restart keeps running afterward instead
+ // of silently going idle.
+ engine.load_persisted_ism_policies();
+ engine.load_persisted_managed_indices();
+
// Spawn the PIT sweeper. Pre-v0.6.2 PITs accumulated forever;
// every open without close was a memory leak. The sweeper
// walks `engine.pits` every `pit.sweep_interval_secs` and
@@ -834,6 +861,7 @@ impl Engine {
&self.data_dir,
)?;
self.indices.insert(name.to_string(), idx);
+ self.record_index_created_at(name);
info!(name, "index created with custom settings");
Ok(())
}
@@ -1098,6 +1126,184 @@ impl Engine {
}
}
+ // ── Lifecycle management (ISM/ILM) persistence ──────────────────────────
+
+ fn ism_policies_path(&self) -> PathBuf {
+ self.data_dir.join("ism_policies.json")
+ }
+
+ fn managed_indices_path(&self) -> PathBuf {
+ self.data_dir.join("ism_managed_indices.json")
+ }
+
+ /// Insert (or overwrite) an ISM-shaped policy and persist the whole
+ /// store, mirroring `flush_aliases`. Both `_plugins/_ism/policies` (as
+ /// entered) and `_ilm/policy` (after translation) write through here —
+ /// see the `ism_policies` field doc.
+ pub fn put_ism_policy(&self, id: String, policy: crate::lifecycle::LifecyclePolicy) {
+ self.ism_policies.insert(id, policy);
+ self.flush_ism_policies();
+ }
+
+ pub fn remove_ism_policy(&self, id: &str) -> bool {
+ let removed = self.ism_policies.remove(id).is_some();
+ if removed {
+ self.flush_ism_policies();
+ }
+ removed
+ }
+
+ fn flush_ism_policies(&self) {
+ let snapshot: std::collections::HashMap<String, crate::lifecycle::LifecyclePolicy> = self
+ .ism_policies
+ .iter()
+ .map(|e| (e.key().clone(), e.value().clone()))
+ .collect();
+ let bytes = match serde_json::to_vec_pretty(&snapshot) {
+ Ok(b) => b,
+ Err(e) => {
+ warn!(error = %e, "failed to serialize ism_policies for persistence");
+ return;
+ }
+ };
+ if let Err(e) = crate::index::write_file_atomic(&self.ism_policies_path(), &bytes) {
+ warn!(error = %e, "failed to persist ism_policies.json (policies work until restart)");
+ }
+ }
+
+ fn load_persisted_ism_policies(&self) {
+ let Ok(bytes) = std::fs::read(self.ism_policies_path()) else {
+ return;
+ };
+ match serde_json::from_slice::<
+ std::collections::HashMap<String, crate::lifecycle::LifecyclePolicy>,
+ >(&bytes)
+ {
+ Ok(map) => {
+ let n = map.len();
+ for (id, policy) in map {
+ self.ism_policies.insert(id, policy);
+ }
+ if n > 0 {
+ info!(count = n, "restored persisted ISM/ILM policies");
+ }
+ }
+ Err(e) => warn!(error = %e, "ignoring corrupt ism_policies.json"),
+ }
+ }
+
+ /// Persist the whole `managed_indices` map. Called by `lifecycle::tick`
+ /// after any change, and by the attach/detach handlers.
+ pub fn persist_managed_indices(&self) {
+ let snapshot: std::collections::HashMap<String, crate::lifecycle::ManagedIndexState> = self
+ .managed_indices
+ .iter()
+ .map(|e| (e.key().clone(), e.value().clone()))
+ .collect();
+ let bytes = match serde_json::to_vec_pretty(&snapshot) {
+ Ok(b) => b,
+ Err(e) => {
+ warn!(error = %e, "failed to serialize managed_indices for persistence");
+ return;
+ }
+ };
+ if let Err(e) = crate::index::write_file_atomic(&self.managed_indices_path(), &bytes) {
+ warn!(error = %e, "failed to persist ism_managed_indices.json (state works until restart)");
+ }
+ }
+
+ fn load_persisted_managed_indices(&self) {
+ let Ok(bytes) = std::fs::read(self.managed_indices_path()) else {
+ return;
+ };
+ match serde_json::from_slice::<
+ std::collections::HashMap<String, crate::lifecycle::ManagedIndexState>,
+ >(&bytes)
+ {
+ Ok(map) => {
+ let n = map.len();
+ for (index_name, state) in map {
+ self.managed_indices.insert(index_name, state);
+ }
+ if n > 0 {
+ info!(count = n, "restored persisted ISM managed-index state");
+ }
+ }
+ Err(e) => warn!(error = %e, "ignoring corrupt ism_managed_indices.json"),
+ }
+ }
+
+ /// Spawn the background lifecycle-execution job: every
+ /// `config.lifecycle.tick_interval_secs` (default 300s = 5 minutes,
+ /// matching OpenSearch ISM's own default job interval), runs
+ /// `lifecycle::tick` over every managed index. Not latency-critical
+ /// (unlike the resource governor's sampler), so a plain `tokio::spawn`
+ /// on an interval is fine — uses a `Weak` self-pointer so it exits
+ /// cleanly when the engine is dropped (matters for tests, which create
+ /// and drop many `Engine`s).
+ pub fn spawn_lifecycle_manager(self: &Arc<Self>) {
+ let interval =
+ std::time::Duration::from_secs(self.config.lifecycle.tick_interval_secs.max(1));
+ let weak = Arc::downgrade(self);
+ tokio::spawn(async move {
+ let mut tick = tokio::time::interval(interval);
+ tick.tick().await; // skip the immediate first tick
+ loop {
+ tick.tick().await;
+ let Some(engine) = weak.upgrade() else {
+ return; // engine dropped
+ };
+ crate::lifecycle::tick(&engine).await;
+ }
+ });
+ }
+
+ // ── Index creation-date tracking ─────────────────────────────────────────
+
+ fn index_created_at_path(&self, name: &str) -> PathBuf {
+ self.data_dir.join(name).join("created_at.json")
+ }
+
+ /// Record `name`'s creation time (now) and persist it — called exactly
+ /// once, at the end of `create_index_with_settings`. Without this,
+ /// `min_index_age` has no ground truth, and `GET /{index}`'s
+ /// `creation_date` was synthesized fresh on every single request
+ /// (always reporting "now"), which is what this replaces.
+ fn record_index_created_at(&self, name: &str) {
+ let now = crate::lifecycle::now_ms();
+ self.index_created_at.insert(name.to_string(), now);
+ match serde_json::to_vec_pretty(&serde_json::json!({ "creation_date_ms": now })) {
+ Ok(bytes) => {
+ if let Err(e) =
+ crate::index::write_file_atomic(&self.index_created_at_path(name), &bytes)
+ {
+ warn!(index = name, error = %e, "failed to persist created_at.json");
+ }
+ }
+ Err(e) => warn!(index = name, error = %e, "failed to serialize created_at.json"),
+ }
+ }
+
+ /// Load `name`'s persisted creation time on startup. An index directory
+ /// that predates this feature (no `created_at.json`) gets a synthetic
+ /// baseline of "now" recorded and persisted on first post-upgrade boot —
+ /// its true history is unknown, so `min_index_age` starts counting from
+ /// the upgrade rather than from the index's real (unrecorded) creation.
+ /// Documented tradeoff, not silently wrong: every index that existed
+ /// before this shipped gets exactly one synthetic reset, once.
+ fn load_or_backfill_index_created_at(&self, name: &str) {
+ let path = self.index_created_at_path(name);
+ if let Ok(bytes) = std::fs::read(&path) {
+ if let Ok(v) = serde_json::from_slice::<Value>(&bytes) {
+ if let Some(ms) = v.get("creation_date_ms").and_then(Value::as_i64) {
+ self.index_created_at.insert(name.to_string(), ms);
+ return;
+ }
+ }
+ }
+ self.record_index_created_at(name);
+ }
+
/// Add an alias pointing to an index.
pub fn add_alias(&self, alias: &str, index: &str) {
let mut entry = self.aliases.entry(alias.to_string()).or_default();
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 1e07ea989..60fadf50f 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -25,6 +25,7 @@ pub mod governor;
pub mod index;
pub mod index_guard;
pub mod ingest_memory;
+pub mod lifecycle;
pub mod memtable;
pub mod painless;
pub mod rbac;
diff --git a/engine/crates/xerj-engine/src/lifecycle.rs b/engine/crates/xerj-engine/src/lifecycle.rs
new file mode 100644
index 000000000..d1a75de31
--- /dev/null
+++ b/engine/crates/xerj-engine/src/lifecycle.rs
@@ -0,0 +1,848 @@
+//! Index lifecycle management — one execution engine, two REST surfaces.
+//!
+//! Modeled on OpenSearch's Index State Management (ISM), not Elasticsearch's
+//! fixed hot/warm/cold/delete phases: ISM's state machine (named states,
+//! each with its own ordered actions and ordered transitions with
+//! conditions) is strictly more general, so it's the internal model.
+//! Elasticsearch's `_ilm/*` surface is a translator into this same model
+//! (see [`translate_ilm_policy`]), not a second engine — an ILM policy with
+//! hot/warm/cold/delete phases becomes an ISM policy whose states are named
+//! `"hot"`/`"warm"`/`"cold"`/`"delete"` in that fixed sequence.
+//!
+//! ## Known simplification vs. real ILM
+//!
+//! Real Elasticsearch measures a phase's `min_age` from the index's
+//! rollover time (not its creation time) once the policy's hot phase has a
+//! `rollover` action — the age clock effectively restarts at rollover. This
+//! engine measures every state's age from when the index *entered that
+//! state* (`state_entered_at_ms`), which is index-attach time for the
+//! initial state, not rollover time. For a policy without `rollover`, or
+//! for reasoning about relative phase durations, this is equivalent; for a
+//! policy that rolls over AND has a downstream phase whose `min_age` is
+//! meant to be measured strictly from rollover, this engine will advance
+//! phases slightly earlier than real ILM would. Documented rather than
+//! silently diverging — see PR discussion for whether this needs closing.
+//!
+//! ## What's NOT supported, and errors honestly instead of faking success
+//!
+//! `replica_count`: XERJ has no per-index configurable replica concept
+//! (single-shard, single-node engine; `number_of_replicas` is accepted in
+//! index settings and echoed back for wire compatibility but never
+//! consumed). The action fails with an explicit error rather than silently
+//! no-op'ing, so a policy relying on it is visibly stuck, not silently
+//! wrong.
+
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Value};
+
+use crate::engine::Engine;
+use crate::{EngineError, Result};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Data model — faithful to OpenSearch ISM's policy JSON shape.
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LifecyclePolicy {
+ #[serde(default)]
+ pub description: Option<String>,
+ pub default_state: String,
+ pub states: Vec<LifecycleState>,
+}
+
+impl LifecyclePolicy {
+ /// Structural validation beyond what serde already enforces: every
+ /// transition target and `default_state` must name a real state, and
+ /// state names must be unique. Real ISM rejects a policy with a
+ /// dangling transition at PUT time rather than discovering it mid-run.
+ pub fn validate(&self) -> std::result::Result<(), String> {
+ if self.states.is_empty() {
+ return Err("policy must declare at least one state".to_string());
+ }
+ let names: std::collections::HashSet<&str> =
+ self.states.iter().map(|s| s.name.as_str()).collect();
+ if names.len() != self.states.len() {
+ return Err("state names must be unique".to_string());
+ }
+ if !names.contains(self.default_state.as_str()) {
+ return Err(format!(
+ "default_state '{}' does not name a declared state",
+ self.default_state
+ ));
+ }
+ for state in &self.states {
+ for t in &state.transitions {
+ if !names.contains(t.state_name.as_str()) {
+ return Err(format!(
+ "state '{}' transitions to undeclared state '{}'",
+ state.name, t.state_name
+ ));
+ }
+ }
+ }
+ Ok(())
+ }
+
+ pub fn state(&self, name: &str) -> Option<&LifecycleState> {
+ self.states.iter().find(|s| s.name == name)
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LifecycleState {
+ pub name: String,
+ #[serde(default)]
+ pub actions: Vec<LifecycleAction>,
+ #[serde(default)]
+ pub transitions: Vec<LifecycleTransition>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LifecycleTransition {
+ pub state_name: String,
+ #[serde(default)]
+ pub conditions: Option<LifecycleConditions>,
+}
+
+/// Conditions shared by transitions (when to leave a state) and the
+/// `rollover` action (when it's actually worth rolling over) — same shape,
+/// same evaluator, two different call sites. A transition/action with no
+/// conditions at all is always eligible the moment it's checked (an
+/// unconditional transition; an unconditional rollover).
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct LifecycleConditions {
+ /// ES/ISM duration string: digits followed by `ms|s|m|h|d`. E.g. `"7d"`.
+ #[serde(default)]
+ pub min_index_age: Option<String>,
+ /// ES/ISM size string: digits followed by `b|kb|mb|gb|tb`. E.g. `"50gb"`.
+ #[serde(default)]
+ pub min_size: Option<String>,
+ #[serde(default)]
+ pub min_doc_count: Option<u64>,
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct EmptyParams {}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct RolloverAction {
+ #[serde(flatten)]
+ pub conditions: LifecycleConditions,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ReplicaCountAction {
+ pub number_of_replicas: u64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum LifecycleAction {
+ Rollover(RolloverAction),
+ Delete(EmptyParams),
+ ReadOnly(EmptyParams),
+ ReplicaCount(ReplicaCountAction),
+}
+
+impl LifecycleAction {
+ pub fn name(&self) -> &'static str {
+ match self {
+ LifecycleAction::Rollover(_) => "rollover",
+ LifecycleAction::Delete(_) => "delete",
+ LifecycleAction::ReadOnly(_) => "read_only",
+ LifecycleAction::ReplicaCount(_) => "replica_count",
+ }
+ }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Managed-index state — the per-index execution cursor.
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ManagedIndexState {
+ pub policy_id: String,
+ pub current_state: String,
+ pub state_entered_at_ms: i64,
+ /// Index into the current state's `actions[]` — which one runs next.
+ /// Equal to `actions.len()` once every action in this state has
+ /// completed and only transition-evaluation remains.
+ #[serde(default)]
+ pub next_action_index: usize,
+ pub last_updated_ms: i64,
+ /// Mirrors ISM explain's free-form `info.message`.
+ #[serde(default)]
+ pub info_message: String,
+ /// Set when a step errors, so `explain` surfaces it instead of the
+ /// index silently retrying the same failing action forever unseen.
+ /// Retries still happen (a transient failure — e.g. disk contention —
+ /// should self-heal); this is visibility, not a halt.
+ #[serde(default)]
+ pub failed: bool,
+}
+
+impl ManagedIndexState {
+ pub fn new(policy_id: String, default_state: String, now_ms: i64) -> Self {
+ Self {
+ policy_id,
+ current_state: default_state,
+ state_entered_at_ms: now_ms,
+ next_action_index: 0,
+ last_updated_ms: now_ms,
+ info_message: "attached, awaiting first execution".to_string(),
+ failed: false,
+ }
+ }
+}
+
+pub fn now_ms() -> i64 {
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map(|d| d.as_millis() as i64)
+ .unwrap_or(0)
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Duration / size string parsing — ES/ISM shorthand.
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Parse an ES/ISM duration string (`"7d"`, `"30m"`, `"500ms"`) into
+/// milliseconds. Unlike ES's full duration grammar this accepts exactly one
+/// integer + one unit, which is all `min_index_age` ever uses in practice.
+pub fn parse_duration_ms(s: &str) -> std::result::Result<i64, String> {
+ let s = s.trim();
+ let split_at = s
+ .find(|c: char| !c.is_ascii_digit())
+ .ok_or_else(|| format!("duration '{s}' has no unit"))?;
+ let (digits, unit) = s.split_at(split_at);
+ let n: i64 = digits
+ .parse()
+ .map_err(|_| format!("duration '{s}' has no numeric prefix"))?;
+ let ms = match unit {
+ "ms" => n,
+ "s" => n * 1_000,
+ "m" => n * 60_000,
+ "h" => n * 3_600_000,
+ "d" => n * 86_400_000,
+ other => return Err(format!("unknown duration unit '{other}' in '{s}'")),
+ };
+ Ok(ms)
+}
+
+/// Parse an ES/ISM size string (`"50gb"`, `"100mb"`) into bytes. Binary
+/// (1024-based) units, matching ES's own `ByteSizeValue`.
+pub fn parse_size_bytes(s: &str) -> std::result::Result<u64, String> {
+ let s = s.trim().to_ascii_lowercase();
+ let split_at = s
+ .find(|c: char| !c.is_ascii_digit())
+ .ok_or_else(|| format!("size '{s}' has no unit"))?;
+ let (digits, unit) = s.split_at(split_at);
+ let n: u64 = digits
+ .parse()
+ .map_err(|_| format!("size '{s}' has no numeric prefix"))?;
+ let bytes = match unit {
+ "b" => n,
+ "kb" => n * 1024,
+ "mb" => n * 1024 * 1024,
+ "gb" => n * 1024 * 1024 * 1024,
+ "tb" => n * 1024 * 1024 * 1024 * 1024,
+ other => return Err(format!("unknown size unit '{other}' in '{s}'")),
+ };
+ Ok(bytes)
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// ES ILM → internal model translator.
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Fixed ILM phase order — ISM states are named exactly this, one per phase
+/// actually present in the policy. A phase absent from the ILM policy
+/// produces no state at all (matching ILM, where an unconfigured phase is
+/// simply skipped).
+const ILM_PHASE_ORDER: [&str; 4] = ["hot", "warm", "cold", "delete"];
+
+/// Translate an ES ILM policy body (`{"policy": {"phases": {...}}}`, the
+/// same shape `PUT _ilm/policy/{name}` accepts) into a [`LifecyclePolicy`].
+///
+/// Each present phase becomes a state named after the phase. A phase's own
+/// `actions` (minus `min_age`, which isn't an action) become that state's
+/// actions. A phase's `min_age` becomes the CONDITION on the transition
+/// INTO that phase from the previous one — i.e. "warm: min_age 7d" means
+/// the state before warm gets a transition to warm gated on
+/// `min_index_age: 7d`. The final present phase gets no transitions
+/// (terminal), matching a real ISM policy's last state.
+pub fn translate_ilm_policy(ilm_body: &Value) -> std::result::Result<LifecyclePolicy, String> {
+ let phases = ilm_body
+ .pointer("/policy/phases")
+ .and_then(Value::as_object)
+ .ok_or_else(|| "ILM policy body missing policy.phases".to_string())?;
+
+ let present: Vec<&str> = ILM_PHASE_ORDER
+ .iter()
+ .copied()
+ .filter(|p| phases.contains_key(*p))
+ .collect();
+ if present.is_empty() {
+ return Err("ILM policy declares no known phase (hot/warm/cold/delete)".to_string());
+ }
+
+ let mut states = Vec::with_capacity(present.len());
+ for (i, phase_name) in present.iter().enumerate() {
+ let phase = &phases[*phase_name];
+ let actions = translate_ilm_phase_actions(phase_name, phase)?;
+
+ let transitions = if let Some(&next_phase) = present.get(i + 1) {
+ let next_min_age = phases[next_phase]
+ .get("min_age")
+ .and_then(Value::as_str)
+ .unwrap_or("0ms");
+ vec![LifecycleTransition {
+ state_name: next_phase.to_string(),
+ conditions: Some(LifecycleConditions {
+ min_index_age: Some(next_min_age.to_string()),
+ min_size: None,
+ min_doc_count: None,
+ }),
+ }]
+ } else {
+ Vec::new()
+ };
+
+ states.push(LifecycleState {
+ name: phase_name.to_string(),
+ actions,
+ transitions,
+ });
+ }
+
+ let policy = LifecyclePolicy {
+ description: ilm_body
+ .pointer("/policy/_meta/description")
+ .and_then(Value::as_str)
+ .map(String::from),
+ default_state: present[0].to_string(),
+ states,
+ };
+ policy.validate()?;
+ Ok(policy)
+}
+
+fn translate_ilm_phase_actions(
+ phase_name: &str,
+ phase: &Value,
+) -> std::result::Result<Vec<LifecycleAction>, String> {
+ let Some(actions_obj) = phase.get("actions").and_then(Value::as_object) else {
+ return Ok(Vec::new());
+ };
+ let mut out = Vec::new();
+ for (action_name, params) in actions_obj {
+ let action = match action_name.as_str() {
+ "rollover" => {
+ let conditions = LifecycleConditions {
+ min_index_age: params
+ .get("max_age")
+ .and_then(Value::as_str)
+ .map(String::from),
+ min_size: params
+ .get("max_size")
+ .and_then(Value::as_str)
+ .map(String::from),
+ min_doc_count: params.get("max_docs").and_then(Value::as_u64),
+ };
+ LifecycleAction::Rollover(RolloverAction { conditions })
+ }
+ "delete" => LifecycleAction::Delete(EmptyParams {}),
+ "readonly" => LifecycleAction::ReadOnly(EmptyParams {}),
+ "set_priority" | "allocate" | "unfollow" | "shrink" | "forcemerge" | "freeze" => {
+ // Real ILM actions with no XERJ equivalent (no shard
+ // allocation awareness, no shrink/forcemerge-as-a-lifecycle-
+ // step wiring, no CCR). Skipped rather than erroring the
+ // whole policy translation — matches this engine's existing
+ // "explicit error only where behavior would be silently
+ // wrong" policy: skipping a no-op-shaped action is honest
+ // (nothing was promised and not delivered), unlike
+ // replica_count which IS user-visible if silently ignored.
+ continue;
+ }
+ other => {
+ return Err(format!(
+ "phase '{phase_name}' action '{other}' has no ISM/XERJ equivalent"
+ ))
+ }
+ };
+ out.push(action);
+ }
+ Ok(out)
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Condition evaluation.
+// ─────────────────────────────────────────────────────────────────────────────
+
+struct IndexFacts {
+ age_ms: i64,
+ size_bytes: u64,
+ doc_count: u64,
+}
+
+async fn gather_index_facts(
+ engine: &Engine,
+ index_name: &str,
+ state_age_ms: i64,
+) -> Option<IndexFacts> {
+ let idx = engine.get_index(index_name).ok()?;
+ let stats = idx.stats().await;
+ let segment_bytes: u64 = idx
+ .store_snapshot()
+ .segments
+ .iter()
+ .map(|s| s.size_bytes)
+ .sum();
+ let size_bytes = segment_bytes + stats.memtable_size_bytes as u64;
+ Some(IndexFacts {
+ age_ms: state_age_ms,
+ size_bytes,
+ doc_count: stats.doc_count,
+ })
+}
+
+/// A `LifecycleConditions` with every field `None` matches immediately —
+/// this is what makes an unconditional transition / unconditional rollover
+/// always-eligible the first time it's checked.
+fn conditions_met(
+ conditions: &LifecycleConditions,
+ facts: &IndexFacts,
+) -> std::result::Result<bool, String> {
+ if let Some(age_str) = &conditions.min_index_age {
+ let threshold_ms = parse_duration_ms(age_str)?;
+ if facts.age_ms < threshold_ms {
+ return Ok(false);
+ }
+ }
+ if let Some(size_str) = &conditions.min_size {
+ let threshold_bytes = parse_size_bytes(size_str)?;
+ if facts.size_bytes < threshold_bytes {
+ return Ok(false);
+ }
+ }
+ if let Some(min_docs) = conditions.min_doc_count {
+ if facts.doc_count < min_docs {
+ return Ok(false);
+ }
+ }
+ Ok(true)
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Action execution.
+// ─────────────────────────────────────────────────────────────────────────────
+
+pub enum ActionOutcome {
+ /// The action completed; advance past it.
+ Done,
+ /// The action's own conditions aren't met yet (rollover only); leave
+ /// `next_action_index` where it is and retry next tick.
+ Waiting,
+ /// The managed index itself is gone (delete action succeeded) — the
+ /// caller must drop this index from `managed_indices` entirely rather
+ /// than continue processing it.
+ IndexDeleted,
+}
+
+async fn execute_action(
+ engine: &Engine,
+ index_name: &str,
+ action: &LifecycleAction,
+ state_age_ms: i64,
+) -> Result<ActionOutcome> {
+ match action {
+ LifecycleAction::Rollover(rollover) => {
+ let has_conditions = rollover.conditions.min_index_age.is_some()
+ || rollover.conditions.min_size.is_some()
+ || rollover.conditions.min_doc_count.is_some();
+ if has_conditions {
+ let Some(facts) = gather_index_facts(engine, index_name, state_age_ms).await else {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::index_not_found(index_name),
+ ));
+ };
+ let met = conditions_met(&rollover.conditions, &facts)
+ .map_err(|e| EngineError::Common(xerj_common::XerjError::invalid_query(e)))?;
+ if !met {
+ return Ok(ActionOutcome::Waiting);
+ }
+ }
+ // Reuses the existing, already-real rollover implementation —
+ // requires `index_name` to be a registered data stream (alias +
+ // generation-numbered backing indices). A plain index that
+ // isn't a data stream fails here with a clear not-found error
+ // rather than silently doing nothing.
+ engine.rollover_data_stream(index_name)?;
+ Ok(ActionOutcome::Done)
+ }
+ LifecycleAction::Delete(_) => {
+ engine.delete_index(index_name).await?;
+ engine.managed_indices.remove(index_name);
+ Ok(ActionOutcome::IndexDeleted)
+ }
+ LifecycleAction::ReadOnly(_) => {
+ let idx = engine.get_index(index_name)?;
+ idx.set_block("write").await?;
+ Ok(ActionOutcome::Done)
+ }
+ LifecycleAction::ReplicaCount(_) => Err(EngineError::Common(
+ xerj_common::XerjError::internal(format!(
+ "the 'replica_count' lifecycle action is not supported: XERJ is a \
+ single-shard, single-node engine with no per-index configurable \
+ replica count to change (number_of_replicas is accepted in index \
+ settings for wire compatibility but never consumed) — policy for \
+ index '{index_name}' cannot proceed past this action"
+ )),
+ )),
+ }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Tick — the background job's one unit of work.
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Run one lifecycle pass over every managed index: execute the current
+/// state's next pending action (if any), or — once all of a state's
+/// actions have completed — evaluate its transitions in order and move to
+/// the first one whose conditions are met.
+pub async fn tick(engine: &Engine) {
+ let snapshot: Vec<(String, ManagedIndexState)> = engine
+ .managed_indices
+ .iter()
+ .map(|e| (e.key().clone(), e.value().clone()))
+ .collect();
+
+ let mut any_change = false;
+ for (index_name, mut managed) in snapshot {
+ let Some(policy) = engine
+ .ism_policies
+ .get(&managed.policy_id)
+ .map(|e| e.value().clone())
+ else {
+ if !managed.failed {
+ managed.failed = true;
+ managed.info_message = format!("policy '{}' no longer exists", managed.policy_id);
+ managed.last_updated_ms = now_ms();
+ engine.managed_indices.insert(index_name, managed);
+ any_change = true;
+ }
+ continue;
+ };
+ let Some(state_def) = policy.state(&managed.current_state).cloned() else {
+ managed.failed = true;
+ managed.info_message = format!(
+ "state '{}' no longer exists in policy '{}'",
+ managed.current_state, managed.policy_id
+ );
+ managed.last_updated_ms = now_ms();
+ engine.managed_indices.insert(index_name, managed);
+ any_change = true;
+ continue;
+ };
+
+ let state_age_ms = now_ms() - managed.state_entered_at_ms;
+
+ if managed.next_action_index < state_def.actions.len() {
+ let action = &state_def.actions[managed.next_action_index];
+ match execute_action(engine, &index_name, action, state_age_ms).await {
+ Ok(ActionOutcome::Done) => {
+ managed.next_action_index += 1;
+ managed.failed = false;
+ managed.info_message = format!("completed action '{}'", action.name());
+ managed.last_updated_ms = now_ms();
+ engine.managed_indices.insert(index_name, managed);
+ any_change = true;
+ }
+ Ok(ActionOutcome::Waiting) => {
+ managed.info_message =
+ format!("waiting on '{}' action conditions", action.name());
+ managed.last_updated_ms = now_ms();
+ engine.managed_indices.insert(index_name, managed);
+ any_change = true;
+ }
+ Ok(ActionOutcome::IndexDeleted) => {
+ // Already removed from managed_indices by execute_action.
+ any_change = true;
+ }
+ Err(e) => {
+ managed.failed = true;
+ managed.info_message = format!("action '{}' failed: {e}", action.name());
+ managed.last_updated_ms = now_ms();
+ engine.managed_indices.insert(index_name, managed);
+ any_change = true;
+ }
+ }
+ continue;
+ }
+
+ // All actions in this state are done — evaluate transitions in
+ // order; first met condition wins. No transitions = terminal state.
+ let Some(facts) = gather_index_facts(engine, &index_name, state_age_ms).await else {
+ managed.failed = true;
+ managed.info_message = "index no longer exists".to_string();
+ managed.last_updated_ms = now_ms();
+ engine.managed_indices.insert(index_name, managed);
+ any_change = true;
+ continue;
+ };
+ let mut transitioned = false;
+ for t in &state_def.transitions {
+ let met = match &t.conditions {
+ None => true,
+ Some(c) => conditions_met(c, &facts).unwrap_or(false),
+ };
+ if met {
+ managed.current_state = t.state_name.clone();
+ managed.state_entered_at_ms = now_ms();
+ managed.next_action_index = 0;
+ managed.failed = false;
+ managed.info_message = format!("transitioned to state '{}'", t.state_name);
+ managed.last_updated_ms = now_ms();
+ engine.managed_indices.insert(index_name, managed);
+ transitioned = true;
+ any_change = true;
+ break;
+ }
+ }
+ if !transitioned {
+ // Nothing to do this tick; leave last_updated_ms untouched so
+ // explain doesn't churn on every quiet tick.
+ }
+ }
+
+ if any_change {
+ engine.persist_managed_indices();
+ }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Explain — the read model both REST surfaces render from.
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Build the `explain` JSON for one managed index, matching the fields real
+/// ISM's `GET _plugins/_ism/explain/{index}` reports (a practical subset —
+/// `policy_seq_no`/`policy_primary_term` are omitted since this engine has
+/// no policy versioning).
+pub fn explain_json(index_name: &str, managed: &ManagedIndexState) -> Value {
+ json!({
+ "index": index_name,
+ "index.plugins.index_state_management.policy_id": managed.policy_id,
+ "policy_id": managed.policy_id,
+ "state": {
+ "name": managed.current_state,
+ "start_time": managed.state_entered_at_ms,
+ },
+ "action": {
+ "name": managed.info_message,
+ "failed": managed.failed,
+ },
+ "info": {
+ "message": managed.info_message,
+ },
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn parse_duration_handles_all_units() {
+ assert_eq!(parse_duration_ms("500ms").unwrap(), 500);
+ assert_eq!(parse_duration_ms("7s").unwrap(), 7_000);
+ assert_eq!(parse_duration_ms("5m").unwrap(), 300_000);
+ assert_eq!(parse_duration_ms("2h").unwrap(), 7_200_000);
+ assert_eq!(parse_duration_ms("30d").unwrap(), 30 * 86_400_000);
+ assert!(parse_duration_ms("30x").is_err());
+ assert!(parse_duration_ms("abc").is_err());
+ }
+
+ #[test]
+ fn parse_size_handles_all_units() {
+ assert_eq!(parse_size_bytes("100b").unwrap(), 100);
+ assert_eq!(parse_size_bytes("1kb").unwrap(), 1024);
+ assert_eq!(parse_size_bytes("50mb").unwrap(), 50 * 1024 * 1024);
+ assert_eq!(parse_size_bytes("2gb").unwrap(), 2 * 1024 * 1024 * 1024);
+ assert!(parse_size_bytes("1zz").is_err());
+ }
+
+ #[test]
+ fn conditions_met_empty_conditions_always_true() {
+ let facts = IndexFacts {
+ age_ms: 0,
+ size_bytes: 0,
+ doc_count: 0,
+ };
+ assert!(conditions_met(&LifecycleConditions::default(), &facts).unwrap());
+ }
+
+ #[test]
+ fn conditions_met_requires_every_declared_condition() {
+ let facts = IndexFacts {
+ age_ms: 10_000,
+ size_bytes: 100,
+ doc_count: 5,
+ };
+ let conditions = LifecycleConditions {
+ min_index_age: Some("1s".to_string()),
+ min_size: None,
+ min_doc_count: Some(10),
+ };
+ // age condition met (10s >= 1s) but doc_count condition not met (5 < 10)
+ assert!(!conditions_met(&conditions, &facts).unwrap());
+
+ let conditions2 = LifecycleConditions {
+ min_index_age: Some("1s".to_string()),
+ min_size: None,
+ min_doc_count: Some(5),
+ };
+ assert!(conditions_met(&conditions2, &facts).unwrap());
+ }
+
+ #[test]
+ fn policy_validate_rejects_dangling_transition() {
+ let policy = LifecyclePolicy {
+ description: None,
+ default_state: "hot".to_string(),
+ states: vec![LifecycleState {
+ name: "hot".to_string(),
+ actions: vec![],
+ transitions: vec![LifecycleTransition {
+ state_name: "warm".to_string(),
+ conditions: None,
+ }],
+ }],
+ };
+ assert!(policy.validate().is_err());
+ }
+
+ #[test]
+ fn policy_validate_rejects_unknown_default_state() {
+ let policy = LifecyclePolicy {
+ description: None,
+ default_state: "nope".to_string(),
+ states: vec![LifecycleState {
+ name: "hot".to_string(),
+ actions: vec![],
+ transitions: vec![],
+ }],
+ };
+ assert!(policy.validate().is_err());
+ }
+
+ #[test]
+ fn policy_validate_accepts_well_formed_policy() {
+ let policy = LifecyclePolicy {
+ description: Some("test".to_string()),
+ default_state: "hot".to_string(),
+ states: vec![
+ LifecycleState {
+ name: "hot".to_string(),
+ actions: vec![LifecycleAction::Rollover(RolloverAction::default())],
+ transitions: vec![LifecycleTransition {
+ state_name: "delete".to_string(),
+ conditions: Some(LifecycleConditions {
+ min_index_age: Some("30d".to_string()),
+ min_size: None,
+ min_doc_count: None,
+ }),
+ }],
+ },
+ LifecycleState {
+ name: "delete".to_string(),
+ actions: vec![LifecycleAction::Delete(EmptyParams {})],
+ transitions: vec![],
+ },
+ ],
+ };
+ assert!(policy.validate().is_ok());
+ }
+
+ #[test]
+ fn action_json_round_trips_expected_shape() {
+ let action = LifecycleAction::Rollover(RolloverAction {
+ conditions: LifecycleConditions {
+ min_doc_count: Some(100),
+ ..Default::default()
+ },
+ });
+ let v = serde_json::to_value(&action).unwrap();
+ assert_eq!(v["rollover"]["min_doc_count"], 100);
+
+ let delete = LifecycleAction::Delete(EmptyParams {});
+ let v = serde_json::to_value(&delete).unwrap();
+ assert_eq!(v, json!({"delete": {}}));
+
+ let ro = LifecycleAction::ReadOnly(EmptyParams {});
+ let v = serde_json::to_value(&ro).unwrap();
+ assert_eq!(v, json!({"read_only": {}}));
+ }
+
+ #[test]
+ fn translate_ilm_maps_phases_to_states_in_order() {
+ let ilm = json!({
+ "policy": {
+ "phases": {
+ "hot": {
+ "min_age": "0ms",
+ "actions": { "rollover": { "max_size": "50gb", "max_docs": 100 } }
+ },
+ "delete": {
+ "min_age": "30d",
+ "actions": { "delete": {} }
+ }
+ }
+ }
+ });
+ let policy = translate_ilm_policy(&ilm).unwrap();
+ assert_eq!(policy.default_state, "hot");
+ assert_eq!(policy.states.len(), 2);
+ assert_eq!(policy.states[0].name, "hot");
+ assert_eq!(policy.states[0].transitions.len(), 1);
+ assert_eq!(policy.states[0].transitions[0].state_name, "delete");
+ assert_eq!(
+ policy.states[0].transitions[0]
+ .conditions
+ .as_ref()
+ .unwrap()
+ .min_index_age,
+ Some("30d".to_string())
+ );
+ assert_eq!(policy.states[1].name, "delete");
+ assert!(policy.states[1].transitions.is_empty());
+ match &policy.states[0].actions[0] {
+ LifecycleAction::Rollover(r) => {
+ assert_eq!(r.conditions.min_size, Some("50gb".to_string()));
+ assert_eq!(r.conditions.min_doc_count, Some(100));
+ }
+ other => panic!("expected rollover action, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn translate_ilm_skips_unmapped_actions_errors_on_unknown() {
+ let ilm = json!({
+ "policy": {
+ "phases": {
+ "hot": {
+ "actions": { "set_priority": { "priority": 100 } }
+ }
+ }
+ }
+ });
+ let policy = translate_ilm_policy(&ilm).unwrap();
+ assert!(policy.states[0].actions.is_empty());
+
+ let ilm_bad = json!({
+ "policy": { "phases": { "hot": { "actions": { "totally_made_up": {} } } } }
+ });
+ assert!(translate_ilm_policy(&ilm_bad).is_err());
+ }
+}
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index baf98e0ab..22bac8301 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -1801,6 +1801,11 @@ async fn async_main() -> Result<()> {
// a 429 circuit_breaking_exception before the kernel OOM-kills us.
state.engine.spawn_resource_sampler();
+ // 12c. ISM/ILM lifecycle-management background job: drives every
+ // managed index's state machine (actions, then transitions) on
+ // `lifecycle.tick_interval_secs` (default 5 minutes).
+ state.engine.spawn_lifecycle_manager();
+
let _ingest_memory_trace = ingest_memory_trace::spawn(&state.engine);
// 13. Start servers concurrently
let rest_tls = tls_config.clone();