Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

43.3 KB
Raw Blame
//! Index lifecycle management — one execution engine, two REST surfaces.
//!
//! Modeled on OpenSearch's Index State Management (ISM), not Elasticsearch's
//! fixed hot/warm/cold/delete phases: ISM's state machine (named states,
//! each with its own ordered actions and ordered transitions with
//! conditions) is strictly more general, so it's the internal model.
//! Elasticsearch's `_ilm/*` surface is a translator into this same model
//! (see [`translate_ilm_policy`]), not a second engine — an ILM policy with
//! hot/warm/cold/delete phases becomes an ISM policy whose states are named
//! `"hot"`/`"warm"`/`"cold"`/`"delete"` in that fixed sequence.
//!
//! ## Known simplification vs. real ILM
//!
//! Real Elasticsearch measures a phase's `min_age` from the index's
//! rollover time (not its creation time) once the policy's hot phase has a
//! `rollover` action — the age clock effectively restarts at rollover. This
//! engine measures every state's age from when the index *entered that
//! state* (`state_entered_at_ms`), which is index-attach time for the
//! initial state, not rollover time. For a policy without `rollover`, or
//! for reasoning about relative phase durations, this is equivalent; for a
//! policy that rolls over AND has a downstream phase whose `min_age` is
//! meant to be measured strictly from rollover, this engine will advance
//! phases slightly earlier than real ILM would. Documented rather than
//! silently diverging — see PR discussion for whether this needs closing.
//!
//! ## What's NOT supported, and errors honestly instead of faking success
//!
//! `replica_count`: XERJ has no per-index configurable replica concept
//! (single-shard, single-node engine; `number_of_replicas` is accepted in
//! index settings and echoed back for wire compatibility but never
//! consumed). The action fails with an explicit error rather than silently
//! no-op'ing, so a policy relying on it is visibly stuck, not silently
//! wrong.

use std::time::{SystemTime, UNIX_EPOCH};

use serde::{Deserialize, Serialize};
use serde_json::{json, Value};

use crate::engine::Engine;
use crate::{EngineError, Result};

// ─────────────────────────────────────────────────────────────────────────────
// Data model — faithful to OpenSearch ISM's policy JSON shape.
// ─────────────────────────────────────────────────────────────────────────────

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LifecyclePolicy {
    #[serde(default)]
    pub description: Option<String>,
    pub default_state: String,
    pub states: Vec<LifecycleState>,
}

impl LifecyclePolicy {
    /// Structural validation beyond what serde already enforces: every
    /// transition target and `default_state` must name a real state, and
    /// state names must be unique. Real ISM rejects a policy with a
    /// dangling transition at PUT time rather than discovering it mid-run.
    pub fn validate(&self) -> std::result::Result<(), String> {
        if self.states.is_empty() {
            return Err("policy must declare at least one state".to_string());
        }
        let names: std::collections::HashSet<&str> =
            self.states.iter().map(|s| s.name.as_str()).collect();
        if names.len() != self.states.len() {
            return Err("state names must be unique".to_string());
        }
        if !names.contains(self.default_state.as_str()) {
            return Err(format!(
                "default_state '{}' does not name a declared state",
                self.default_state
            ));
        }
        for state in &self.states {
            for t in &state.transitions {
                if !names.contains(t.state_name.as_str()) {
                    return Err(format!(
                        "state '{}' transitions to undeclared state '{}'",
                        state.name, t.state_name
                    ));
                }
            }
        }
        Ok(())
    }

    pub fn state(&self, name: &str) -> Option<&LifecycleState> {
        self.states.iter().find(|s| s.name == name)
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LifecycleState {
    pub name: String,
    #[serde(default)]
    pub actions: Vec<LifecycleAction>,
    #[serde(default)]
    pub transitions: Vec<LifecycleTransition>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LifecycleTransition {
    pub state_name: String,
    #[serde(default)]
    pub conditions: Option<LifecycleConditions>,
}

/// Conditions shared by transitions (when to leave a state) and the
/// `rollover` action (when it's actually worth rolling over) — same shape,
/// same evaluator, two different call sites. A transition/action with no
/// conditions at all is always eligible the moment it's checked (an
/// unconditional transition; an unconditional rollover).
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct LifecycleConditions {
    /// ES/ISM duration string: digits followed by `ms|s|m|h|d`. E.g. `"7d"`.
    #[serde(default)]
    pub min_index_age: Option<String>,
    /// ES/ISM size string: digits followed by `b|kb|mb|gb|tb`. E.g. `"50gb"`.
    #[serde(default)]
    pub min_size: Option<String>,
    #[serde(default)]
    pub min_doc_count: Option<u64>,
}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct EmptyParams {}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct RolloverAction {
    #[serde(flatten)]
    pub conditions: LifecycleConditions,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReplicaCountAction {
    pub number_of_replicas: u64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LifecycleAction {
    Rollover(RolloverAction),
    Delete(EmptyParams),
    ReadOnly(EmptyParams),
    ReplicaCount(ReplicaCountAction),
}

impl LifecycleAction {
    pub fn name(&self) -> &'static str {
        match self {
            LifecycleAction::Rollover(_) => "rollover",
            LifecycleAction::Delete(_) => "delete",
            LifecycleAction::ReadOnly(_) => "read_only",
            LifecycleAction::ReplicaCount(_) => "replica_count",
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Managed-index state — the per-index execution cursor.
// ─────────────────────────────────────────────────────────────────────────────

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ManagedIndexState {
    pub policy_id: String,
    pub current_state: String,
    pub state_entered_at_ms: i64,
    /// Index into the current state's `actions[]` — which one runs next.
    /// Equal to `actions.len()` once every action in this state has
    /// completed and only transition-evaluation remains.
    #[serde(default)]
    pub next_action_index: usize,
    pub last_updated_ms: i64,
    /// Mirrors ISM explain's free-form `info.message`.
    #[serde(default)]
    pub info_message: String,
    /// Set when a step errors, so `explain` surfaces it instead of the
    /// index silently retrying the same failing action forever unseen.
    /// Retries still happen (a transient failure — e.g. disk contention —
    /// should self-heal); this is visibility, not a halt.
    #[serde(default)]
    pub failed: bool,
}

impl ManagedIndexState {
    pub fn new(policy_id: String, default_state: String, now_ms: i64) -> Self {
        Self {
            policy_id,
            current_state: default_state,
            state_entered_at_ms: now_ms,
            next_action_index: 0,
            last_updated_ms: now_ms,
            info_message: "attached, awaiting first execution".to_string(),
            failed: false,
        }
    }
}

pub fn now_ms() -> i64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|d| d.as_millis() as i64)
        .unwrap_or(0)
}

// ─────────────────────────────────────────────────────────────────────────────
// Duration / size string parsing — ES/ISM shorthand.
// ─────────────────────────────────────────────────────────────────────────────

/// Parse an ES/ISM duration string (`"7d"`, `"30m"`, `"500ms"`) into
/// milliseconds. Unlike ES's full duration grammar this accepts exactly one
/// integer + one unit, which is all `min_index_age` ever uses in practice.
pub fn parse_duration_ms(s: &str) -> std::result::Result<i64, String> {
    let s = s.trim();
    let split_at = s
        .find(|c: char| !c.is_ascii_digit())
        .ok_or_else(|| format!("duration '{s}' has no unit"))?;
    let (digits, unit) = s.split_at(split_at);
    let n: i64 = digits
        .parse()
        .map_err(|_| format!("duration '{s}' has no numeric prefix"))?;
    let ms = match unit {
        "ms" => n,
        "s" => n * 1_000,
        "m" => n * 60_000,
        "h" => n * 3_600_000,
        "d" => n * 86_400_000,
        other => return Err(format!("unknown duration unit '{other}' in '{s}'")),
    };
    Ok(ms)
}

/// Parse an ES/ISM size string (`"50gb"`, `"100mb"`) into bytes. Binary
/// (1024-based) units, matching ES's own `ByteSizeValue`.
pub fn parse_size_bytes(s: &str) -> std::result::Result<u64, String> {
    let s = s.trim().to_ascii_lowercase();
    let split_at = s
        .find(|c: char| !c.is_ascii_digit())
        .ok_or_else(|| format!("size '{s}' has no unit"))?;
    let (digits, unit) = s.split_at(split_at);
    let n: u64 = digits
        .parse()
        .map_err(|_| format!("size '{s}' has no numeric prefix"))?;
    let bytes = match unit {
        "b" => n,
        "kb" => n * 1024,
        "mb" => n * 1024 * 1024,
        "gb" => n * 1024 * 1024 * 1024,
        "tb" => n * 1024 * 1024 * 1024 * 1024,
        other => return Err(format!("unknown size unit '{other}' in '{s}'")),
    };
    Ok(bytes)
}

// ─────────────────────────────────────────────────────────────────────────────
// ES ILM → internal model translator.
// ─────────────────────────────────────────────────────────────────────────────

/// Fixed ILM phase order — ISM states are named exactly this, one per phase
/// actually present in the policy. A phase absent from the ILM policy
/// produces no state at all (matching ILM, where an unconfigured phase is
/// simply skipped).
const ILM_PHASE_ORDER: [&str; 4] = ["hot", "warm", "cold", "delete"];

/// Translate an ES ILM policy body (`{"policy": {"phases": {...}}}`, the
/// same shape `PUT _ilm/policy/{name}` accepts) into a [`LifecyclePolicy`].
///
/// Each present phase becomes a state named after the phase. A phase's own
/// `actions` (minus `min_age`, which isn't an action) become that state's
/// actions. A phase's `min_age` becomes the CONDITION on the transition
/// INTO that phase from the previous one — i.e. "warm: min_age 7d" means
/// the state before warm gets a transition to warm gated on
/// `min_index_age: 7d`. The final present phase gets no transitions
/// (terminal), matching a real ISM policy's last state.
pub fn translate_ilm_policy(ilm_body: &Value) -> std::result::Result<LifecyclePolicy, String> {
    let phases = ilm_body
        .pointer("/policy/phases")
        .and_then(Value::as_object)
        .ok_or_else(|| "ILM policy body missing policy.phases".to_string())?;

    let present: Vec<&str> = ILM_PHASE_ORDER
        .iter()
        .copied()
        .filter(|p| phases.contains_key(*p))
        .collect();
    if present.is_empty() {
        return Err("ILM policy declares no known phase (hot/warm/cold/delete)".to_string());
    }

    let mut states = Vec::with_capacity(present.len());
    for (i, phase_name) in present.iter().enumerate() {
        let phase = &phases[*phase_name];
        let actions = translate_ilm_phase_actions(phase_name, phase)?;

        let transitions = if let Some(&next_phase) = present.get(i + 1) {
            let next_min_age = phases[next_phase]
                .get("min_age")
                .and_then(Value::as_str)
                .unwrap_or("0ms");
            vec![LifecycleTransition {
                state_name: next_phase.to_string(),
                conditions: Some(LifecycleConditions {
                    min_index_age: Some(next_min_age.to_string()),
                    min_size: None,
                    min_doc_count: None,
                }),
            }]
        } else {
            Vec::new()
        };

        states.push(LifecycleState {
            name: phase_name.to_string(),
            actions,
            transitions,
        });
    }

    let policy = LifecyclePolicy {
        description: ilm_body
            .pointer("/policy/_meta/description")
            .and_then(Value::as_str)
            .map(String::from),
        default_state: present[0].to_string(),
        states,
    };
    policy.validate()?;
    Ok(policy)
}

fn translate_ilm_phase_actions(
    phase_name: &str,
    phase: &Value,
) -> std::result::Result<Vec<LifecycleAction>, String> {
    let Some(actions_obj) = phase.get("actions").and_then(Value::as_object) else {
        return Ok(Vec::new());
    };
    let mut out = Vec::new();
    for (action_name, params) in actions_obj {
        let action = match action_name.as_str() {
            "rollover" => {
                let conditions = LifecycleConditions {
                    min_index_age: params
                        .get("max_age")
                        .and_then(Value::as_str)
                        .map(String::from),
                    min_size: params
                        .get("max_size")
                        .and_then(Value::as_str)
                        .map(String::from),
                    min_doc_count: params.get("max_docs").and_then(Value::as_u64),
                };
                LifecycleAction::Rollover(RolloverAction { conditions })
            }
            "delete" => LifecycleAction::Delete(EmptyParams {}),
            "readonly" => LifecycleAction::ReadOnly(EmptyParams {}),
            // Fail-closed allowlist (issue #282, ported from #262's
            // `validate_policy`): an action this engine cannot execute is
            // refused with its name, never silently dropped. This used to
            // skip `set_priority`/`allocate`/`unfollow`/`shrink`/
            // `forcemerge`/`freeze` on the argument that a no-op-shaped
            // action is harmless — but a policy is a contract, and
            // accepting one while quietly not honouring part of it is this
            // repo's dominant bug class (#204). The operator can remove the
            // action from the policy; they cannot remove a silent skip.
            other => {
                return Err(format!(
                    "phase '{phase_name}' action '{other}' is not executable by this engine \
                     (executable ILM actions: rollover, delete, readonly) — refusing the \
                     policy rather than accepting it and skipping the action"
                ))
            }
        };
        out.push(action);
    }
    Ok(out)
}

// ─────────────────────────────────────────────────────────────────────────────
// Condition evaluation.
// ─────────────────────────────────────────────────────────────────────────────

struct IndexFacts {
    age_ms: i64,
    size_bytes: u64,
    doc_count: u64,
}

async fn gather_index_facts(
    engine: &Engine,
    index_name: &str,
    state_age_ms: i64,
) -> Option<IndexFacts> {
    let idx = engine.get_index(index_name).ok()?;
    let stats = idx.stats().await;
    let segment_bytes: u64 = idx
        .store_snapshot()
        .segments
        .iter()
        .map(|s| s.size_bytes)
        .sum();
    let size_bytes = segment_bytes + stats.memtable_size_bytes as u64;
    Some(IndexFacts {
        age_ms: state_age_ms,
        size_bytes,
        doc_count: stats.doc_count,
    })
}

/// A `LifecycleConditions` with every field `None` matches immediately —
/// this is what makes an unconditional transition / unconditional rollover
/// always-eligible the first time it's checked.
fn conditions_met(
    conditions: &LifecycleConditions,
    facts: &IndexFacts,
) -> std::result::Result<bool, String> {
    if let Some(age_str) = &conditions.min_index_age {
        let threshold_ms = parse_duration_ms(age_str)?;
        if facts.age_ms < threshold_ms {
            return Ok(false);
        }
    }
    if let Some(size_str) = &conditions.min_size {
        let threshold_bytes = parse_size_bytes(size_str)?;
        if facts.size_bytes < threshold_bytes {
            return Ok(false);
        }
    }
    if let Some(min_docs) = conditions.min_doc_count {
        if facts.doc_count < min_docs {
            return Ok(false);
        }
    }
    Ok(true)
}

// ─────────────────────────────────────────────────────────────────────────────
// Action execution.
// ─────────────────────────────────────────────────────────────────────────────

pub enum ActionOutcome {
    /// The action completed; advance past it.
    Done,
    /// The action's own conditions aren't met yet (rollover only); leave
    /// `next_action_index` where it is and retry next tick.
    Waiting,
    /// The managed index itself is gone (delete action succeeded) — the
    /// caller must drop this index from `managed_indices` entirely rather
    /// than continue processing it.
    IndexDeleted,
}

/// Why the delete action must not fire on `index`, if it must not — the
/// delete rails (issue #282, ported from #262's `ilm_delete_block_reason`).
///
/// One function, one answer: the executor's refusal and the operator-visible
/// failure message can never disagree about what happened.
///
///  * Dot-prefixed names are XERJ/Kibana internals (`.xerj-memory-*`
///    brains, `.kibana*`, security stores). A wildcard settings update or a
///    copy-pasted attach must not be able to point a 7-day retention policy
///    at a user's second brain. Data-stream backing indices (`.ds-*`) are
///    the one dotted family lifecycle deletion is *for*, so they are exempt
///    from this rail (and covered by the next one while current).
///  * A data stream's current write index is where new documents are
///    landing right now; ES's own delete phase refuses it too.
///  * An index whose age cannot be established must be skipped, never
///    deleted on a guess (quickwit's janitor makes the same call for
///    undatable splits — `retention_policy_execution.rs:65-76`, Apache-2.0,
///    approach adapted). In this engine's cursor model the age clock is
///    `state_entered_at_ms`; a cursor whose entry time is missing (0, from
///    a hand-edited or partially-corrupt state file) or in the future
///    (clock skew) establishes nothing.
fn delete_block_reason(engine: &Engine, index: &str, state_entered_at_ms: i64) -> Option<String> {
    if index.starts_with('.') && !index.starts_with(".ds-") {
        return Some(
            "dot-prefixed internal index — the lifecycle delete action never removes one"
                .to_string(),
        );
    }
    let write_index_owner =
        engine
            .data_streams
            .iter()
            .find_map(|e| match e.value().backing_indices.last() {
                Some(write_index) if write_index == index => Some(e.key().clone()),
                _ => None,
            });
    if let Some(stream) = write_index_owner {
        return Some(format!("current write index of data stream '{stream}'"));
    }
    if state_entered_at_ms <= 0 || now_ms() < state_entered_at_ms {
        return Some(
            "index age cannot be established (state-entry time is missing or in the future)"
                .to_string(),
        );
    }
    None
}

async fn execute_action(
    engine: &Engine,
    index_name: &str,
    action: &LifecycleAction,
    state_age_ms: i64,
    state_entered_at_ms: i64,
) -> Result<ActionOutcome> {
    match action {
        LifecycleAction::Rollover(rollover) => {
            let has_conditions = rollover.conditions.min_index_age.is_some()
                || rollover.conditions.min_size.is_some()
                || rollover.conditions.min_doc_count.is_some();
            if has_conditions {
                let Some(facts) = gather_index_facts(engine, index_name, state_age_ms).await else {
                    return Err(EngineError::Common(
                        xerj_common::XerjError::index_not_found(index_name),
                    ));
                };
                let met = conditions_met(&rollover.conditions, &facts)
                    .map_err(|e| EngineError::Common(xerj_common::XerjError::invalid_query(e)))?;
                if !met {
                    return Ok(ActionOutcome::Waiting);
                }
            }
            // Reuses the existing, already-real rollover implementation —
            // requires `index_name` to be a registered data stream (alias +
            // generation-numbered backing indices). A plain index that
            // isn't a data stream fails here with a clear not-found error
            // rather than silently doing nothing.
            engine.rollover_data_stream(index_name)?;
            Ok(ActionOutcome::Done)
        }
        LifecycleAction::Delete(_) => {
            if let Some(reason) = delete_block_reason(engine, index_name, state_entered_at_ms) {
                // An Err here surfaces as `failed: true` + this message in
                // explain — visible refusal, not a silent skip (#204).
                return Err(EngineError::Common(xerj_common::XerjError::invalid_query(
                    format!("refusing lifecycle delete of '{index_name}': {reason}"),
                )));
            }
            // Before deleting the index: detach persists cluster metadata, so
            // a blocked boot must refuse before any backing directory or live
            // map changes.
            engine.ensure_cluster_state_writable()?;
            engine.delete_index(index_name).await?;
            engine.managed_indices.remove(index_name);
            engine.detach_data_stream_backing_index(index_name)?;
            Ok(ActionOutcome::IndexDeleted)
        }
        LifecycleAction::ReadOnly(_) => {
            let idx = engine.get_index(index_name)?;
            idx.set_block("write").await?;
            Ok(ActionOutcome::Done)
        }
        LifecycleAction::ReplicaCount(_) => Err(EngineError::Common(
            xerj_common::XerjError::internal(format!(
                "the 'replica_count' lifecycle action is not supported: XERJ is a \
                 single-shard, single-node engine with no per-index configurable \
                 replica count to change (number_of_replicas is accepted in index \
                 settings for wire compatibility but never consumed) — policy for \
                 index '{index_name}' cannot proceed past this action"
            )),
        )),
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Tick — the background job's one unit of work.
// ─────────────────────────────────────────────────────────────────────────────

/// Run one lifecycle pass over every managed index: execute the current
/// state's next pending action (if any), or — once all of a state's
/// actions have completed — evaluate its transitions in order and move to
/// the first one whose conditions are met.
pub async fn tick(engine: &Engine) {
    // Operator kill switch (`POST /_ilm/stop`, issue #282): while stopped,
    // a tick observes nothing and touches nothing.
    if !engine.lifecycle_execution_running() {
        return;
    }
    let snapshot: Vec<(String, ManagedIndexState)> = engine
        .managed_indices
        .iter()
        .map(|e| (e.key().clone(), e.value().clone()))
        .collect();

    let mut any_change = false;
    for (index_name, mut managed) in snapshot {
        let Some(policy) = engine
            .ism_policies
            .get(&managed.policy_id)
            .map(|e| e.value().clone())
        else {
            if !managed.failed {
                managed.failed = true;
                managed.info_message = format!("policy '{}' no longer exists", managed.policy_id);
                managed.last_updated_ms = now_ms();
                engine.managed_indices.insert(index_name, managed);
                any_change = true;
            }
            continue;
        };
        let Some(state_def) = policy.state(&managed.current_state).cloned() else {
            managed.failed = true;
            managed.info_message = format!(
                "state '{}' no longer exists in policy '{}'",
                managed.current_state, managed.policy_id
            );
            managed.last_updated_ms = now_ms();
            engine.managed_indices.insert(index_name, managed);
            any_change = true;
            continue;
        };

        let state_age_ms = now_ms() - managed.state_entered_at_ms;

        if managed.next_action_index < state_def.actions.len() {
            let action = &state_def.actions[managed.next_action_index];
            match execute_action(
                engine,
                &index_name,
                action,
                state_age_ms,
                managed.state_entered_at_ms,
            )
            .await
            {
                Ok(ActionOutcome::Done) => {
                    managed.next_action_index += 1;
                    managed.failed = false;
                    managed.info_message = format!("completed action '{}'", action.name());
                    managed.last_updated_ms = now_ms();
                    engine.managed_indices.insert(index_name, managed);
                    any_change = true;
                }
                Ok(ActionOutcome::Waiting) => {
                    managed.info_message =
                        format!("waiting on '{}' action conditions", action.name());
                    managed.last_updated_ms = now_ms();
                    engine.managed_indices.insert(index_name, managed);
                    any_change = true;
                }
                Ok(ActionOutcome::IndexDeleted) => {
                    // Already removed from managed_indices by execute_action.
                    any_change = true;
                }
                Err(e) => {
                    managed.failed = true;
                    managed.info_message = format!("action '{}' failed: {e}", action.name());
                    managed.last_updated_ms = now_ms();
                    engine.managed_indices.insert(index_name, managed);
                    any_change = true;
                }
            }
            continue;
        }

        // All actions in this state are done — evaluate transitions in
        // order; first met condition wins. No transitions = terminal state.
        let Some(facts) = gather_index_facts(engine, &index_name, state_age_ms).await else {
            managed.failed = true;
            managed.info_message = "index no longer exists".to_string();
            managed.last_updated_ms = now_ms();
            engine.managed_indices.insert(index_name, managed);
            any_change = true;
            continue;
        };
        let mut transitioned = false;
        for t in &state_def.transitions {
            let met = match &t.conditions {
                None => true,
                Some(c) => conditions_met(c, &facts).unwrap_or(false),
            };
            if met {
                managed.current_state = t.state_name.clone();
                managed.state_entered_at_ms = now_ms();
                managed.next_action_index = 0;
                managed.failed = false;
                managed.info_message = format!("transitioned to state '{}'", t.state_name);
                managed.last_updated_ms = now_ms();
                engine.managed_indices.insert(index_name, managed);
                transitioned = true;
                any_change = true;
                break;
            }
        }
        if !transitioned {
            // Nothing to do this tick; leave last_updated_ms untouched so
            // explain doesn't churn on every quiet tick.
        }
    }

    if any_change {
        if let Err(error) = engine.persist_managed_indices() {
            tracing::warn!(%error, "lifecycle state was not persisted");
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Explain — the read model both REST surfaces render from.
// ─────────────────────────────────────────────────────────────────────────────

/// Build the `explain` JSON for one managed index, matching the fields real
/// ISM's `GET _plugins/_ism/explain/{index}` reports (a practical subset —
/// `policy_seq_no`/`policy_primary_term` are omitted since this engine has
/// no policy versioning).
pub fn explain_json(index_name: &str, managed: &ManagedIndexState) -> Value {
    json!({
        "index": index_name,
        "index.plugins.index_state_management.policy_id": managed.policy_id,
        "policy_id": managed.policy_id,
        "state": {
            "name": managed.current_state,
            "start_time": managed.state_entered_at_ms,
        },
        "action": {
            "name": managed.info_message,
            "failed": managed.failed,
        },
        "info": {
            "message": managed.info_message,
        },
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn cluster_state_format1_lifecycle_delete_and_detach_guard_before_side_effects() {
        fn storage_snapshot(root: &std::path::Path) -> Vec<(String, Vec<u8>)> {
            fn walk(
                root: &std::path::Path,
                path: &std::path::Path,
                out: &mut Vec<(String, Vec<u8>)>,
            ) {
                let mut entries: Vec<_> = std::fs::read_dir(path)
                    .unwrap()
                    .map(|entry| entry.unwrap())
                    .collect();
                entries.sort_by_key(|entry| entry.file_name());
                for entry in entries {
                    let path = entry.path();
                    let relative = path.strip_prefix(root).unwrap().to_string_lossy();
                    let kind = entry.file_type().unwrap();
                    if kind.is_dir() {
                        out.push((format!("d:{relative}"), Vec::new()));
                        walk(root, &path, out);
                    } else {
                        out.push((format!("f:{relative}"), std::fs::read(path).unwrap()));
                    }
                }
            }
            let mut out = Vec::new();
            walk(root, root, &mut out);
            out
        }

        let dir = tempfile::tempdir().unwrap();
        let mut config = xerj_common::config::Config::default();
        config.server.data_dir = dir.path().to_string_lossy().into_owned();
        config.storage.wal_sync = xerj_common::config::WalSync::Async;
        {
            let engine = Engine::new(config.clone()).unwrap();
            engine
                .create_index("logs-1", xerj_common::types::Schema::empty())
                .unwrap();
        }

        let future =
            include_bytes!("../tests/fixtures/cluster_state/v2-format-version-minimal.json");
        let state_path = dir.path().join("cluster_state.json");
        std::fs::write(&state_path, future).unwrap();
        // The only reacher of the boot diagnostic outside `cluster_state::tests`
        // in this `--lib` binary: preflight classifies this fixture as
        // `UnsupportedFormat` and `PreparedClusterState::blocked` emits its
        // `error!` from this thread, which has no subscriber.  Whichever thread
        // first registers that callsite decides its process-global `Interest`,
        // and a subscriber-less registration pins it to `never`, blanking any
        // concurrent `cluster_state::tests` capture — issue #372.  Today this
        // cannot lose that race: libtest runs a byte-sorted name list, and
        // `cluster_state::tests::*` ends at #106 of 601 while this test is #436
        // (measured with `--list` at this commit), so registration is long
        // settled.  The lock is what stops the guarantee from resting on that.
        // See `cluster_state::PREFLIGHT_CALLSITE`.
        let engine = {
            let _serialised = crate::cluster_state::lock_preflight_callsite();
            Engine::new(config).unwrap()
        };
        assert!(matches!(
            engine.get_index("logs-1"),
            Err(EngineError::Common(
                xerj_common::XerjError::ClusterStateUnavailable
            ))
        ));
        engine.data_streams.insert(
            "events".to_owned(),
            crate::engine::DataStream {
                name: "events".to_owned(),
                backing_indices: vec!["logs-1".to_owned(), "write-current".to_owned()],
                timestamp_field: "@timestamp".to_owned(),
                generation: 2,
            },
        );
        let before_files = storage_snapshot(dir.path());
        let before_stream = engine.data_streams.get("events").unwrap().clone();

        let direct = engine.detach_data_stream_backing_index("logs-1");
        assert!(matches!(
            direct,
            Err(EngineError::Common(
                xerj_common::XerjError::ClusterStateUnavailable
            ))
        ));
        assert_eq!(
            engine.data_streams.get("events").unwrap().backing_indices,
            before_stream.backing_indices,
            "the direct detach mutator must fail before editing the live stream"
        );
        assert_eq!(
            storage_snapshot(dir.path()),
            before_files,
            "the direct detach mutator must fail before rewriting any file"
        );

        let result = execute_action(
            &engine,
            "logs-1",
            &LifecycleAction::Delete(EmptyParams {}),
            1,
            now_ms().saturating_sub(1),
        )
        .await;
        assert!(matches!(
            result,
            Err(EngineError::Common(
                xerj_common::XerjError::ClusterStateUnavailable
            ))
        ));
        assert!(dir.path().join("logs-1").is_dir());
        assert_eq!(
            engine.data_streams.get("events").unwrap().backing_indices,
            before_stream.backing_indices,
            "the detach half must not edit the live stream"
        );
        assert_eq!(
            storage_snapshot(dir.path()),
            before_files,
            "blocked lifecycle delete/detach must not rewrite any file"
        );

        assert_eq!(std::fs::read(state_path).unwrap(), future.as_slice());
    }

    #[test]
    fn parse_duration_handles_all_units() {
        assert_eq!(parse_duration_ms("500ms").unwrap(), 500);
        assert_eq!(parse_duration_ms("7s").unwrap(), 7_000);
        assert_eq!(parse_duration_ms("5m").unwrap(), 300_000);
        assert_eq!(parse_duration_ms("2h").unwrap(), 7_200_000);
        assert_eq!(parse_duration_ms("30d").unwrap(), 30 * 86_400_000);
        assert!(parse_duration_ms("30x").is_err());
        assert!(parse_duration_ms("abc").is_err());
    }

    #[test]
    fn parse_size_handles_all_units() {
        assert_eq!(parse_size_bytes("100b").unwrap(), 100);
        assert_eq!(parse_size_bytes("1kb").unwrap(), 1024);
        assert_eq!(parse_size_bytes("50mb").unwrap(), 50 * 1024 * 1024);
        assert_eq!(parse_size_bytes("2gb").unwrap(), 2 * 1024 * 1024 * 1024);
        assert!(parse_size_bytes("1zz").is_err());
    }

    #[test]
    fn conditions_met_empty_conditions_always_true() {
        let facts = IndexFacts {
            age_ms: 0,
            size_bytes: 0,
            doc_count: 0,
        };
        assert!(conditions_met(&LifecycleConditions::default(), &facts).unwrap());
    }

    #[test]
    fn conditions_met_requires_every_declared_condition() {
        let facts = IndexFacts {
            age_ms: 10_000,
            size_bytes: 100,
            doc_count: 5,
        };
        let conditions = LifecycleConditions {
            min_index_age: Some("1s".to_string()),
            min_size: None,
            min_doc_count: Some(10),
        };
        // age condition met (10s >= 1s) but doc_count condition not met (5 < 10)
        assert!(!conditions_met(&conditions, &facts).unwrap());

        let conditions2 = LifecycleConditions {
            min_index_age: Some("1s".to_string()),
            min_size: None,
            min_doc_count: Some(5),
        };
        assert!(conditions_met(&conditions2, &facts).unwrap());
    }

    #[test]
    fn policy_validate_rejects_dangling_transition() {
        let policy = LifecyclePolicy {
            description: None,
            default_state: "hot".to_string(),
            states: vec![LifecycleState {
                name: "hot".to_string(),
                actions: vec![],
                transitions: vec![LifecycleTransition {
                    state_name: "warm".to_string(),
                    conditions: None,
                }],
            }],
        };
        assert!(policy.validate().is_err());
    }

    #[test]
    fn policy_validate_rejects_unknown_default_state() {
        let policy = LifecyclePolicy {
            description: None,
            default_state: "nope".to_string(),
            states: vec![LifecycleState {
                name: "hot".to_string(),
                actions: vec![],
                transitions: vec![],
            }],
        };
        assert!(policy.validate().is_err());
    }

    #[test]
    fn policy_validate_accepts_well_formed_policy() {
        let policy = LifecyclePolicy {
            description: Some("test".to_string()),
            default_state: "hot".to_string(),
            states: vec![
                LifecycleState {
                    name: "hot".to_string(),
                    actions: vec![LifecycleAction::Rollover(RolloverAction::default())],
                    transitions: vec![LifecycleTransition {
                        state_name: "delete".to_string(),
                        conditions: Some(LifecycleConditions {
                            min_index_age: Some("30d".to_string()),
                            min_size: None,
                            min_doc_count: None,
                        }),
                    }],
                },
                LifecycleState {
                    name: "delete".to_string(),
                    actions: vec![LifecycleAction::Delete(EmptyParams {})],
                    transitions: vec![],
                },
            ],
        };
        assert!(policy.validate().is_ok());
    }

    #[test]
    fn action_json_round_trips_expected_shape() {
        let action = LifecycleAction::Rollover(RolloverAction {
            conditions: LifecycleConditions {
                min_doc_count: Some(100),
                ..Default::default()
            },
        });
        let v = serde_json::to_value(&action).unwrap();
        assert_eq!(v["rollover"]["min_doc_count"], 100);

        let delete = LifecycleAction::Delete(EmptyParams {});
        let v = serde_json::to_value(&delete).unwrap();
        assert_eq!(v, json!({"delete": {}}));

        let ro = LifecycleAction::ReadOnly(EmptyParams {});
        let v = serde_json::to_value(&ro).unwrap();
        assert_eq!(v, json!({"read_only": {}}));
    }

    #[test]
    fn translate_ilm_maps_phases_to_states_in_order() {
        let ilm = json!({
            "policy": {
                "phases": {
                    "hot": {
                        "min_age": "0ms",
                        "actions": { "rollover": { "max_size": "50gb", "max_docs": 100 } }
                    },
                    "delete": {
                        "min_age": "30d",
                        "actions": { "delete": {} }
                    }
                }
            }
        });
        let policy = translate_ilm_policy(&ilm).unwrap();
        assert_eq!(policy.default_state, "hot");
        assert_eq!(policy.states.len(), 2);
        assert_eq!(policy.states[0].name, "hot");
        assert_eq!(policy.states[0].transitions.len(), 1);
        assert_eq!(policy.states[0].transitions[0].state_name, "delete");
        assert_eq!(
            policy.states[0].transitions[0]
                .conditions
                .as_ref()
                .unwrap()
                .min_index_age,
            Some("30d".to_string())
        );
        assert_eq!(policy.states[1].name, "delete");
        assert!(policy.states[1].transitions.is_empty());
        match &policy.states[0].actions[0] {
            LifecycleAction::Rollover(r) => {
                assert_eq!(r.conditions.min_size, Some("50gb".to_string()));
                assert_eq!(r.conditions.min_doc_count, Some(100));
            }
            other => panic!("expected rollover action, got {other:?}"),
        }
    }

    #[test]
    fn translate_ilm_refuses_unexecutable_actions_naming_them() {
        // Issue #282: `set_priority` (and friends) used to be silently
        // skipped; the fail-closed allowlist refuses them with the action
        // named so the operator learns at PUT time, not from a policy that
        // quietly does less than it says.
        let ilm = json!({
            "policy": {
                "phases": {
                    "hot": {
                        "actions": { "set_priority": { "priority": 100 } }
                    }
                }
            }
        });
        let err = translate_ilm_policy(&ilm).unwrap_err();
        assert!(err.contains("set_priority"), "must name the action: {err}");

        let ilm_bad = json!({
            "policy": { "phases": { "hot": { "actions": { "totally_made_up": {} } } } }
        });
        let err = translate_ilm_policy(&ilm_bad).unwrap_err();
        assert!(
            err.contains("totally_made_up"),
            "must name the action: {err}"
        );
    }
}