Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(api): implement missing ISM remove/change_policy/retry endpoints

Vincenzo Lombardo393291197fb5
diff --git a/engine/crates/xerj-api/src/ism_api.rs b/engine/crates/xerj-api/src/ism_api.rs
index f7062e2fa..0f280c5c3 100644
--- a/engine/crates/xerj-api/src/ism_api.rs
+++ b/engine/crates/xerj-api/src/ism_api.rs
@@ -10,9 +10,17 @@
//! after live OSD UI verification showed the "State
//! management policies" table 500ing without it)
//! POST /_plugins/_ism/add/{index}
+//! POST /_plugins/_ism/remove/{index}
+//! POST /_plugins/_ism/change_policy/{index}
+//! POST /_plugins/_ism/retry/{index}
//! GET /_plugins/_ism/explain/{index}
//! GET /_plugins/_ism/explain (list-all — same
//! reason, for the "Policy managed indexes" table)
+//!
+//! `remove`/`change_policy`/`retry` were added after a systematic pass over
+//! OSD's Index Management UI actions (the "Remove policy" / "Change policy"
+//! / "Retry policy" buttons on the Managed Indexes table) found they had no
+//! backend at all — same discovery method as the list-form gaps above.
use axum::{
extract::{Path, Query, State},
@@ -24,7 +32,29 @@ use serde_json::{json, Value};
use xerj_engine::lifecycle::{self, LifecyclePolicy, ManagedIndexState};
-use crate::{error::ApiError, state::AppState};
+use crate::{error::ApiError, extract::OptionalJson, state::AppState};
+
+/// The `{"failures": bool, "updated_indices": N, "failed_indices": [...]}`
+/// shape real ISM's bulk-style index-targeting endpoints (`add`, `remove`,
+/// `change_policy`, `retry`) all share. A single-index failure here is a
+/// normal, expected outcome (e.g. "not currently managed") — not an error
+/// worth a non-200 status, matching how OSD's own client reads the body's
+/// `failures` flag rather than the HTTP status to decide success.
+fn not_managed_failure(index: &str, reason: &str) -> Value {
+ json!({
+ "failures": true,
+ "updated_indices": 0,
+ "failed_indices": [{
+ "index_name": index,
+ "index_uuid": "_na_",
+ "reason": reason,
+ }],
+ })
+}
+
+fn single_index_success() -> Value {
+ json!({ "failures": false, "updated_indices": 1, "failed_indices": [] })
+}
// ─────────────────────────────────────────────────────────────────────────────
// PUT/GET/DELETE /_plugins/_ism/policies/{policy_id}
@@ -212,6 +242,204 @@ pub async fn attach_policy(state: &AppState, index: &str, policy_id: &str) -> Re
.into_response()
}
+// ─────────────────────────────────────────────────────────────────────────────
+// POST /_plugins/_ism/remove/{index}
+// ─────────────────────────────────────────────────────────────────────────────
+
+pub async fn remove_ism_policy(
+ State(state): State<AppState>,
+ Path(index): Path<String>,
+) -> impl IntoResponse {
+ if state.engine.managed_indices.remove(&index).is_some() {
+ state.engine.persist_managed_indices();
+ Json(single_index_success()).into_response()
+ } else {
+ Json(not_managed_failure(
+ &index,
+ "This index is not being managed.",
+ ))
+ .into_response()
+ }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// POST /_plugins/_ism/change_policy/{index}
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[derive(Debug, Deserialize)]
+pub struct IncludeState {
+ pub state: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct ChangePolicyBody {
+ pub policy_id: String,
+ #[serde(default)]
+ pub state: Option<String>,
+ #[serde(default)]
+ pub include: Vec<IncludeState>,
+}
+
+/// Real ISM's `change_policy` doesn't apply immediately mid-step; xerj has
+/// no concept of a mid-step (states execute all their actions, then
+/// transition), so this applies as soon as the request lands. `include` is
+/// still honored as the safety check it's documented as: if given, the
+/// index's *current* state name must be one of the listed states, or the
+/// change is refused — this is what stops a well-intentioned bulk
+/// `change_policy` call from silently reassigning indices that already
+/// moved on to a state the caller didn't expect.
+pub async fn change_ism_policy(
+ State(state): State<AppState>,
+ Path(index): Path<String>,
+ Json(body): Json<ChangePolicyBody>,
+) -> impl IntoResponse {
+ let Some(mut managed) = state
+ .engine
+ .managed_indices
+ .get(&index)
+ .map(|e| e.value().clone())
+ else {
+ return Json(not_managed_failure(
+ &index,
+ "This index is not being managed.",
+ ))
+ .into_response();
+ };
+
+ if !body.include.is_empty()
+ && !body
+ .include
+ .iter()
+ .any(|i| i.state == managed.current_state)
+ {
+ return Json(not_managed_failure(
+ &index,
+ &format!(
+ "Index is currently in state '{}', which does not match the provided 'include' safety condition.",
+ managed.current_state
+ ),
+ ))
+ .into_response();
+ }
+
+ let Some(new_policy) = state
+ .engine
+ .ism_policies
+ .get(&body.policy_id)
+ .map(|e| e.value().clone())
+ else {
+ let e = xerj_common::XerjError::index_not_found(format!(
+ "policy [{}] not found",
+ body.policy_id
+ ));
+ return ApiError::new(e).into_response();
+ };
+
+ let target_state = match &body.state {
+ Some(s) if new_policy.state(s).is_some() => s.clone(),
+ Some(s) => {
+ let e = xerj_common::XerjError::invalid_query(format!(
+ "state '{s}' does not exist in policy '{}'",
+ body.policy_id
+ ));
+ return ApiError::new(e).into_response();
+ }
+ None if new_policy.state(&managed.current_state).is_some() => managed.current_state.clone(),
+ None => new_policy.default_state.clone(),
+ };
+
+ managed.policy_id = body.policy_id.clone();
+ managed.current_state = target_state;
+ managed.state_entered_at_ms = lifecycle::now_ms();
+ managed.next_action_index = 0;
+ managed.failed = false;
+ managed.info_message = format!("policy changed to '{}'", body.policy_id);
+ managed.last_updated_ms = lifecycle::now_ms();
+ state.engine.managed_indices.insert(index, managed);
+ state.engine.persist_managed_indices();
+
+ Json(single_index_success()).into_response()
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// POST /_plugins/_ism/retry/{index}
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[derive(Debug, Deserialize, Default)]
+pub struct RetryBody {
+ #[serde(default)]
+ pub state: Option<String>,
+}
+
+/// xerj's engine already retries a failing action on every tick without
+/// operator intervention (see `lifecycle::ManagedIndexState::failed`'s doc
+/// comment — `failed` is visibility, not a halt), so this doesn't need to
+/// "unstick" anything internally. It still needs to exist: OSD's Managed
+/// Indexes table renders a "Retry policy" button whenever `explain` reports
+/// `failed: true`, and that button 404ing is a real, visible break even
+/// though the underlying index was never actually stuck. Real ISM also
+/// rejects retrying an index that isn't currently failed, so this mirrors
+/// that rather than silently succeeding as a no-op.
+pub async fn retry_ism_index(
+ State(state): State<AppState>,
+ Path(index): Path<String>,
+ body: OptionalJson<RetryBody>,
+) -> impl IntoResponse {
+ let body = body.0.unwrap_or_default();
+ let Some(mut managed) = state
+ .engine
+ .managed_indices
+ .get(&index)
+ .map(|e| e.value().clone())
+ else {
+ return Json(not_managed_failure(
+ &index,
+ "This index is not being managed.",
+ ))
+ .into_response();
+ };
+
+ if !managed.failed {
+ return Json(not_managed_failure(
+ &index,
+ "This index is not in a failed state.",
+ ))
+ .into_response();
+ }
+
+ if let Some(s) = &body.state {
+ let Some(policy) = state
+ .engine
+ .ism_policies
+ .get(&managed.policy_id)
+ .map(|e| e.value().clone())
+ else {
+ let e = xerj_common::XerjError::index_not_found(format!(
+ "policy [{}] not found",
+ managed.policy_id
+ ));
+ return ApiError::new(e).into_response();
+ };
+ if policy.state(s).is_none() {
+ let e = xerj_common::XerjError::invalid_query(format!(
+ "state '{s}' does not exist in policy '{}'",
+ managed.policy_id
+ ));
+ return ApiError::new(e).into_response();
+ }
+ managed.current_state = s.clone();
+ managed.state_entered_at_ms = lifecycle::now_ms();
+ }
+ managed.next_action_index = 0;
+ managed.failed = false;
+ managed.info_message = "retrying".to_string();
+ managed.last_updated_ms = lifecycle::now_ms();
+ state.engine.managed_indices.insert(index, managed);
+ state.engine.persist_managed_indices();
+
+ Json(single_index_success()).into_response()
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// GET /_plugins/_ism/explain/{index}
// ─────────────────────────────────────────────────────────────────────────────
@@ -270,3 +498,229 @@ pub async fn explain_ism_index(
}
}
}
+
+#[cfg(test)]
+mod managed_index_action_tests {
+ use super::*;
+ use axum::{body::Body, http::Request, http::StatusCode};
+ use tower::ServiceExt;
+
+ fn test_state() -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ async fn call(
+ app: &axum::Router,
+ method: &str,
+ path: &str,
+ body: Option<Value>,
+ ) -> (StatusCode, Value) {
+ let mut builder = Request::builder().method(method).uri(path);
+ let body = match body {
+ Some(v) => {
+ builder = builder.header("content-type", "application/json");
+ Body::from(v.to_string())
+ }
+ None => Body::empty(),
+ };
+ let response = app
+ .clone()
+ .oneshot(builder.body(body).unwrap())
+ .await
+ .unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let json = if bytes.is_empty() {
+ Value::Null
+ } else {
+ serde_json::from_slice(&bytes).unwrap()
+ };
+ (status, json)
+ }
+
+ fn simple_policy(default_state: &str) -> Value {
+ json!({
+ "policy": {
+ "default_state": default_state,
+ "states": [
+ {"name": default_state, "actions": [], "transitions": []}
+ ]
+ }
+ })
+ }
+
+ #[tokio::test]
+ async fn remove_detaches_a_managed_index_and_is_idempotent_about_reporting_it() {
+ let state = test_state();
+ let app = crate::router::build_es_compat_router(state);
+
+ let (status, _) = call(&app, "PUT", "/remove-test-idx", Some(json!({}))).await;
+ assert_eq!(status, StatusCode::OK);
+ let (status, _) = call(
+ &app,
+ "PUT",
+ "/_plugins/_ism/policies/remove-test-policy",
+ Some(simple_policy("only")),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ let (status, _) = call(
+ &app,
+ "POST",
+ "/_plugins/_ism/add/remove-test-idx",
+ Some(json!({"policy_id": "remove-test-policy"})),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+
+ let (status, body) =
+ call(&app, "POST", "/_plugins/_ism/remove/remove-test-idx", None).await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["failures"], false);
+ assert_eq!(body["updated_indices"], 1);
+
+ // Removing again reports the (now expected) "not managed" failure
+ // rather than pretending success or 404ing.
+ let (status, body) =
+ call(&app, "POST", "/_plugins/_ism/remove/remove-test-idx", None).await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["failures"], true);
+ assert_eq!(body["failed_indices"][0]["index_name"], "remove-test-idx");
+ }
+
+ #[tokio::test]
+ async fn change_policy_switches_policy_and_honors_include_safety_check() {
+ let state = test_state();
+ let app = crate::router::build_es_compat_router(state);
+
+ call(&app, "PUT", "/change-policy-idx", Some(json!({}))).await;
+ call(
+ &app,
+ "PUT",
+ "/_plugins/_ism/policies/policy-a",
+ Some(simple_policy("state-a")),
+ )
+ .await;
+ call(
+ &app,
+ "PUT",
+ "/_plugins/_ism/policies/policy-b",
+ Some(simple_policy("state-b")),
+ )
+ .await;
+ call(
+ &app,
+ "POST",
+ "/_plugins/_ism/add/change-policy-idx",
+ Some(json!({"policy_id": "policy-a"})),
+ )
+ .await;
+
+ // Wrong `include` safety condition (index is in "state-a", not
+ // "some-other-state") must refuse the change.
+ let (status, body) = call(
+ &app,
+ "POST",
+ "/_plugins/_ism/change_policy/change-policy-idx",
+ Some(json!({
+ "policy_id": "policy-b",
+ "include": [{"state": "some-other-state"}]
+ })),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["failures"], true);
+
+ let (status, body) = call(
+ &app,
+ "POST",
+ "/_plugins/_ism/change_policy/change-policy-idx",
+ Some(json!({"policy_id": "policy-b"})),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["failures"], false);
+
+ let (_, explain) = call(
+ &app,
+ "GET",
+ "/_plugins/_ism/explain/change-policy-idx",
+ None,
+ )
+ .await;
+ assert_eq!(
+ explain["change-policy-idx"]["policy_id"], "policy-b",
+ "explain: {explain}"
+ );
+ assert_eq!(
+ explain["change-policy-idx"]["state"]["name"], "state-b",
+ "explain: {explain}"
+ );
+ }
+
+ #[tokio::test]
+ async fn retry_clears_failed_and_rejects_a_non_failed_index() {
+ let state = test_state();
+ // `AppState`'s `engine` field is an `Arc`, so this handle keeps
+ // reaching the same `managed_indices` map the router (built from
+ // `state` below, consumed by value) operates on — the same
+ // technique `passage_scroll_tests` uses to seed engine state
+ // directly instead of only driving it through HTTP.
+ let state_handle = state.clone();
+ let app = crate::router::build_es_compat_router(state);
+
+ call(&app, "PUT", "/retry-idx", Some(json!({}))).await;
+ call(
+ &app,
+ "PUT",
+ "/_plugins/_ism/policies/retry-policy",
+ Some(simple_policy("only")),
+ )
+ .await;
+ call(
+ &app,
+ "POST",
+ "/_plugins/_ism/add/retry-idx",
+ Some(json!({"policy_id": "retry-policy"})),
+ )
+ .await;
+
+ // Not failed yet — retry must refuse, matching real ISM.
+ let (status, body) = call(&app, "POST", "/_plugins/_ism/retry/retry-idx", None).await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["failures"], true);
+
+ // Force the managed index into a failed state directly (this is
+ // what `lifecycle::tick` does internally on a real action error —
+ // simulated here rather than driving a real failing action).
+ {
+ let mut managed = state_handle
+ .engine
+ .managed_indices
+ .get_mut("retry-idx")
+ .expect("managed index exists");
+ managed.failed = true;
+ managed.info_message = "simulated failure for test".to_string();
+ }
+
+ let (status, body) = call(&app, "POST", "/_plugins/_ism/retry/retry-idx", None).await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["failures"], false, "body: {body}");
+ assert!(
+ !state_handle
+ .engine
+ .managed_indices
+ .get("retry-idx")
+ .unwrap()
+ .failed
+ );
+ }
+}
diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rs
index c77de4206..cc5bd44a1 100644
--- a/engine/crates/xerj-api/src/router.rs
+++ b/engine/crates/xerj-api/src/router.rs
@@ -467,6 +467,18 @@ pub fn build_es_compat_router(state: AppState) -> Router {
)
.route("/_plugins/_ism/policies", get(ism_api::list_ism_policies))
.route("/_plugins/_ism/add/:index", post(ism_api::add_ism_policy))
+ .route(
+ "/_plugins/_ism/remove/:index",
+ post(ism_api::remove_ism_policy),
+ )
+ .route(
+ "/_plugins/_ism/change_policy/:index",
+ post(ism_api::change_ism_policy),
+ )
+ .route(
+ "/_plugins/_ism/retry/:index",
+ post(ism_api::retry_ism_index),
+ )
.route(
"/_plugins/_ism/explain/:index",
get(ism_api::explain_ism_index),