AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(api): implement missing ISM remove/change_policy/retry endpoints
Vincenzo Lombardo393291197fb5
diff --git a/engine/crates/xerj-api/src/ism_api.rs b/engine/crates/xerj-api/src/ism_api.rsindex f7062e2fa..0f280c5c3 100644--- a/engine/crates/xerj-api/src/ism_api.rs+++ b/engine/crates/xerj-api/src/ism_api.rs@@ -10,9 +10,17 @@//! after live OSD UI verification showed the "State//! management policies" table 500ing without it)//! POST /_plugins/_ism/add/{index}+//! POST /_plugins/_ism/remove/{index}+//! POST /_plugins/_ism/change_policy/{index}+//! POST /_plugins/_ism/retry/{index}//! GET /_plugins/_ism/explain/{index}//! GET /_plugins/_ism/explain (list-all — same//! reason, for the "Policy managed indexes" table)+//!+//! `remove`/`change_policy`/`retry` were added after a systematic pass over+//! OSD's Index Management UI actions (the "Remove policy" / "Change policy"+//! / "Retry policy" buttons on the Managed Indexes table) found they had no+//! backend at all — same discovery method as the list-form gaps above.use axum::{extract::{Path, Query, State},@@ -24,7 +32,29 @@ use serde_json::{json, Value};use xerj_engine::lifecycle::{self, LifecyclePolicy, ManagedIndexState};-use crate::{error::ApiError, state::AppState};+use crate::{error::ApiError, extract::OptionalJson, state::AppState};++/// The `{"failures": bool, "updated_indices": N, "failed_indices": [...]}`+/// shape real ISM's bulk-style index-targeting endpoints (`add`, `remove`,+/// `change_policy`, `retry`) all share. A single-index failure here is a+/// normal, expected outcome (e.g. "not currently managed") — not an error+/// worth a non-200 status, matching how OSD's own client reads the body's+/// `failures` flag rather than the HTTP status to decide success.+fn not_managed_failure(index: &str, reason: &str) -> Value {+ json!({+ "failures": true,+ "updated_indices": 0,+ "failed_indices": [{+ "index_name": index,+ "index_uuid": "_na_",+ "reason": reason,+ }],+ })+}++fn single_index_success() -> Value {+ json!({ "failures": false, "updated_indices": 1, "failed_indices": [] })+}// ─────────────────────────────────────────────────────────────────────────────// PUT/GET/DELETE /_plugins/_ism/policies/{policy_id}@@ -212,6 +242,204 @@ pub async fn attach_policy(state: &AppState, index: &str, policy_id: &str) -> Re.into_response()}+// ─────────────────────────────────────────────────────────────────────────────+// POST /_plugins/_ism/remove/{index}+// ─────────────────────────────────────────────────────────────────────────────++pub async fn remove_ism_policy(+ State(state): State<AppState>,+ Path(index): Path<String>,+) -> impl IntoResponse {+ if state.engine.managed_indices.remove(&index).is_some() {+ state.engine.persist_managed_indices();+ Json(single_index_success()).into_response()+ } else {+ Json(not_managed_failure(+ &index,+ "This index is not being managed.",+ ))+ .into_response()+ }+}++// ─────────────────────────────────────────────────────────────────────────────+// POST /_plugins/_ism/change_policy/{index}+// ─────────────────────────────────────────────────────────────────────────────++#[derive(Debug, Deserialize)]+pub struct IncludeState {+ pub state: String,+}++#[derive(Debug, Deserialize)]+pub struct ChangePolicyBody {+ pub policy_id: String,+ #[serde(default)]+ pub state: Option<String>,+ #[serde(default)]+ pub include: Vec<IncludeState>,+}++/// Real ISM's `change_policy` doesn't apply immediately mid-step; xerj has+/// no concept of a mid-step (states execute all their actions, then+/// transition), so this applies as soon as the request lands. `include` is+/// still honored as the safety check it's documented as: if given, the+/// index's *current* state name must be one of the listed states, or the+/// change is refused — this is what stops a well-intentioned bulk+/// `change_policy` call from silently reassigning indices that already+/// moved on to a state the caller didn't expect.+pub async fn change_ism_policy(+ State(state): State<AppState>,+ Path(index): Path<String>,+ Json(body): Json<ChangePolicyBody>,+) -> impl IntoResponse {+ let Some(mut managed) = state+ .engine+ .managed_indices+ .get(&index)+ .map(|e| e.value().clone())+ else {+ return Json(not_managed_failure(+ &index,+ "This index is not being managed.",+ ))+ .into_response();+ };++ if !body.include.is_empty()+ && !body+ .include+ .iter()+ .any(|i| i.state == managed.current_state)+ {+ return Json(not_managed_failure(+ &index,+ &format!(+ "Index is currently in state '{}', which does not match the provided 'include' safety condition.",+ managed.current_state+ ),+ ))+ .into_response();+ }++ let Some(new_policy) = state+ .engine+ .ism_policies+ .get(&body.policy_id)+ .map(|e| e.value().clone())+ else {+ let e = xerj_common::XerjError::index_not_found(format!(+ "policy [{}] not found",+ body.policy_id+ ));+ return ApiError::new(e).into_response();+ };++ let target_state = match &body.state {+ Some(s) if new_policy.state(s).is_some() => s.clone(),+ Some(s) => {+ let e = xerj_common::XerjError::invalid_query(format!(+ "state '{s}' does not exist in policy '{}'",+ body.policy_id+ ));+ return ApiError::new(e).into_response();+ }+ None if new_policy.state(&managed.current_state).is_some() => managed.current_state.clone(),+ None => new_policy.default_state.clone(),+ };++ managed.policy_id = body.policy_id.clone();+ managed.current_state = target_state;+ managed.state_entered_at_ms = lifecycle::now_ms();+ managed.next_action_index = 0;+ managed.failed = false;+ managed.info_message = format!("policy changed to '{}'", body.policy_id);+ managed.last_updated_ms = lifecycle::now_ms();+ state.engine.managed_indices.insert(index, managed);+ state.engine.persist_managed_indices();++ Json(single_index_success()).into_response()+}++// ─────────────────────────────────────────────────────────────────────────────+// POST /_plugins/_ism/retry/{index}+// ─────────────────────────────────────────────────────────────────────────────++#[derive(Debug, Deserialize, Default)]+pub struct RetryBody {+ #[serde(default)]+ pub state: Option<String>,+}++/// xerj's engine already retries a failing action on every tick without+/// operator intervention (see `lifecycle::ManagedIndexState::failed`'s doc+/// comment — `failed` is visibility, not a halt), so this doesn't need to+/// "unstick" anything internally. It still needs to exist: OSD's Managed+/// Indexes table renders a "Retry policy" button whenever `explain` reports+/// `failed: true`, and that button 404ing is a real, visible break even+/// though the underlying index was never actually stuck. Real ISM also+/// rejects retrying an index that isn't currently failed, so this mirrors+/// that rather than silently succeeding as a no-op.+pub async fn retry_ism_index(+ State(state): State<AppState>,+ Path(index): Path<String>,+ body: OptionalJson<RetryBody>,+) -> impl IntoResponse {+ let body = body.0.unwrap_or_default();+ let Some(mut managed) = state+ .engine+ .managed_indices+ .get(&index)+ .map(|e| e.value().clone())+ else {+ return Json(not_managed_failure(+ &index,+ "This index is not being managed.",+ ))+ .into_response();+ };++ if !managed.failed {+ return Json(not_managed_failure(+ &index,+ "This index is not in a failed state.",+ ))+ .into_response();+ }++ if let Some(s) = &body.state {+ let Some(policy) = state+ .engine+ .ism_policies+ .get(&managed.policy_id)+ .map(|e| e.value().clone())+ else {+ let e = xerj_common::XerjError::index_not_found(format!(+ "policy [{}] not found",+ managed.policy_id+ ));+ return ApiError::new(e).into_response();+ };+ if policy.state(s).is_none() {+ let e = xerj_common::XerjError::invalid_query(format!(+ "state '{s}' does not exist in policy '{}'",+ managed.policy_id+ ));+ return ApiError::new(e).into_response();+ }+ managed.current_state = s.clone();+ managed.state_entered_at_ms = lifecycle::now_ms();+ }+ managed.next_action_index = 0;+ managed.failed = false;+ managed.info_message = "retrying".to_string();+ managed.last_updated_ms = lifecycle::now_ms();+ state.engine.managed_indices.insert(index, managed);+ state.engine.persist_managed_indices();++ Json(single_index_success()).into_response()+}+// ─────────────────────────────────────────────────────────────────────────────// GET /_plugins/_ism/explain/{index}// ─────────────────────────────────────────────────────────────────────────────@@ -270,3 +498,229 @@ pub async fn explain_ism_index(}}}++#[cfg(test)]+mod managed_index_action_tests {+ use super::*;+ use axum::{body::Body, http::Request, http::StatusCode};+ use tower::ServiceExt;++ fn test_state() -> AppState {+ let dir = tempfile::tempdir().expect("tempdir").keep();+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ AppState::new(config, engine, metrics)+ }++ async fn call(+ app: &axum::Router,+ method: &str,+ path: &str,+ body: Option<Value>,+ ) -> (StatusCode, Value) {+ let mut builder = Request::builder().method(method).uri(path);+ let body = match body {+ Some(v) => {+ builder = builder.header("content-type", "application/json");+ Body::from(v.to_string())+ }+ None => Body::empty(),+ };+ let response = app+ .clone()+ .oneshot(builder.body(body).unwrap())+ .await+ .unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ let json = if bytes.is_empty() {+ Value::Null+ } else {+ serde_json::from_slice(&bytes).unwrap()+ };+ (status, json)+ }++ fn simple_policy(default_state: &str) -> Value {+ json!({+ "policy": {+ "default_state": default_state,+ "states": [+ {"name": default_state, "actions": [], "transitions": []}+ ]+ }+ })+ }++ #[tokio::test]+ async fn remove_detaches_a_managed_index_and_is_idempotent_about_reporting_it() {+ let state = test_state();+ let app = crate::router::build_es_compat_router(state);++ let (status, _) = call(&app, "PUT", "/remove-test-idx", Some(json!({}))).await;+ assert_eq!(status, StatusCode::OK);+ let (status, _) = call(+ &app,+ "PUT",+ "/_plugins/_ism/policies/remove-test-policy",+ Some(simple_policy("only")),+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ let (status, _) = call(+ &app,+ "POST",+ "/_plugins/_ism/add/remove-test-idx",+ Some(json!({"policy_id": "remove-test-policy"})),+ )+ .await;+ assert_eq!(status, StatusCode::OK);++ let (status, body) =+ call(&app, "POST", "/_plugins/_ism/remove/remove-test-idx", None).await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["failures"], false);+ assert_eq!(body["updated_indices"], 1);++ // Removing again reports the (now expected) "not managed" failure+ // rather than pretending success or 404ing.+ let (status, body) =+ call(&app, "POST", "/_plugins/_ism/remove/remove-test-idx", None).await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["failures"], true);+ assert_eq!(body["failed_indices"][0]["index_name"], "remove-test-idx");+ }++ #[tokio::test]+ async fn change_policy_switches_policy_and_honors_include_safety_check() {+ let state = test_state();+ let app = crate::router::build_es_compat_router(state);++ call(&app, "PUT", "/change-policy-idx", Some(json!({}))).await;+ call(+ &app,+ "PUT",+ "/_plugins/_ism/policies/policy-a",+ Some(simple_policy("state-a")),+ )+ .await;+ call(+ &app,+ "PUT",+ "/_plugins/_ism/policies/policy-b",+ Some(simple_policy("state-b")),+ )+ .await;+ call(+ &app,+ "POST",+ "/_plugins/_ism/add/change-policy-idx",+ Some(json!({"policy_id": "policy-a"})),+ )+ .await;++ // Wrong `include` safety condition (index is in "state-a", not+ // "some-other-state") must refuse the change.+ let (status, body) = call(+ &app,+ "POST",+ "/_plugins/_ism/change_policy/change-policy-idx",+ Some(json!({+ "policy_id": "policy-b",+ "include": [{"state": "some-other-state"}]+ })),+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["failures"], true);++ let (status, body) = call(+ &app,+ "POST",+ "/_plugins/_ism/change_policy/change-policy-idx",+ Some(json!({"policy_id": "policy-b"})),+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["failures"], false);++ let (_, explain) = call(+ &app,+ "GET",+ "/_plugins/_ism/explain/change-policy-idx",+ None,+ )+ .await;+ assert_eq!(+ explain["change-policy-idx"]["policy_id"], "policy-b",+ "explain: {explain}"+ );+ assert_eq!(+ explain["change-policy-idx"]["state"]["name"], "state-b",+ "explain: {explain}"+ );+ }++ #[tokio::test]+ async fn retry_clears_failed_and_rejects_a_non_failed_index() {+ let state = test_state();+ // `AppState`'s `engine` field is an `Arc`, so this handle keeps+ // reaching the same `managed_indices` map the router (built from+ // `state` below, consumed by value) operates on — the same+ // technique `passage_scroll_tests` uses to seed engine state+ // directly instead of only driving it through HTTP.+ let state_handle = state.clone();+ let app = crate::router::build_es_compat_router(state);++ call(&app, "PUT", "/retry-idx", Some(json!({}))).await;+ call(+ &app,+ "PUT",+ "/_plugins/_ism/policies/retry-policy",+ Some(simple_policy("only")),+ )+ .await;+ call(+ &app,+ "POST",+ "/_plugins/_ism/add/retry-idx",+ Some(json!({"policy_id": "retry-policy"})),+ )+ .await;++ // Not failed yet — retry must refuse, matching real ISM.+ let (status, body) = call(&app, "POST", "/_plugins/_ism/retry/retry-idx", None).await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["failures"], true);++ // Force the managed index into a failed state directly (this is+ // what `lifecycle::tick` does internally on a real action error —+ // simulated here rather than driving a real failing action).+ {+ let mut managed = state_handle+ .engine+ .managed_indices+ .get_mut("retry-idx")+ .expect("managed index exists");+ managed.failed = true;+ managed.info_message = "simulated failure for test".to_string();+ }++ let (status, body) = call(&app, "POST", "/_plugins/_ism/retry/retry-idx", None).await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["failures"], false, "body: {body}");+ assert!(+ !state_handle+ .engine+ .managed_indices+ .get("retry-idx")+ .unwrap()+ .failed+ );+ }+}diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rsindex c77de4206..cc5bd44a1 100644--- a/engine/crates/xerj-api/src/router.rs+++ b/engine/crates/xerj-api/src/router.rs@@ -467,6 +467,18 @@ pub fn build_es_compat_router(state: AppState) -> Router {).route("/_plugins/_ism/policies", get(ism_api::list_ism_policies)).route("/_plugins/_ism/add/:index", post(ism_api::add_ism_policy))+ .route(+ "/_plugins/_ism/remove/:index",+ post(ism_api::remove_ism_policy),+ )+ .route(+ "/_plugins/_ism/change_policy/:index",+ post(ism_api::change_ism_policy),+ )+ .route(+ "/_plugins/_ism/retry/:index",+ post(ism_api::retry_ism_index),+ ).route("/_plugins/_ism/explain/:index",get(ism_api::explain_ism_index),