AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): honor index.lifecycle.name: null as a real ILM detach
Vincenzo Lombardoa4f0be74db69
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 3d84e0549..e5c9e3d44 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -1142,7 +1142,9 @@ pub async fn create_index(// silent on a missing/untranslatable policy — real ILM is// equally lenient here: it discovers the policy by name lazily// and simply doesn't start managing until one exists.- if let Some(policy_name) = ilm_policy_name_from_settings(&body) {+ if let LifecycleDirective::Attach(policy_name) =+ ilm_lifecycle_directive_from_settings(&body)+ {let _ = crate::ism_api::attach_policy(&state, &index, &policy_name).await;}@@ -22114,28 +22116,60 @@ pub async fn put_settings(// Same lazy, best-effort `index.lifecycle.name` attach as create_index —// `PUT {index}/_settings {"index.lifecycle.name": "policy"}` is the// other real ILM entry point (attaching to an index that already- // exists, not just at creation).- if let Some(policy_name) = ilm_policy_name_from_settings(&body) {- for idx in &targets {- let _ = crate::ism_api::attach_policy(&state, idx, &policy_name).await;+ // exists, not just at creation) — and, symmetrically, this is also+ // where a real client detaches: `{"index.lifecycle.name": null}`.+ match ilm_lifecycle_directive_from_settings(&body) {+ LifecycleDirective::Attach(policy_name) => {+ for idx in &targets {+ let _ = crate::ism_api::attach_policy(&state, idx, &policy_name).await;+ }+ }+ LifecycleDirective::Detach => {+ for idx in &targets {+ crate::ism_api::detach_ilm_policy(&state, idx);+ }}+ LifecycleDirective::Absent => {}}Json(json!({ "acknowledged": true })).into_response()}-/// Extract `index.lifecycle.name` from an index-settings body, accepting-/// both the nested (`{"index":{"lifecycle":{"name":"..."}}}`,+/// What a settings body says about `index.lifecycle.name`. A bare+/// `Option<String>` cannot tell "the key was not mentioned" apart from+/// "the key was explicitly set to `null`" — and those mean opposite things:+/// the first is "leave lifecycle management alone", the second is "stop+/// managing this index" (real ES's documented detach idiom). Collapsing+/// them into one `None` is exactly the bug found in #262 (closed as+/// superseded by this engine, but its finding stood): a client's+/// `PUT {index}/_settings {"index.lifecycle.name": null}` got back+/// `{"acknowledged": true}` while the index stayed attached and was later+/// deleted by its policy anyway — confirmed live against this engine before+/// this fix, `GET` on the "detached" index returned 404 after the next+/// lifecycle tick.+enum LifecycleDirective {+ Absent,+ Detach,+ Attach(String),+}++/// Extract the `index.lifecycle.name` directive from an index-settings body,+/// accepting both the nested (`{"index":{"lifecycle":{"name":"..."}}}`,/// `{"settings":{...}}`-wrapped at create time) and flat dotted-key/// (`{"index.lifecycle.name":"..."}`) forms ES accepts for every index-/// setting.-fn ilm_policy_name_from_settings(body: &Value) -> Option<String> {+/// setting. A non-string, non-null value (malformed input) is treated as+/// absent rather than erroring — matches the pre-existing leniency this+/// function already had for every other shape it didn't recognize.+fn ilm_lifecycle_directive_from_settings(body: &Value) -> LifecycleDirective {let settings = body.get("settings").unwrap_or(body);- settings+ let raw = settings.pointer("/index/lifecycle/name")- .or_else(|| settings.get("index.lifecycle.name"))- .and_then(Value::as_str)- .map(String::from)+ .or_else(|| settings.get("index.lifecycle.name"));+ match raw {+ Some(Value::String(s)) => LifecycleDirective::Attach(s.clone()),+ Some(Value::Null) => LifecycleDirective::Detach,+ _ => LifecycleDirective::Absent,+ }}// ─────────────────────────────────────────────────────────────────────────────@@ -24223,6 +24257,25 @@ pub async fn ilm_explain(Json(json!({ "indices": Value::Object(indices) })).into_response()}+/// `POST {index}/_ilm/remove` — real ES's dedicated detach verb, sharing+/// the same removal path the `index.lifecycle.name: null` settings+/// directive uses (`ism_api::detach_ilm_policy`). Validates the index+/// exists first (404 otherwise) rather than silently acknowledging a call+/// against a name that was never real — the same accept-and-ignore shape+/// #262 found and fixed on this exact endpoint in its own (closed,+/// superseded) implementation.+pub async fn ilm_remove(+ State(state): State<AppState>,+ Path(index): Path<String>,+) -> impl IntoResponse {+ if state.engine.get_index(&index).is_err() {+ let e = xerj_common::XerjError::index_not_found(&index);+ return ApiError::new(e).into_response();+ }+ crate::ism_api::detach_ilm_policy(&state, &index);+ Json(json!({ "has_failures": false, "failed_indexes": [] })).into_response()+}+// ─────────────────────────────────────────────────────────────────────────────// Component Templates// PUT /_component_template/{name}diff --git a/engine/crates/xerj-api/src/ism_api.rs b/engine/crates/xerj-api/src/ism_api.rsindex 0f280c5c3..2f2d7bccb 100644--- a/engine/crates/xerj-api/src/ism_api.rs+++ b/engine/crates/xerj-api/src/ism_api.rs@@ -242,6 +242,27 @@ pub async fn attach_policy(state: &AppState, index: &str, policy_id: &str) -> Re.into_response()}+/// Shared detach implementation — the counterpart to `attach_policy`, used+/// by both the ES-shape `index.lifecycle.name: null` settings directive+/// (`es_compat::put_settings`) and `POST {index}/_ilm/remove` (real ES's+/// own dedicated detach verb). Removing the `managed_indices` entry is+/// authoritative here: nothing else in this engine ever re-derives "is+/// this index managed" from stored index settings, so there is no+/// fallback path that could resurrect a detached index — unlike the+/// architecture #262 found this exact defect in, which needed a separate+/// persisted tombstone because its resolver fell back to settings.json+/// when its own map had no entry. Here, absence from `managed_indices` IS+/// unambiguously "not managed", so a plain removal is the whole fix.+/// Returns whether the index was actually managed (false is not an error —+/// detaching an already-unmanaged index is idempotent, matching real ES).+pub fn detach_ilm_policy(state: &AppState, index: &str) -> bool {+ let removed = state.engine.managed_indices.remove(index).is_some();+ if removed {+ state.engine.persist_managed_indices();+ }+ removed+}+// ─────────────────────────────────────────────────────────────────────────────// POST /_plugins/_ism/remove/{index}// ─────────────────────────────────────────────────────────────────────────────@@ -723,4 +744,145 @@ mod managed_index_action_tests {.failed);}++ /// #262 (a since-closed, independent ILM implementation superseded by+ /// this one) found that an acknowledged+ /// `PUT {index}/_settings {"index.lifecycle.name": null}` detach was+ /// silently ignored, and the index was deleted by its policy anyway on+ /// the next pass. Confirmed live against this engine before this fix:+ /// the detach call returned `acknowledged: true`, `_ilm/explain` still+ /// showed `managed: true`, and the index was gone (404) after the next+ /// lifecycle tick. This test reproduces that exact scenario through the+ /// real HTTP router and asserts the index survives.+ #[tokio::test]+ async fn settings_null_detach_actually_prevents_the_scheduled_delete() {+ let state = test_state();+ let state_handle = state.clone();+ let app = crate::router::build_es_compat_router(state);++ call(&app, "PUT", "/detach-idx", Some(json!({}))).await;+ call(&app, "PUT", "/control-idx", Some(json!({}))).await;+ // A policy whose one and only action is an unconditional delete —+ // the next tick deletes any index still pointing at it.+ let delete_policy = json!({+ "policy": {+ "phases": {+ "delete": {"min_age": "0ms", "actions": {"delete": {}}}+ }+ }+ });+ call(&app, "PUT", "/_ilm/policy/delete-now", Some(delete_policy)).await;+ call(+ &app,+ "PUT",+ "/detach-idx/_settings",+ Some(json!({"index.lifecycle.name": "delete-now"})),+ )+ .await;+ call(+ &app,+ "PUT",+ "/control-idx/_settings",+ Some(json!({"index.lifecycle.name": "delete-now"})),+ )+ .await;++ let (_, explain) = call(&app, "GET", "/detach-idx/_ilm/explain", None).await;+ assert_eq!(+ explain["indices"]["detach-idx"]["managed"], true,+ "explain: {explain}"+ );++ // Detach exactly the way a real client does it.+ let (status, body) = call(+ &app,+ "PUT",+ "/detach-idx/_settings",+ Some(json!({"index.lifecycle.name": null})),+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["acknowledged"], true);++ let (_, explain) = call(&app, "GET", "/detach-idx/_ilm/explain", None).await;+ assert_eq!(+ explain["indices"]["detach-idx"]["managed"], false,+ "detach must be reflected immediately, not just acknowledged: {explain}"+ );++ // Drive one real lifecycle pass — the same code path a live+ // background tick would run.+ xerj_engine::lifecycle::tick(&state_handle.engine).await;++ let get_detach = call(&app, "GET", "/detach-idx", None).await;+ assert_eq!(+ get_detach.0,+ StatusCode::OK,+ "the detached index must survive the pass that would have deleted it"+ );+ let get_control = call(&app, "GET", "/control-idx", None).await;+ assert_eq!(+ get_control.0,+ StatusCode::NOT_FOUND,+ "the control index (never detached) must actually be deleted by the same pass \+ — otherwise this test would pass even if detach were a no-op"+ );+ }++ #[tokio::test]+ async fn ilm_remove_detaches_and_404s_on_a_nonexistent_index() {+ let state = test_state();+ let app = crate::router::build_es_compat_router(state);++ call(&app, "PUT", "/remove-verb-idx", Some(json!({}))).await;+ call(+ &app,+ "PUT",+ "/_ilm/policy/remove-verb-policy",+ Some(simple_policy("only")),+ )+ .await;+ call(+ &app,+ "PUT",+ "/remove-verb-idx/_settings",+ Some(json!({"index.lifecycle.name": "remove-verb-policy"})),+ )+ .await;++ let (status, body) = call(&app, "POST", "/remove-verb-idx/_ilm/remove", None).await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["has_failures"], false);++ let (_, explain) = call(&app, "GET", "/remove-verb-idx/_ilm/explain", None).await;+ assert_eq!(explain["indices"]["remove-verb-idx"]["managed"], false);++ // Real ES's `ignore_unavailable=false` default: a name that was+ // never an index 404s rather than silently acknowledging — the+ // same accept-and-ignore shape #262 found on this exact endpoint+ // in its own implementation.+ let (status, _) = call(&app, "POST", "/does-not-exist/_ilm/remove", None).await;+ assert_eq!(status, StatusCode::NOT_FOUND);+ }++ #[tokio::test]+ async fn detaching_an_already_unmanaged_index_is_idempotent_not_an_error() {+ let state = test_state();+ let app = crate::router::build_es_compat_router(state);+ call(&app, "PUT", "/never-managed-idx", Some(json!({}))).await;++ let (status, body) = call(+ &app,+ "PUT",+ "/never-managed-idx/_settings",+ Some(json!({"index.lifecycle.name": null})),+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["acknowledged"], true);++ let (status, body) = call(&app, "POST", "/never-managed-idx/_ilm/remove", None).await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["has_failures"], false);+ }}diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rsindex 69acda4f9..d64512812 100644--- a/engine/crates/xerj-api/src/router.rs+++ b/engine/crates/xerj-api/src/router.rs@@ -483,6 +483,7 @@ pub fn build_es_compat_router(state: AppState) -> Router {.delete(es_compat::delete_ilm_policy),).route("/:index/_ilm/explain", get(es_compat::ilm_explain))+ .route("/:index/_ilm/remove", post(es_compat::ilm_remove))// ── ISM (OpenSearch Index State Management) ─────────────────────────// Same execution engine as ILM above — see `xerj_engine::lifecycle`..route(