Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): honor index.lifecycle.name: null as a real ILM detach

Vincenzo Lombardoa4f0be74db69
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 3d84e0549..e5c9e3d44 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -1142,7 +1142,9 @@ pub async fn create_index(
// silent on a missing/untranslatable policy — real ILM is
// equally lenient here: it discovers the policy by name lazily
// and simply doesn't start managing until one exists.
- if let Some(policy_name) = ilm_policy_name_from_settings(&body) {
+ if let LifecycleDirective::Attach(policy_name) =
+ ilm_lifecycle_directive_from_settings(&body)
+ {
let _ = crate::ism_api::attach_policy(&state, &index, &policy_name).await;
}
@@ -22114,28 +22116,60 @@ pub async fn put_settings(
// Same lazy, best-effort `index.lifecycle.name` attach as create_index —
// `PUT {index}/_settings {"index.lifecycle.name": "policy"}` is the
// other real ILM entry point (attaching to an index that already
- // exists, not just at creation).
- if let Some(policy_name) = ilm_policy_name_from_settings(&body) {
- for idx in &targets {
- let _ = crate::ism_api::attach_policy(&state, idx, &policy_name).await;
+ // exists, not just at creation) — and, symmetrically, this is also
+ // where a real client detaches: `{"index.lifecycle.name": null}`.
+ match ilm_lifecycle_directive_from_settings(&body) {
+ LifecycleDirective::Attach(policy_name) => {
+ for idx in &targets {
+ let _ = crate::ism_api::attach_policy(&state, idx, &policy_name).await;
+ }
+ }
+ LifecycleDirective::Detach => {
+ for idx in &targets {
+ crate::ism_api::detach_ilm_policy(&state, idx);
+ }
}
+ LifecycleDirective::Absent => {}
}
Json(json!({ "acknowledged": true })).into_response()
}
-/// Extract `index.lifecycle.name` from an index-settings body, accepting
-/// both the nested (`{"index":{"lifecycle":{"name":"..."}}}`,
+/// What a settings body says about `index.lifecycle.name`. A bare
+/// `Option<String>` cannot tell "the key was not mentioned" apart from
+/// "the key was explicitly set to `null`" — and those mean opposite things:
+/// the first is "leave lifecycle management alone", the second is "stop
+/// managing this index" (real ES's documented detach idiom). Collapsing
+/// them into one `None` is exactly the bug found in #262 (closed as
+/// superseded by this engine, but its finding stood): a client's
+/// `PUT {index}/_settings {"index.lifecycle.name": null}` got back
+/// `{"acknowledged": true}` while the index stayed attached and was later
+/// deleted by its policy anyway — confirmed live against this engine before
+/// this fix, `GET` on the "detached" index returned 404 after the next
+/// lifecycle tick.
+enum LifecycleDirective {
+ Absent,
+ Detach,
+ Attach(String),
+}
+
+/// Extract the `index.lifecycle.name` directive from an index-settings body,
+/// accepting both the nested (`{"index":{"lifecycle":{"name":"..."}}}`,
/// `{"settings":{...}}`-wrapped at create time) and flat dotted-key
/// (`{"index.lifecycle.name":"..."}`) forms ES accepts for every index
-/// setting.
-fn ilm_policy_name_from_settings(body: &Value) -> Option<String> {
+/// setting. A non-string, non-null value (malformed input) is treated as
+/// absent rather than erroring — matches the pre-existing leniency this
+/// function already had for every other shape it didn't recognize.
+fn ilm_lifecycle_directive_from_settings(body: &Value) -> LifecycleDirective {
let settings = body.get("settings").unwrap_or(body);
- settings
+ let raw = settings
.pointer("/index/lifecycle/name")
- .or_else(|| settings.get("index.lifecycle.name"))
- .and_then(Value::as_str)
- .map(String::from)
+ .or_else(|| settings.get("index.lifecycle.name"));
+ match raw {
+ Some(Value::String(s)) => LifecycleDirective::Attach(s.clone()),
+ Some(Value::Null) => LifecycleDirective::Detach,
+ _ => LifecycleDirective::Absent,
+ }
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -24223,6 +24257,25 @@ pub async fn ilm_explain(
Json(json!({ "indices": Value::Object(indices) })).into_response()
}
+/// `POST {index}/_ilm/remove` — real ES's dedicated detach verb, sharing
+/// the same removal path the `index.lifecycle.name: null` settings
+/// directive uses (`ism_api::detach_ilm_policy`). Validates the index
+/// exists first (404 otherwise) rather than silently acknowledging a call
+/// against a name that was never real — the same accept-and-ignore shape
+/// #262 found and fixed on this exact endpoint in its own (closed,
+/// superseded) implementation.
+pub async fn ilm_remove(
+ State(state): State<AppState>,
+ Path(index): Path<String>,
+) -> impl IntoResponse {
+ if state.engine.get_index(&index).is_err() {
+ let e = xerj_common::XerjError::index_not_found(&index);
+ return ApiError::new(e).into_response();
+ }
+ crate::ism_api::detach_ilm_policy(&state, &index);
+ Json(json!({ "has_failures": false, "failed_indexes": [] })).into_response()
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Component Templates
// PUT /_component_template/{name}
diff --git a/engine/crates/xerj-api/src/ism_api.rs b/engine/crates/xerj-api/src/ism_api.rs
index 0f280c5c3..2f2d7bccb 100644
--- a/engine/crates/xerj-api/src/ism_api.rs
+++ b/engine/crates/xerj-api/src/ism_api.rs
@@ -242,6 +242,27 @@ pub async fn attach_policy(state: &AppState, index: &str, policy_id: &str) -> Re
.into_response()
}
+/// Shared detach implementation — the counterpart to `attach_policy`, used
+/// by both the ES-shape `index.lifecycle.name: null` settings directive
+/// (`es_compat::put_settings`) and `POST {index}/_ilm/remove` (real ES's
+/// own dedicated detach verb). Removing the `managed_indices` entry is
+/// authoritative here: nothing else in this engine ever re-derives "is
+/// this index managed" from stored index settings, so there is no
+/// fallback path that could resurrect a detached index — unlike the
+/// architecture #262 found this exact defect in, which needed a separate
+/// persisted tombstone because its resolver fell back to settings.json
+/// when its own map had no entry. Here, absence from `managed_indices` IS
+/// unambiguously "not managed", so a plain removal is the whole fix.
+/// Returns whether the index was actually managed (false is not an error —
+/// detaching an already-unmanaged index is idempotent, matching real ES).
+pub fn detach_ilm_policy(state: &AppState, index: &str) -> bool {
+ let removed = state.engine.managed_indices.remove(index).is_some();
+ if removed {
+ state.engine.persist_managed_indices();
+ }
+ removed
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// POST /_plugins/_ism/remove/{index}
// ─────────────────────────────────────────────────────────────────────────────
@@ -723,4 +744,145 @@ mod managed_index_action_tests {
.failed
);
}
+
+ /// #262 (a since-closed, independent ILM implementation superseded by
+ /// this one) found that an acknowledged
+ /// `PUT {index}/_settings {"index.lifecycle.name": null}` detach was
+ /// silently ignored, and the index was deleted by its policy anyway on
+ /// the next pass. Confirmed live against this engine before this fix:
+ /// the detach call returned `acknowledged: true`, `_ilm/explain` still
+ /// showed `managed: true`, and the index was gone (404) after the next
+ /// lifecycle tick. This test reproduces that exact scenario through the
+ /// real HTTP router and asserts the index survives.
+ #[tokio::test]
+ async fn settings_null_detach_actually_prevents_the_scheduled_delete() {
+ let state = test_state();
+ let state_handle = state.clone();
+ let app = crate::router::build_es_compat_router(state);
+
+ call(&app, "PUT", "/detach-idx", Some(json!({}))).await;
+ call(&app, "PUT", "/control-idx", Some(json!({}))).await;
+ // A policy whose one and only action is an unconditional delete —
+ // the next tick deletes any index still pointing at it.
+ let delete_policy = json!({
+ "policy": {
+ "phases": {
+ "delete": {"min_age": "0ms", "actions": {"delete": {}}}
+ }
+ }
+ });
+ call(&app, "PUT", "/_ilm/policy/delete-now", Some(delete_policy)).await;
+ call(
+ &app,
+ "PUT",
+ "/detach-idx/_settings",
+ Some(json!({"index.lifecycle.name": "delete-now"})),
+ )
+ .await;
+ call(
+ &app,
+ "PUT",
+ "/control-idx/_settings",
+ Some(json!({"index.lifecycle.name": "delete-now"})),
+ )
+ .await;
+
+ let (_, explain) = call(&app, "GET", "/detach-idx/_ilm/explain", None).await;
+ assert_eq!(
+ explain["indices"]["detach-idx"]["managed"], true,
+ "explain: {explain}"
+ );
+
+ // Detach exactly the way a real client does it.
+ let (status, body) = call(
+ &app,
+ "PUT",
+ "/detach-idx/_settings",
+ Some(json!({"index.lifecycle.name": null})),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["acknowledged"], true);
+
+ let (_, explain) = call(&app, "GET", "/detach-idx/_ilm/explain", None).await;
+ assert_eq!(
+ explain["indices"]["detach-idx"]["managed"], false,
+ "detach must be reflected immediately, not just acknowledged: {explain}"
+ );
+
+ // Drive one real lifecycle pass — the same code path a live
+ // background tick would run.
+ xerj_engine::lifecycle::tick(&state_handle.engine).await;
+
+ let get_detach = call(&app, "GET", "/detach-idx", None).await;
+ assert_eq!(
+ get_detach.0,
+ StatusCode::OK,
+ "the detached index must survive the pass that would have deleted it"
+ );
+ let get_control = call(&app, "GET", "/control-idx", None).await;
+ assert_eq!(
+ get_control.0,
+ StatusCode::NOT_FOUND,
+ "the control index (never detached) must actually be deleted by the same pass \
+ — otherwise this test would pass even if detach were a no-op"
+ );
+ }
+
+ #[tokio::test]
+ async fn ilm_remove_detaches_and_404s_on_a_nonexistent_index() {
+ let state = test_state();
+ let app = crate::router::build_es_compat_router(state);
+
+ call(&app, "PUT", "/remove-verb-idx", Some(json!({}))).await;
+ call(
+ &app,
+ "PUT",
+ "/_ilm/policy/remove-verb-policy",
+ Some(simple_policy("only")),
+ )
+ .await;
+ call(
+ &app,
+ "PUT",
+ "/remove-verb-idx/_settings",
+ Some(json!({"index.lifecycle.name": "remove-verb-policy"})),
+ )
+ .await;
+
+ let (status, body) = call(&app, "POST", "/remove-verb-idx/_ilm/remove", None).await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["has_failures"], false);
+
+ let (_, explain) = call(&app, "GET", "/remove-verb-idx/_ilm/explain", None).await;
+ assert_eq!(explain["indices"]["remove-verb-idx"]["managed"], false);
+
+ // Real ES's `ignore_unavailable=false` default: a name that was
+ // never an index 404s rather than silently acknowledging — the
+ // same accept-and-ignore shape #262 found on this exact endpoint
+ // in its own implementation.
+ let (status, _) = call(&app, "POST", "/does-not-exist/_ilm/remove", None).await;
+ assert_eq!(status, StatusCode::NOT_FOUND);
+ }
+
+ #[tokio::test]
+ async fn detaching_an_already_unmanaged_index_is_idempotent_not_an_error() {
+ let state = test_state();
+ let app = crate::router::build_es_compat_router(state);
+ call(&app, "PUT", "/never-managed-idx", Some(json!({}))).await;
+
+ let (status, body) = call(
+ &app,
+ "PUT",
+ "/never-managed-idx/_settings",
+ Some(json!({"index.lifecycle.name": null})),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["acknowledged"], true);
+
+ let (status, body) = call(&app, "POST", "/never-managed-idx/_ilm/remove", None).await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["has_failures"], false);
+ }
}
diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rs
index 69acda4f9..d64512812 100644
--- a/engine/crates/xerj-api/src/router.rs
+++ b/engine/crates/xerj-api/src/router.rs
@@ -483,6 +483,7 @@ pub fn build_es_compat_router(state: AppState) -> Router {
.delete(es_compat::delete_ilm_policy),
)
.route("/:index/_ilm/explain", get(es_compat::ilm_explain))
+ .route("/:index/_ilm/remove", post(es_compat::ilm_remove))
// ── ISM (OpenSearch Index State Management) ─────────────────────────
// Same execution engine as ILM above — see `xerj_engine::lifecycle`.
.route(