Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): keep index deletion independent of stalled statistics

xerj-org81dfb19aae5b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6d30a2018..76f57a407 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Index deletion no longer waits for unrelated index statistics while
+ resolving its selectors. Index listings and health checks also release
+ their index-map locks before awaiting statistics, so a stalled schema
+ update does not hold those locks against DELETE (part of #2405; extracted
+ from #2489). Merge/flush cancellation and the process memory cap remain
+ separate follow-ups.
- `xerj autoindex` compresses per-file phase-B stages with zstd level 1 and
uses temporary files cleaned up by the operating system when the client
exits (#2493, part of #2427). It finishes buffered compression before
diff --git a/engine/crates/xerj-api/Cargo.toml b/engine/crates/xerj-api/Cargo.toml
index 3a6b1f2bf..6f8573f91 100644
--- a/engine/crates/xerj-api/Cargo.toml
+++ b/engine/crates/xerj-api/Cargo.toml
@@ -75,8 +75,8 @@ sha2 = "0.10"
[dev-dependencies]
tempfile = { workspace = true }
-# #2234: the HTTP refresh test holds one index inside `Index::refresh`.
-# The feature stays off the release graph (resolver v2, dev-dependency only).
-xerj-engine = { path = "../xerj-engine", features = ["refresh-test-hook"] }
+# HTTP refresh and deletion tests hold one index's refresh/schema lock.
+# These hooks stay off the release graph (resolver v2, dev-dependency only).
+xerj-engine = { path = "../xerj-engine", features = ["refresh-test-hook", "lock-test-hook"] }
tower = { workspace = true, features = ["util"] }
http-body-util = "0.1"
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 027cd99b6..86acb6958 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -2605,8 +2605,10 @@ pub async fn delete_index(
.map(|v| v == "true")
.unwrap_or(true);
- let all = state.engine.list_indices().await;
- let mut all_names: Vec<String> = all.iter().map(|i| i.name.clone()).collect();
+ // Selector resolution needs names, not statistics. Waiting for every
+ // sibling's schema/memtable locks can stall an otherwise independent
+ // DELETE before it even reaches the requested index (#2405).
+ let mut all_names = state.engine.index_name_list();
// A failed index is deletable. Its name has to take part in selector
// resolution or a literal `DELETE /broken` 404s and a wildcard silently
// skips it — which is exactly the stuck state issue #206 describes.
diff --git a/engine/crates/xerj-api/tests/delete_while_stats_stalled.rs b/engine/crates/xerj-api/tests/delete_while_stats_stalled.rs
new file mode 100644
index 000000000..c9769e387
--- /dev/null
+++ b/engine/crates/xerj-api/tests/delete_while_stats_stalled.rs
@@ -0,0 +1,148 @@
+//! #2405: stalled statistics must not hold index deletion behind a schema
+//! lock or a DashMap iterator. The schema hold models stalled work without
+//! depending on host RSS, an OOM, or scheduler sleeps.
+
+use std::future::{poll_fn, Future};
+use std::sync::Arc;
+use std::task::Poll;
+use std::time::Duration;
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+fn app(dir: &tempfile::TempDir) -> (axum::Router, Arc<xerj_engine::Engine>) {
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ config.storage.flush_idle_secs = 0;
+ config.storage.flush_interval_secs = 3600;
+ let metrics = xerj_common::metrics::Metrics::new().unwrap();
+ let engine = xerj_engine::Engine::new(config.clone()).unwrap();
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ let engine = Arc::clone(&state.engine);
+ (xerj_api::router::build_es_compat_router(state), engine)
+}
+
+async fn request(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let response = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method(method)
+ .uri(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (status, serde_json::from_slice(&bytes).unwrap())
+}
+
+async fn create(app: &axum::Router, name: &str) {
+ let (status, body) = request(app, "PUT", &format!("/{name}"), json!({})).await;
+ assert_eq!(status, StatusCode::OK, "create {name}: {body}");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn delete_selector_does_not_collect_a_stalled_siblings_stats() {
+ let dir = tempfile::tempdir().unwrap();
+ let (app, engine) = app(&dir);
+ create(&app, "victim-one").await;
+ create(&app, "victim-two").await;
+ create(&app, "busy").await;
+ let busy = engine.get_index("busy").unwrap();
+ busy.index_document(Some("kept".into()), json!({"value": "survives"}))
+ .await
+ .unwrap();
+ let hold = busy.test_hold_schema_write().await;
+
+ let deletion = tokio::time::timeout(
+ Duration::from_secs(5),
+ request(&app, "DELETE", "/victim-one,victim-two", json!({})),
+ )
+ .await;
+ drop(hold);
+ let (status, body) = deletion.expect("DELETE waited for an unrelated index's statistics");
+ assert_eq!(status, StatusCode::OK, "delete: {body}");
+ assert_eq!(body["acknowledged"], true);
+ for name in ["victim-one", "victim-two"] {
+ assert!(engine.get_index(name).is_err());
+ assert!(!dir.path().join(name).exists());
+ }
+ let (status, body) = request(&app, "GET", "/busy/_doc/kept", json!({})).await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(body["_source"]["value"], "survives");
+
+ // The name is reusable after deletion; no stale handle owns the new index.
+ create(&app, "victim-one").await;
+ let (status, _) = request(
+ &app,
+ "PUT",
+ "/victim-one/_doc/new",
+ json!({"value": "new generation"}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::CREATED);
+}
+
+async fn delete_while_stats_sweep_is_pending(health: bool) {
+ let dir = tempfile::tempdir().unwrap();
+ let (app, engine) = app(&dir);
+ create(&app, "busy").await;
+ let busy = engine.get_index("busy").unwrap();
+ let hold = busy.test_hold_schema_write().await;
+ let mut sweep = Box::pin(async {
+ if health {
+ let _ = engine.health().await;
+ } else {
+ let _ = engine.list_indices().await;
+ }
+ });
+ // Poll once ourselves: the only index is held, so this proves that the
+ // sweep reached stats(). No sleep guesses whether the task has started.
+ poll_fn(|cx| {
+ assert!(sweep.as_mut().poll(cx).is_pending());
+ Poll::Ready(())
+ })
+ .await;
+
+ let deleter = Arc::clone(&engine);
+ let mut task = tokio::spawn(async move { deleter.delete_index("busy").await });
+ let finished = tokio::time::timeout(Duration::from_secs(5), &mut task).await;
+ // Release both holds even on the negative control, where the delete
+ // worker is blocked synchronously on the map shard held by `sweep`.
+ drop(sweep);
+ drop(hold);
+ let completed_while_held = finished.is_ok();
+ let result = match finished {
+ Ok(result) => result,
+ Err(_) => tokio::time::timeout(Duration::from_secs(10), task)
+ .await
+ .expect("delete did not recover after releasing the sweep"),
+ };
+ result.expect("delete task").expect("delete index");
+ assert!(
+ completed_while_held,
+ "DELETE waited for a stalled {} sweep's map lock",
+ if health { "health" } else { "listing" }
+ );
+ assert!(!dir.path().join("busy").exists());
+ assert!(engine.get_index("busy").is_err());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn delete_does_not_wait_for_a_parked_listing() {
+ delete_while_stats_sweep_is_pending(false).await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn delete_does_not_wait_for_a_parked_health_check() {
+ delete_while_stats_sweep_is_pending(true).await;
+}
diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.toml
index 28e64dada..c95f54f21 100644
--- a/engine/crates/xerj-engine/Cargo.toml
+++ b/engine/crates/xerj-engine/Cargo.toml
@@ -85,6 +85,8 @@ stackprobe = []
# off. `xerj-api`'s dev-dependency enables it so an HTTP test can hold one
# index's refresh and prove a sibling index is not waiting behind it.
refresh-test-hook = []
+# Schema-lock hold for HTTP deletion/listing liveness tests. Dev-only.
+lock-test-hook = []
[dev-dependencies]
xerj-storage = { path = "../xerj-storage", features = ["test-hooks"] }
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 43b006afa..12a7002f9 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -4016,11 +4016,11 @@ impl Engine {
/// blanket refusal.
pub async fn list_indices(&self) -> Vec<IndexInfo> {
let mut list = Vec::new();
- for entry in self.indices.iter() {
- if !crate::index_guard::visible(entry.key()) {
+ for (name, idx) in self.open_index_handles() {
+ if !crate::index_guard::visible(&name) {
continue;
}
- let stats = entry.value().stats().await;
+ let stats = idx.stats().await;
list.push(IndexInfo {
name: stats.name,
doc_count: stats.doc_count,
@@ -4031,6 +4031,17 @@ impl Engine {
list
}
+ /// Snapshot handles for read-only statistics, releasing every DashMap
+ /// iterator guard before awaiting an index. A stalled schema lock must
+ /// not hold the map shard against an index deletion or insertion (#2405).
+ /// This preserves the existing observational (non-atomic) stats contract.
+ fn open_index_handles(&self) -> Vec<(String, Arc<Index>)> {
+ self.indices
+ .iter()
+ .map(|entry| (entry.key().clone(), Arc::clone(entry.value())))
+ .collect()
+ }
+
/// Get the stats for a single index.
pub async fn index_stats(&self, name: &str) -> Result<IndexStats> {
let idx = self.get_index(name)?;
@@ -4117,8 +4128,8 @@ impl Engine {
let mut total_docs = 0u64;
let mut has_memtable_only = false;
- for entry in self.indices.iter() {
- let stats = entry.value().stats().await;
+ for (_, idx) in self.open_index_handles() {
+ let stats = idx.stats().await;
total_docs += stats.doc_count;
// Yellow condition: any index has in-memory data that hasn't been
// flushed to a segment yet.
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index e07c1149e..ad9d604f3 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -10712,6 +10712,13 @@ pub struct Index {
merge_batches_deferred: std::sync::atomic::AtomicBool,
}
+/// Keeps one index's schema unavailable to statistics until dropped.
+#[cfg(any(test, feature = "lock-test-hook"))]
+#[doc(hidden)]
+pub struct TestSchemaWriteHold(
+ #[allow(dead_code)] tokio::sync::OwnedRwLockWriteGuard<ManagedSchema>,
+);
+
/// Counts one `Index::refresh` that got past the test hold, on every exit.
#[cfg(any(test, feature = "refresh-test-hook"))]
struct TestRefreshFinished<'a>(&'a Index);
@@ -28593,6 +28600,13 @@ impl Index {
out
}
+ /// Deterministic stand-in for a stalled schema update (#2405).
+ #[cfg(any(test, feature = "lock-test-hook"))]
+ #[doc(hidden)]
+ pub async fn test_hold_schema_write(&self) -> TestSchemaWriteHold {
+ TestSchemaWriteHold(Arc::clone(&self.schema).write_owned().await)
+ }
+
pub async fn stats(&self) -> IndexStats {
let snap = self.store.snapshot();
let segment_count = snap.segments.len();