AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): keep index deletion independent of stalled statistics
xerj-org81dfb19aae5b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6d30a2018..76f57a407 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Index deletion no longer waits for unrelated index statistics while+ resolving its selectors. Index listings and health checks also release+ their index-map locks before awaiting statistics, so a stalled schema+ update does not hold those locks against DELETE (part of #2405; extracted+ from #2489). Merge/flush cancellation and the process memory cap remain+ separate follow-ups.- `xerj autoindex` compresses per-file phase-B stages with zstd level 1 anduses temporary files cleaned up by the operating system when the clientexits (#2493, part of #2427). It finishes buffered compression beforediff --git a/engine/crates/xerj-api/Cargo.toml b/engine/crates/xerj-api/Cargo.tomlindex 3a6b1f2bf..6f8573f91 100644--- a/engine/crates/xerj-api/Cargo.toml+++ b/engine/crates/xerj-api/Cargo.toml@@ -75,8 +75,8 @@ sha2 = "0.10"[dev-dependencies]tempfile = { workspace = true }-# #2234: the HTTP refresh test holds one index inside `Index::refresh`.-# The feature stays off the release graph (resolver v2, dev-dependency only).-xerj-engine = { path = "../xerj-engine", features = ["refresh-test-hook"] }+# HTTP refresh and deletion tests hold one index's refresh/schema lock.+# These hooks stay off the release graph (resolver v2, dev-dependency only).+xerj-engine = { path = "../xerj-engine", features = ["refresh-test-hook", "lock-test-hook"] }tower = { workspace = true, features = ["util"] }http-body-util = "0.1"diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 027cd99b6..86acb6958 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -2605,8 +2605,10 @@ pub async fn delete_index(.map(|v| v == "true").unwrap_or(true);- let all = state.engine.list_indices().await;- let mut all_names: Vec<String> = all.iter().map(|i| i.name.clone()).collect();+ // Selector resolution needs names, not statistics. Waiting for every+ // sibling's schema/memtable locks can stall an otherwise independent+ // DELETE before it even reaches the requested index (#2405).+ let mut all_names = state.engine.index_name_list();// A failed index is deletable. Its name has to take part in selector// resolution or a literal `DELETE /broken` 404s and a wildcard silently// skips it — which is exactly the stuck state issue #206 describes.diff --git a/engine/crates/xerj-api/tests/delete_while_stats_stalled.rs b/engine/crates/xerj-api/tests/delete_while_stats_stalled.rsnew file mode 100644index 000000000..c9769e387--- /dev/null+++ b/engine/crates/xerj-api/tests/delete_while_stats_stalled.rs@@ -0,0 +1,148 @@+//! #2405: stalled statistics must not hold index deletion behind a schema+//! lock or a DashMap iterator. The schema hold models stalled work without+//! depending on host RSS, an OOM, or scheduler sleeps.++use std::future::{poll_fn, Future};+use std::sync::Arc;+use std::task::Poll;+use std::time::Duration;++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++fn app(dir: &tempfile::TempDir) -> (axum::Router, Arc<xerj_engine::Engine>) {+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ config.storage.flush_idle_secs = 0;+ config.storage.flush_interval_secs = 3600;+ let metrics = xerj_common::metrics::Metrics::new().unwrap();+ let engine = xerj_engine::Engine::new(config.clone()).unwrap();+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ let engine = Arc::clone(&state.engine);+ (xerj_api::router::build_es_compat_router(state), engine)+}++async fn request(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let response = app+ .clone()+ .oneshot(+ Request::builder()+ .method(method)+ .uri(path)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .unwrap(),+ )+ .await+ .unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (status, serde_json::from_slice(&bytes).unwrap())+}++async fn create(app: &axum::Router, name: &str) {+ let (status, body) = request(app, "PUT", &format!("/{name}"), json!({})).await;+ assert_eq!(status, StatusCode::OK, "create {name}: {body}");+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn delete_selector_does_not_collect_a_stalled_siblings_stats() {+ let dir = tempfile::tempdir().unwrap();+ let (app, engine) = app(&dir);+ create(&app, "victim-one").await;+ create(&app, "victim-two").await;+ create(&app, "busy").await;+ let busy = engine.get_index("busy").unwrap();+ busy.index_document(Some("kept".into()), json!({"value": "survives"}))+ .await+ .unwrap();+ let hold = busy.test_hold_schema_write().await;++ let deletion = tokio::time::timeout(+ Duration::from_secs(5),+ request(&app, "DELETE", "/victim-one,victim-two", json!({})),+ )+ .await;+ drop(hold);+ let (status, body) = deletion.expect("DELETE waited for an unrelated index's statistics");+ assert_eq!(status, StatusCode::OK, "delete: {body}");+ assert_eq!(body["acknowledged"], true);+ for name in ["victim-one", "victim-two"] {+ assert!(engine.get_index(name).is_err());+ assert!(!dir.path().join(name).exists());+ }+ let (status, body) = request(&app, "GET", "/busy/_doc/kept", json!({})).await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(body["_source"]["value"], "survives");++ // The name is reusable after deletion; no stale handle owns the new index.+ create(&app, "victim-one").await;+ let (status, _) = request(+ &app,+ "PUT",+ "/victim-one/_doc/new",+ json!({"value": "new generation"}),+ )+ .await;+ assert_eq!(status, StatusCode::CREATED);+}++async fn delete_while_stats_sweep_is_pending(health: bool) {+ let dir = tempfile::tempdir().unwrap();+ let (app, engine) = app(&dir);+ create(&app, "busy").await;+ let busy = engine.get_index("busy").unwrap();+ let hold = busy.test_hold_schema_write().await;+ let mut sweep = Box::pin(async {+ if health {+ let _ = engine.health().await;+ } else {+ let _ = engine.list_indices().await;+ }+ });+ // Poll once ourselves: the only index is held, so this proves that the+ // sweep reached stats(). No sleep guesses whether the task has started.+ poll_fn(|cx| {+ assert!(sweep.as_mut().poll(cx).is_pending());+ Poll::Ready(())+ })+ .await;++ let deleter = Arc::clone(&engine);+ let mut task = tokio::spawn(async move { deleter.delete_index("busy").await });+ let finished = tokio::time::timeout(Duration::from_secs(5), &mut task).await;+ // Release both holds even on the negative control, where the delete+ // worker is blocked synchronously on the map shard held by `sweep`.+ drop(sweep);+ drop(hold);+ let completed_while_held = finished.is_ok();+ let result = match finished {+ Ok(result) => result,+ Err(_) => tokio::time::timeout(Duration::from_secs(10), task)+ .await+ .expect("delete did not recover after releasing the sweep"),+ };+ result.expect("delete task").expect("delete index");+ assert!(+ completed_while_held,+ "DELETE waited for a stalled {} sweep's map lock",+ if health { "health" } else { "listing" }+ );+ assert!(!dir.path().join("busy").exists());+ assert!(engine.get_index("busy").is_err());+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn delete_does_not_wait_for_a_parked_listing() {+ delete_while_stats_sweep_is_pending(false).await;+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn delete_does_not_wait_for_a_parked_health_check() {+ delete_while_stats_sweep_is_pending(true).await;+}diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.tomlindex 28e64dada..c95f54f21 100644--- a/engine/crates/xerj-engine/Cargo.toml+++ b/engine/crates/xerj-engine/Cargo.toml@@ -85,6 +85,8 @@ stackprobe = []# off. `xerj-api`'s dev-dependency enables it so an HTTP test can hold one# index's refresh and prove a sibling index is not waiting behind it.refresh-test-hook = []+# Schema-lock hold for HTTP deletion/listing liveness tests. Dev-only.+lock-test-hook = [][dev-dependencies]xerj-storage = { path = "../xerj-storage", features = ["test-hooks"] }diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 43b006afa..12a7002f9 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -4016,11 +4016,11 @@ impl Engine {/// blanket refusal.pub async fn list_indices(&self) -> Vec<IndexInfo> {let mut list = Vec::new();- for entry in self.indices.iter() {- if !crate::index_guard::visible(entry.key()) {+ for (name, idx) in self.open_index_handles() {+ if !crate::index_guard::visible(&name) {continue;}- let stats = entry.value().stats().await;+ let stats = idx.stats().await;list.push(IndexInfo {name: stats.name,doc_count: stats.doc_count,@@ -4031,6 +4031,17 @@ impl Engine {list}+ /// Snapshot handles for read-only statistics, releasing every DashMap+ /// iterator guard before awaiting an index. A stalled schema lock must+ /// not hold the map shard against an index deletion or insertion (#2405).+ /// This preserves the existing observational (non-atomic) stats contract.+ fn open_index_handles(&self) -> Vec<(String, Arc<Index>)> {+ self.indices+ .iter()+ .map(|entry| (entry.key().clone(), Arc::clone(entry.value())))+ .collect()+ }+/// Get the stats for a single index.pub async fn index_stats(&self, name: &str) -> Result<IndexStats> {let idx = self.get_index(name)?;@@ -4117,8 +4128,8 @@ impl Engine {let mut total_docs = 0u64;let mut has_memtable_only = false;- for entry in self.indices.iter() {- let stats = entry.value().stats().await;+ for (_, idx) in self.open_index_handles() {+ let stats = idx.stats().await;total_docs += stats.doc_count;// Yellow condition: any index has in-memory data that hasn't been// flushed to a segment yet.diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex e07c1149e..ad9d604f3 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -10712,6 +10712,13 @@ pub struct Index {merge_batches_deferred: std::sync::atomic::AtomicBool,}+/// Keeps one index's schema unavailable to statistics until dropped.+#[cfg(any(test, feature = "lock-test-hook"))]+#[doc(hidden)]+pub struct TestSchemaWriteHold(+ #[allow(dead_code)] tokio::sync::OwnedRwLockWriteGuard<ManagedSchema>,+);+/// Counts one `Index::refresh` that got past the test hold, on every exit.#[cfg(any(test, feature = "refresh-test-hook"))]struct TestRefreshFinished<'a>(&'a Index);@@ -28593,6 +28600,13 @@ impl Index {out}+ /// Deterministic stand-in for a stalled schema update (#2405).+ #[cfg(any(test, feature = "lock-test-hook"))]+ #[doc(hidden)]+ pub async fn test_hold_schema_write(&self) -> TestSchemaWriteHold {+ TestSchemaWriteHold(Arc::clone(&self.schema).write_owned().await)+ }+pub async fn stats(&self) -> IndexStats {let snap = self.store.snapshot();let segment_count = snap.segments.len();