Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(engine): close_index/reopen_index that actually release+reconstruct RAM (#463, WIP engine core)

xerj-orgbcca8bf4c449
diff --git a/engine/crates/xerj-api/tests/close_frees_memory.rs b/engine/crates/xerj-api/tests/close_frees_memory.rs
new file mode 100644
index 000000000..1a6c092f5
--- /dev/null
+++ b/engine/crates/xerj-api/tests/close_frees_memory.rs
@@ -0,0 +1,133 @@
+//! Issue #463: `POST /{index}/_close` marks an index unqueryable but frees ~no
+//! memory — `close_index` only sets the `closed_indices` flag; the `Arc<Index>`
+//! (memtable + per-segment caches + hydration budget) stays in `engine.indices`.
+//!
+//! This probe establishes the fail-before observable: after `_close`, the global
+//! `segment_hydration.current_in_bytes` gauge does NOT drop (memory retained),
+//! and a subsequent `_open` + query must still return every doc (the fix must
+//! flush before releasing, so no data is lost).
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+
+ let mut schema = Schema::empty();
+ schema
+ .add_field(FieldConfig::new("body", FieldType::Text))
+ .expect("body");
+ schema
+ .add_field(FieldConfig::new("n", FieldType::Long))
+ .expect("n");
+ state.engine.create_index("docs", schema).expect("create");
+ let idx = state.engine.get_index("docs").expect("get");
+ for i in 0..200 {
+ idx.index_document(
+ Some(format!("d{i}")),
+ json!({"body": format!("document number {i} with some searchable prose content to hydrate"), "n": i}),
+ )
+ .await
+ .expect("index");
+ }
+ idx.refresh().await.expect("refresh");
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+/// Recursively find the first `segment_hydration.current_in_bytes` in the stats.
+fn hydration_current(stats: &Value) -> Option<u64> {
+ fn walk(v: &Value) -> Option<u64> {
+ if let Some(obj) = v.as_object() {
+ if let Some(h) = obj.get("segment_hydration") {
+ if let Some(c) = h.get("current_in_bytes").and_then(Value::as_u64) {
+ return Some(c);
+ }
+ }
+ for (_, child) in obj {
+ if let Some(c) = walk(child) {
+ return Some(c);
+ }
+ }
+ } else if let Some(arr) = v.as_array() {
+ for child in arr {
+ if let Some(c) = walk(child) {
+ return Some(c);
+ }
+ }
+ }
+ None
+ }
+ walk(stats)
+}
+
+#[tokio::test]
+async fn probe_close_hydration_and_reopen_roundtrip() {
+ let (app, _dir) = app().await;
+
+ // Hydrate: run a query that reads segments.
+ let (st, _) = call(
+ &app,
+ "POST",
+ "/docs/_search",
+ json!({"query": {"match": {"body": "document"}}, "size": 50}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK);
+
+ let (_, before) = call(&app, "GET", "/_nodes/stats", Value::Null).await;
+ let hyd_before = hydration_current(&before);
+ eprintln!("PROBE hydration current_in_bytes BEFORE close = {hyd_before:?}");
+
+ // Close.
+ let (st, cbody) = call(&app, "POST", "/docs/_close", Value::Null).await;
+ eprintln!("PROBE close status={st} body={cbody}");
+ assert_eq!(st, StatusCode::OK, "close: {cbody}");
+
+ let (_, after) = call(&app, "GET", "/_nodes/stats", Value::Null).await;
+ let hyd_after = hydration_current(&after);
+ eprintln!("PROBE hydration current_in_bytes AFTER close = {hyd_after:?}");
+
+ // Reopen + query must return docs (lossless) — this must hold before AND
+ // after the fix (the fix must flush before releasing).
+ let (st, _open) = call(&app, "POST", "/docs/_open", Value::Null).await;
+ eprintln!("PROBE open status={st}");
+ let (st, q) = call(
+ &app,
+ "POST",
+ "/docs/_search",
+ json!({"query": {"match_all": {}}, "size": 0}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "post-open query: {q}");
+ let total = q.pointer("/hits/total/value").and_then(Value::as_u64);
+ eprintln!("PROBE post-open total hits = {total:?}");
+ assert_eq!(total, Some(200), "reopen must be lossless: {q}");
+}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 8814408bc..bc7d5272c 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -3187,6 +3187,58 @@ impl Engine {
}
}
+ /// Whether the concrete index `name` is currently held in memory (its
+ /// `Arc<Index>` — memtable, per-segment caches, hydration budget — is live
+ /// in `self.indices`). A `_close` that actually reclaims RAM drops the handle
+ /// here (#463); a still-loaded name after close means the close was a no-op.
+ /// Not alias-resolving — callers pass a concrete index name.
+ pub fn is_index_loaded(&self, name: &str) -> bool {
+ self.indices.contains_key(name)
+ }
+
+ /// `_close` that actually reclaims RAM (#463). Marks the index unqueryable
+ /// AND releases its in-memory handle — the memtable, the per-segment caches,
+ /// and the hydration budget deallocate once no in-flight request holds a
+ /// clone of the `Arc<Index>`. The on-disk bytes remain untouched, so
+ /// [`reopen_index`](Self::reopen_index) reconstructs the index losslessly.
+ ///
+ /// The memtable is flushed to disk FIRST: dropping the handle without
+ /// flushing would discard every not-yet-published document — turning a memory
+ /// win into silent data loss. A flush failure aborts the close and leaves the
+ /// index fully in service.
+ pub async fn close_index(&self, name: &str) -> Result<()> {
+ self.ensure_cluster_state_writable()?;
+ if self.indices.contains_key(name) {
+ // Durability barrier before the handle is dropped.
+ self.flush_index(name).await?;
+ // Drop the strong ref held by the engine. Any in-flight request that
+ // cloned the `Arc` keeps its own copy alive until it completes, then
+ // the index (and its governor-registered segment caches) frees.
+ self.indices.remove(name);
+ }
+ self.closed_indices.insert(name.to_string(), true);
+ info!(name, "index closed; in-memory handle released");
+ Ok(())
+ }
+
+ /// `_open` counterpart to [`close_index`](Self::close_index): clears the
+ /// closed flag and, if the index was memory-released, reconstructs its
+ /// `Arc<Index>` from disk via [`Index::open`] (which replays the WAL and
+ /// republishes segments). A no-op for an index that is already loaded.
+ pub fn reopen_index(&self, name: &str) -> Result<()> {
+ self.ensure_cluster_state_writable()?;
+ if !self.indices.contains_key(name) {
+ let index_name = IndexName::new(name).map_err(EngineError::Common)?;
+ // `Index::open` returns an `Arc<Index>` (replays the WAL and
+ // republishes segments from disk) — insert it directly.
+ let idx = Index::open(index_name, &self.config, &self.data_dir)?;
+ self.indices.insert(name.to_string(), idx);
+ }
+ self.closed_indices.remove(name);
+ info!(name, "index reopened");
+ Ok(())
+ }
+
/// Get a reference to an index by name, resolving aliases first.
/// If the name is an alias pointing to multiple indices, returns the first one.
///
diff --git a/engine/crates/xerj-engine/tests/close_releases_index.rs b/engine/crates/xerj-engine/tests/close_releases_index.rs
new file mode 100644
index 000000000..c5775744d
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/close_releases_index.rs
@@ -0,0 +1,100 @@
+//! Issue #463: `_close` must actually reclaim RAM, not just flip a flag.
+//!
+//! Before the fix, `close_index` only set the `closed_indices` gate; the
+//! `Arc<Index>` (memtable, per-segment caches, hydration budget) stayed in
+//! `Engine::indices`, so `_close` freed ~0.03% of an index's memory. The fix
+//! releases the in-memory handle on close and reconstructs it from disk on
+//! reopen — and must flush first so no not-yet-published document is lost.
+
+use serde_json::json;
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::Engine;
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+async fn count_all(engine: &Engine, name: &str) -> u64 {
+ let idx = engine.get_index(name).expect("get index");
+ let req = parse_request(&json!({ "query": { "match_all": {} }, "size": 0, "from": 0 }))
+ .expect("parse_request");
+ idx.search(&req).await.expect("search").total.value
+}
+
+async fn seed(engine: &Engine, name: &str, n: usize) {
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("body", FieldType::Text));
+ engine.create_index(name, schema).expect("create");
+ let idx = engine.get_index(name).expect("get");
+ for i in 0..n {
+ idx.index_document(
+ Some(format!("d{i}")),
+ json!({ "body": format!("document number {i}") }),
+ )
+ .await
+ .expect("index");
+ }
+}
+
+/// `close_index` releases the in-memory handle; `reopen_index` reconstructs it
+/// from disk with no data loss. FAIL-BEFORE: with the release hunk reverted the
+/// handle stays loaded after close, so the `!is_index_loaded` assertion fails.
+#[tokio::test]
+async fn close_releases_handle_and_reopen_is_lossless() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ seed(&engine, "docs", 50).await;
+ engine.get_index("docs").unwrap().refresh().await.unwrap();
+
+ assert!(engine.is_index_loaded("docs"), "loaded before close");
+ assert_eq!(count_all(&engine, "docs").await, 50);
+
+ engine.close_index("docs").await.expect("close");
+ assert!(
+ !engine.is_index_loaded("docs"),
+ "#463: _close must release the in-memory Arc<Index>, not just flip a flag"
+ );
+ assert!(
+ engine.closed_indices.contains_key("docs"),
+ "close still marks the index closed"
+ );
+
+ engine.reopen_index("docs").expect("reopen");
+ assert!(engine.is_index_loaded("docs"), "reopen reloads the handle");
+ assert!(
+ !engine.closed_indices.contains_key("docs"),
+ "reopen clears the closed flag"
+ );
+ assert_eq!(
+ count_all(&engine, "docs").await,
+ 50,
+ "#463: reopen must be lossless"
+ );
+}
+
+/// Data-safety: documents indexed but NOT refreshed/flushed before close must
+/// survive close→open (close flushes; reopen replays). Guards against a memory
+/// win that silently drops the memtable.
+#[tokio::test]
+async fn close_flushes_so_unrefreshed_docs_survive_reopen() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ seed(&engine, "docs", 30).await; // NO refresh — docs sit in the memtable/WAL
+
+ engine.close_index("docs").await.expect("close");
+ assert!(!engine.is_index_loaded("docs"), "released on close");
+
+ engine.reopen_index("docs").expect("reopen");
+ assert_eq!(
+ count_all(&engine, "docs").await,
+ 30,
+ "#463: close must flush before releasing — no doc may be lost"
+ );
+}