AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(engine): close_index/reopen_index that actually release+reconstruct RAM (#463, WIP engine core)
xerj-orgbcca8bf4c449
diff --git a/engine/crates/xerj-api/tests/close_frees_memory.rs b/engine/crates/xerj-api/tests/close_frees_memory.rsnew file mode 100644index 000000000..1a6c092f5--- /dev/null+++ b/engine/crates/xerj-api/tests/close_frees_memory.rs@@ -0,0 +1,133 @@+//! Issue #463: `POST /{index}/_close` marks an index unqueryable but frees ~no+//! memory — `close_index` only sets the `closed_indices` flag; the `Arc<Index>`+//! (memtable + per-segment caches + hydration budget) stays in `engine.indices`.+//!+//! This probe establishes the fail-before observable: after `_close`, the global+//! `segment_hydration.current_in_bytes` gauge does NOT drop (memory retained),+//! and a subsequent `_open` + query must still return every doc (the fix must+//! flush before releasing, so no data is lost).++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;+use xerj_common::types::{FieldConfig, FieldType, Schema};++async fn app() -> (axum::Router, tempfile::TempDir) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);++ let mut schema = Schema::empty();+ schema+ .add_field(FieldConfig::new("body", FieldType::Text))+ .expect("body");+ schema+ .add_field(FieldConfig::new("n", FieldType::Long))+ .expect("n");+ state.engine.create_index("docs", schema).expect("create");+ let idx = state.engine.get_index("docs").expect("get");+ for i in 0..200 {+ idx.index_document(+ Some(format!("d{i}")),+ json!({"body": format!("document number {i} with some searchable prose content to hydrate"), "n": i}),+ )+ .await+ .expect("index");+ }+ idx.refresh().await.expect("refresh");+ (xerj_api::router::build_es_compat_router(state), dir)+}++async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+}++/// Recursively find the first `segment_hydration.current_in_bytes` in the stats.+fn hydration_current(stats: &Value) -> Option<u64> {+ fn walk(v: &Value) -> Option<u64> {+ if let Some(obj) = v.as_object() {+ if let Some(h) = obj.get("segment_hydration") {+ if let Some(c) = h.get("current_in_bytes").and_then(Value::as_u64) {+ return Some(c);+ }+ }+ for (_, child) in obj {+ if let Some(c) = walk(child) {+ return Some(c);+ }+ }+ } else if let Some(arr) = v.as_array() {+ for child in arr {+ if let Some(c) = walk(child) {+ return Some(c);+ }+ }+ }+ None+ }+ walk(stats)+}++#[tokio::test]+async fn probe_close_hydration_and_reopen_roundtrip() {+ let (app, _dir) = app().await;++ // Hydrate: run a query that reads segments.+ let (st, _) = call(+ &app,+ "POST",+ "/docs/_search",+ json!({"query": {"match": {"body": "document"}}, "size": 50}),+ )+ .await;+ assert_eq!(st, StatusCode::OK);++ let (_, before) = call(&app, "GET", "/_nodes/stats", Value::Null).await;+ let hyd_before = hydration_current(&before);+ eprintln!("PROBE hydration current_in_bytes BEFORE close = {hyd_before:?}");++ // Close.+ let (st, cbody) = call(&app, "POST", "/docs/_close", Value::Null).await;+ eprintln!("PROBE close status={st} body={cbody}");+ assert_eq!(st, StatusCode::OK, "close: {cbody}");++ let (_, after) = call(&app, "GET", "/_nodes/stats", Value::Null).await;+ let hyd_after = hydration_current(&after);+ eprintln!("PROBE hydration current_in_bytes AFTER close = {hyd_after:?}");++ // Reopen + query must return docs (lossless) — this must hold before AND+ // after the fix (the fix must flush before releasing).+ let (st, _open) = call(&app, "POST", "/docs/_open", Value::Null).await;+ eprintln!("PROBE open status={st}");+ let (st, q) = call(+ &app,+ "POST",+ "/docs/_search",+ json!({"query": {"match_all": {}}, "size": 0}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "post-open query: {q}");+ let total = q.pointer("/hits/total/value").and_then(Value::as_u64);+ eprintln!("PROBE post-open total hits = {total:?}");+ assert_eq!(total, Some(200), "reopen must be lossless: {q}");+}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 8814408bc..bc7d5272c 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -3187,6 +3187,58 @@ impl Engine {}}+ /// Whether the concrete index `name` is currently held in memory (its+ /// `Arc<Index>` — memtable, per-segment caches, hydration budget — is live+ /// in `self.indices`). A `_close` that actually reclaims RAM drops the handle+ /// here (#463); a still-loaded name after close means the close was a no-op.+ /// Not alias-resolving — callers pass a concrete index name.+ pub fn is_index_loaded(&self, name: &str) -> bool {+ self.indices.contains_key(name)+ }++ /// `_close` that actually reclaims RAM (#463). Marks the index unqueryable+ /// AND releases its in-memory handle — the memtable, the per-segment caches,+ /// and the hydration budget deallocate once no in-flight request holds a+ /// clone of the `Arc<Index>`. The on-disk bytes remain untouched, so+ /// [`reopen_index`](Self::reopen_index) reconstructs the index losslessly.+ ///+ /// The memtable is flushed to disk FIRST: dropping the handle without+ /// flushing would discard every not-yet-published document — turning a memory+ /// win into silent data loss. A flush failure aborts the close and leaves the+ /// index fully in service.+ pub async fn close_index(&self, name: &str) -> Result<()> {+ self.ensure_cluster_state_writable()?;+ if self.indices.contains_key(name) {+ // Durability barrier before the handle is dropped.+ self.flush_index(name).await?;+ // Drop the strong ref held by the engine. Any in-flight request that+ // cloned the `Arc` keeps its own copy alive until it completes, then+ // the index (and its governor-registered segment caches) frees.+ self.indices.remove(name);+ }+ self.closed_indices.insert(name.to_string(), true);+ info!(name, "index closed; in-memory handle released");+ Ok(())+ }++ /// `_open` counterpart to [`close_index`](Self::close_index): clears the+ /// closed flag and, if the index was memory-released, reconstructs its+ /// `Arc<Index>` from disk via [`Index::open`] (which replays the WAL and+ /// republishes segments). A no-op for an index that is already loaded.+ pub fn reopen_index(&self, name: &str) -> Result<()> {+ self.ensure_cluster_state_writable()?;+ if !self.indices.contains_key(name) {+ let index_name = IndexName::new(name).map_err(EngineError::Common)?;+ // `Index::open` returns an `Arc<Index>` (replays the WAL and+ // republishes segments from disk) — insert it directly.+ let idx = Index::open(index_name, &self.config, &self.data_dir)?;+ self.indices.insert(name.to_string(), idx);+ }+ self.closed_indices.remove(name);+ info!(name, "index reopened");+ Ok(())+ }+/// Get a reference to an index by name, resolving aliases first./// If the name is an alias pointing to multiple indices, returns the first one.///diff --git a/engine/crates/xerj-engine/tests/close_releases_index.rs b/engine/crates/xerj-engine/tests/close_releases_index.rsnew file mode 100644index 000000000..c5775744d--- /dev/null+++ b/engine/crates/xerj-engine/tests/close_releases_index.rs@@ -0,0 +1,100 @@+//! Issue #463: `_close` must actually reclaim RAM, not just flip a flag.+//!+//! Before the fix, `close_index` only set the `closed_indices` gate; the+//! `Arc<Index>` (memtable, per-segment caches, hydration budget) stayed in+//! `Engine::indices`, so `_close` freed ~0.03% of an index's memory. The fix+//! releases the in-memory handle on close and reconstructs it from disk on+//! reopen — and must flush first so no not-yet-published document is lost.++use serde_json::json;+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::Engine;+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++async fn count_all(engine: &Engine, name: &str) -> u64 {+ let idx = engine.get_index(name).expect("get index");+ let req = parse_request(&json!({ "query": { "match_all": {} }, "size": 0, "from": 0 }))+ .expect("parse_request");+ idx.search(&req).await.expect("search").total.value+}++async fn seed(engine: &Engine, name: &str, n: usize) {+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("body", FieldType::Text));+ engine.create_index(name, schema).expect("create");+ let idx = engine.get_index(name).expect("get");+ for i in 0..n {+ idx.index_document(+ Some(format!("d{i}")),+ json!({ "body": format!("document number {i}") }),+ )+ .await+ .expect("index");+ }+}++/// `close_index` releases the in-memory handle; `reopen_index` reconstructs it+/// from disk with no data loss. FAIL-BEFORE: with the release hunk reverted the+/// handle stays loaded after close, so the `!is_index_loaded` assertion fails.+#[tokio::test]+async fn close_releases_handle_and_reopen_is_lossless() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine, "docs", 50).await;+ engine.get_index("docs").unwrap().refresh().await.unwrap();++ assert!(engine.is_index_loaded("docs"), "loaded before close");+ assert_eq!(count_all(&engine, "docs").await, 50);++ engine.close_index("docs").await.expect("close");+ assert!(+ !engine.is_index_loaded("docs"),+ "#463: _close must release the in-memory Arc<Index>, not just flip a flag"+ );+ assert!(+ engine.closed_indices.contains_key("docs"),+ "close still marks the index closed"+ );++ engine.reopen_index("docs").expect("reopen");+ assert!(engine.is_index_loaded("docs"), "reopen reloads the handle");+ assert!(+ !engine.closed_indices.contains_key("docs"),+ "reopen clears the closed flag"+ );+ assert_eq!(+ count_all(&engine, "docs").await,+ 50,+ "#463: reopen must be lossless"+ );+}++/// Data-safety: documents indexed but NOT refreshed/flushed before close must+/// survive close→open (close flushes; reopen replays). Guards against a memory+/// win that silently drops the memtable.+#[tokio::test]+async fn close_flushes_so_unrefreshed_docs_survive_reopen() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine, "docs", 30).await; // NO refresh — docs sit in the memtable/WAL++ engine.close_index("docs").await.expect("close");+ assert!(!engine.is_index_loaded("docs"), "released on close");++ engine.reopen_index("docs").expect("reopen");+ assert_eq!(+ count_all(&engine, "docs").await,+ 30,+ "#463: close must flush before releasing — no doc may be lost"+ );+}