Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

refactor(#463): reclaim RAM by releasing caches, keep the index loaded (B)

xerj-orgab185cd33f10
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 4c365067c..a1647bd5c 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -668,18 +668,6 @@ async fn cat_indices_inner(
// operator can delete or retry, and omitting them is what made a failed
// index invisible to every dashboard (issue #206).
let failed = state.engine.list_failed_indices();
- // #463: a `_close` that reclaimed RAM released the index's in-memory handle,
- // so it is absent from `list_indices` (loaded-only). List those indices with
- // status `close` rather than omitting them — an operator who closed an index
- // must still see it (the same visibility guarantee #206 restored for failed
- // indices). No doc count is shown: we do not load the index just to count.
- let closed_all: Vec<String> = state
- .engine
- .closed_indices
- .iter()
- .map(|e| e.key().clone())
- .filter(|n| !indices.iter().any(|i| &i.name == n) && !failed.iter().any(|f| &f.name == n))
- .collect();
// Narrow to the path pattern when present (`/_cat/indices/{pattern}`):
// a comma-separated list of concrete names and/or `*` globs. ES rules:
@@ -699,7 +687,6 @@ async fn cat_indices_inner(
if !is_wildcard
&& !indices.iter().any(|i| i.name == p)
&& !failed.iter().any(|f| f.name == p)
- && !closed_all.iter().any(|n| n == p)
{
return ApiError::new(xerj_common::XerjError::index_not_found(p))
.into_response();
@@ -765,21 +752,6 @@ async fn cat_indices_inner(
.map(|f| (f.name.clone(), stable_index_uuid(&f.name)))
.collect();
- // #463: closed indices selected by the same pattern rules as above.
- let selected_closed: Vec<String> = match pattern.as_deref() {
- None => closed_all.clone(),
- Some(pat) => closed_all
- .iter()
- .filter(|n| {
- pat.split(',')
- .map(str::trim)
- .filter(|s| !s.is_empty())
- .any(|p| p == "_all" || p == "*" || n.as_str() == p || glob_match_simple(p, n))
- })
- .cloned()
- .collect(),
- };
-
if params.format.as_deref() == Some("json") {
let mut arr: Vec<Value> = rows
.iter()
@@ -812,20 +784,6 @@ async fn cat_indices_inner(
"pri.store.size": "0b",
})
}));
- arr.extend(selected_closed.iter().map(|name| {
- json!({
- "health": "",
- "status": "close",
- "index": name,
- "uuid": stable_index_uuid(name),
- "pri": "1",
- "rep": "0",
- "docs.count": "0",
- "docs.deleted": "0",
- "store.size": "0b",
- "pri.store.size": "0b",
- })
- }));
return Json(arr).into_response();
}
@@ -838,13 +796,6 @@ async fn cat_indices_inner(
for (name, uuid) in &failed_rows {
lines.push(format!("red open {name} {uuid} 1 0 0 0 0b 0b 0b"));
}
- for name in &selected_closed {
- // Empty health column (rendered `-`), status `close` (#463).
- lines.push(format!(
- "- close {name} {} 1 0 0 0 0b 0b 0b",
- stable_index_uuid(name)
- ));
- }
// ES returns an empty body (not a bare newline) when nothing matches.
let body = if lines.is_empty() {
String::new()
@@ -27875,28 +27826,6 @@ pub async fn open_index(
State(state): State<AppState>,
Path(index): Path<String>,
) -> impl IntoResponse {
- // #463: a `_close` that reclaimed RAM dropped the index's in-memory handle,
- // so it is not in the loaded set that `resolve_indices_for_op` sees. Any
- // closed index the spec names is reconstructed from disk FIRST, so the
- // resolve + reopen loop below finds it. Concrete names match directly;
- // wildcards/_all match against the closed set.
- let closed_names: Vec<String> = state
- .engine
- .closed_indices
- .iter()
- .map(|e| e.key().clone())
- .collect();
- for name in &closed_names {
- let named = index
- .split(',')
- .map(str::trim)
- .any(|p| p == "_all" || p == "*" || p == name || glob_match_simple(p, name));
- if named && !state.engine.is_index_loaded(name) {
- if let Err(e) = state.engine.reopen_index(name) {
- return ApiError::new(xerj_common::XerjError::from(e)).into_response();
- }
- }
- }
let handles = match resolve_indices_for_op(&state, &index).await {
Ok(h) => h,
Err(e) => return ApiError::new(e).into_response(),
diff --git a/engine/crates/xerj-api/tests/close_frees_memory.rs b/engine/crates/xerj-api/tests/close_frees_memory.rs
index a11d229a6..33fe4364c 100644
--- a/engine/crates/xerj-api/tests/close_frees_memory.rs
+++ b/engine/crates/xerj-api/tests/close_frees_memory.rs
@@ -132,11 +132,12 @@ async fn probe_close_hydration_and_reopen_roundtrip() {
assert_eq!(total, Some(200), "reopen must be lossless: {q}");
}
-/// #463 (+#206 visibility guarantee): a closed index whose in-memory handle was
-/// released must still appear in `_cat/indices` with status `close` — omitting
-/// it is the exact invisibility regression #206 fixed for failed indices. This
-/// is the fail-before for the release change (before the cat_indices fix the row
-/// vanishes because the released index is absent from the loaded-only listing).
+/// #463: releasing a closed index's RAM must NOT make it disappear from the
+/// loaded-index views. Because the `Arc<Index>` shell stays in `Engine::indices`,
+/// a closed index remains listed in `_cat/indices` (and `_cluster/health`, etc.)
+/// exactly as before — this guards against a regression back to the
+/// remove-from-`indices` approach that vanished closed indices (the #206-class
+/// invisibility that broke ES-compat conformance).
#[tokio::test]
async fn closed_index_stays_visible_in_cat_indices() {
let (app, _dir) = app().await;
@@ -155,17 +156,11 @@ async fn closed_index_stays_visible_in_cat_indices() {
let (st, after) = call(&app, "GET", "/_cat/indices?format=json", Value::Null).await;
assert_eq!(st, StatusCode::OK);
- let row = after
- .as_array()
- .and_then(|a| a.iter().find(|r| r["index"] == "docs"));
assert!(
- row.is_some(),
- "#463/#206: a closed index must stay visible in _cat/indices, not vanish: {after}"
- );
- assert_eq!(
- row.unwrap()["status"],
- "close",
- "#463: the closed index must report status close: {after}"
+ after
+ .as_array()
+ .is_some_and(|a| a.iter().any(|r| r["index"] == "docs")),
+ "#463: a closed index must stay visible in _cat/indices, not vanish: {after}"
);
// And a concrete-name lookup of the closed index must not 404.
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index bc7d5272c..3c796055e 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -3208,32 +3208,30 @@ impl Engine {
/// index fully in service.
pub async fn close_index(&self, name: &str) -> Result<()> {
self.ensure_cluster_state_writable()?;
- if self.indices.contains_key(name) {
- // Durability barrier before the handle is dropped.
- self.flush_index(name).await?;
- // Drop the strong ref held by the engine. Any in-flight request that
- // cloned the `Arc` keeps its own copy alive until it completes, then
- // the index (and its governor-registered segment caches) frees.
- self.indices.remove(name);
+ if let Some(idx) = self.indices.get(name).map(|r| Arc::clone(r.value())) {
+ // Durability barrier: flush the memtable to disk BEFORE releasing the
+ // caches, so no not-yet-published document is lost. A flush failure
+ // aborts the close and leaves the index fully in service.
+ idx.flush().await?;
+ // Free the rebuildable in-memory caches (memtable is now empty; the
+ // per-segment caches re-hydrate from disk on the next read). The
+ // `Arc<Index>` shell STAYS in `self.indices`, so every loaded-index
+ // view (`_cat/indices`, `_cluster/health`, `_cluster/state`, the
+ // `expand_wildcards` filtering) is unchanged — a closed index is
+ // loaded-and-flagged, matching the rest of the codebase's model.
+ idx.release_memory();
}
self.closed_indices.insert(name.to_string(), true);
- info!(name, "index closed; in-memory handle released");
+ info!(name, "index closed; in-memory caches released");
Ok(())
}
/// `_open` counterpart to [`close_index`](Self::close_index): clears the
- /// closed flag and, if the index was memory-released, reconstructs its
- /// `Arc<Index>` from disk via [`Index::open`] (which replays the WAL and
- /// republishes segments). A no-op for an index that is already loaded.
+ /// closed flag. The index handle was never removed (only its caches were
+ /// released), so there is nothing to reconstruct — subsequent reads
+ /// re-hydrate segments from disk on demand.
pub fn reopen_index(&self, name: &str) -> Result<()> {
self.ensure_cluster_state_writable()?;
- if !self.indices.contains_key(name) {
- let index_name = IndexName::new(name).map_err(EngineError::Common)?;
- // `Index::open` returns an `Arc<Index>` (replays the WAL and
- // republishes segments from disk) — insert it directly.
- let idx = Index::open(index_name, &self.config, &self.data_dir)?;
- self.indices.insert(name.to_string(), idx);
- }
self.closed_indices.remove(name);
info!(name, "index reopened");
Ok(())
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 4048e3be9..b55c7cc43 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -18887,6 +18887,54 @@ impl Index {
self.flush_byte_threshold
}
+ /// Release this index's rebuildable in-memory caches (#463). Every one of
+ /// these DashMaps holds hydrated-from-disk data (or recomputable results):
+ /// clearing them frees the resident payload — the `Resident` wrappers refund
+ /// their hydration budget on drop — and the next read re-hydrates from disk,
+ /// exactly as a cache eviction under memory pressure already does. The caller
+ /// MUST flush the memtable first (see `Engine::close_index`) so no
+ /// not-yet-published document is lost; caches only ever mirror durable data.
+ /// The `Arc<Index>` shell itself stays in `Engine::indices` so every
+ /// loaded-index view (`_cat/indices`, `_cluster/health`, …) is unchanged.
+ pub fn release_memory(&self) {
+ self.dv_cache.clear();
+ self.sort_shadow_cache.clear();
+ self.range_prefilter_cache.clear();
+ self.id_pos_cache.clear();
+ self.row_seq_cache.clear();
+ self.stored_value_cache.clear();
+ self.stored_slices_cache.clear();
+ self.decoded_stored_cache.clear();
+ self.fts_reader_cache.clear();
+ self.shortcut_count_cache.clear();
+ self.ghost_positions_cache.clear();
+ self.regexp_expand_cache.clear();
+ self.fast_date_cache.clear();
+ self.fast_date_sorted_cache.clear();
+ self.query_cache.clear();
+ }
+
+ /// Total resident cache entries across every per-segment cache — a
+ /// deterministic observable for the `release_memory` effect (#463 tests):
+ /// non-zero after a query hydrates segments, zero after `release_memory`.
+ pub fn total_cache_entries(&self) -> usize {
+ self.dv_cache.len()
+ + self.sort_shadow_cache.len()
+ + self.range_prefilter_cache.len()
+ + self.id_pos_cache.len()
+ + self.row_seq_cache.len()
+ + self.stored_value_cache.len()
+ + self.stored_slices_cache.len()
+ + self.decoded_stored_cache.len()
+ + self.fts_reader_cache.len()
+ + self.shortcut_count_cache.len()
+ + self.ghost_positions_cache.len()
+ + self.regexp_expand_cache.len()
+ + self.fast_date_cache.len()
+ + self.fast_date_sorted_cache.len()
+ + self.query_cache.len()
+ }
+
// ── Stats ─────────────────────────────────────────────────────────────────
/// Return statistics for this index.
diff --git a/engine/crates/xerj-engine/tests/close_releases_index.rs b/engine/crates/xerj-engine/tests/close_releases_index.rs
index c5775744d..7e8560da2 100644
--- a/engine/crates/xerj-engine/tests/close_releases_index.rs
+++ b/engine/crates/xerj-engine/tests/close_releases_index.rs
@@ -43,31 +43,50 @@ async fn seed(engine: &Engine, name: &str, n: usize) {
}
}
-/// `close_index` releases the in-memory handle; `reopen_index` reconstructs it
-/// from disk with no data loss. FAIL-BEFORE: with the release hunk reverted the
-/// handle stays loaded after close, so the `!is_index_loaded` assertion fails.
+/// Run a size>0 search so the per-segment caches actually hydrate.
+async fn hydrate(engine: &Engine, name: &str) -> u64 {
+ let idx = engine.get_index(name).expect("get index");
+ let req = parse_request(&json!({ "query": { "match": { "body": "document" } }, "size": 25 }))
+ .expect("parse_request");
+ idx.search(&req).await.expect("search").total.value
+}
+
+/// `close_index` releases the resident caches while keeping the `Arc<Index>`
+/// shell loaded (so `_cat`/`_cluster` views are unchanged); reads re-hydrate on
+/// reopen with no data loss. FAIL-BEFORE: with the `idx.release_memory()` call
+/// reverted, the caches stay populated so `total_cache_entries == 0` fails.
#[tokio::test]
-async fn close_releases_handle_and_reopen_is_lossless() {
+async fn close_releases_caches_and_reopen_is_lossless() {
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
seed(&engine, "docs", 50).await;
engine.get_index("docs").unwrap().refresh().await.unwrap();
assert!(engine.is_index_loaded("docs"), "loaded before close");
- assert_eq!(count_all(&engine, "docs").await, 50);
+ hydrate(&engine, "docs").await;
+ let cached_before = engine.get_index("docs").unwrap().total_cache_entries();
+ assert!(
+ cached_before > 0,
+ "a query should have hydrated the caches, got {cached_before}"
+ );
engine.close_index("docs").await.expect("close");
+ // The shell stays loaded (loaded-index views unchanged), but caches are freed.
assert!(
- !engine.is_index_loaded("docs"),
- "#463: _close must release the in-memory Arc<Index>, not just flip a flag"
+ engine.is_index_loaded("docs"),
+ "a closed index stays loaded (shell) — matches _cat/_cluster's model"
);
assert!(
engine.closed_indices.contains_key("docs"),
- "close still marks the index closed"
+ "close marks the index closed"
+ );
+ let cached_after = engine.get_index("docs").unwrap().total_cache_entries();
+ assert_eq!(
+ cached_after, 0,
+ "#463: _close must release the resident caches (was {cached_before}, now {cached_after})"
);
engine.reopen_index("docs").expect("reopen");
- assert!(engine.is_index_loaded("docs"), "reopen reloads the handle");
assert!(
!engine.closed_indices.contains_key("docs"),
"reopen clears the closed flag"
@@ -75,13 +94,13 @@ async fn close_releases_handle_and_reopen_is_lossless() {
assert_eq!(
count_all(&engine, "docs").await,
50,
- "#463: reopen must be lossless"
+ "#463: reopen re-hydrates losslessly"
);
}
/// Data-safety: documents indexed but NOT refreshed/flushed before close must
-/// survive close→open (close flushes; reopen replays). Guards against a memory
-/// win that silently drops the memtable.
+/// survive close→reopen (close flushes the memtable BEFORE releasing caches).
+/// Guards against a memory win that silently drops the memtable.
#[tokio::test]
async fn close_flushes_so_unrefreshed_docs_survive_reopen() {
let dir = TempDir::new().unwrap();
@@ -89,8 +108,6 @@ async fn close_flushes_so_unrefreshed_docs_survive_reopen() {
seed(&engine, "docs", 30).await; // NO refresh — docs sit in the memtable/WAL
engine.close_index("docs").await.expect("close");
- assert!(!engine.is_index_loaded("docs"), "released on close");
-
engine.reopen_index("docs").expect("reopen");
assert_eq!(
count_all(&engine, "docs").await,