AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): bound transient read builds and stop the breaker cache thrash (#2712)
xerj-org314d8be97de3
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ab60ae2fb..d84a9cd1f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Search under a memory cap: when the segment hydration cache is full, a query that has to build a segment's stored slices or FTS reader outside the cache now reserves that memory from a separate read budget (`XERJ_READ_TRANSIENT_MB`, default a tenth of `XERJ_MAX_PROCESS_MEMORY_MB`). It waits up to `XERJ_READ_TRANSIENT_WAIT_MS` (default 5 s) for room instead of stacking whole decompressed sections on top of each other. Background cache warms that the full cache would refuse are skipped. While the memory breaker is engaged, its drain still flushes and purges on every tick but releases the read caches at most once per `XERJ_BREAKER_CACHE_RELEASE_SECS` (default 30 s), so queries no longer re-hydrate cold every two seconds. `GET /_nodes/stats` reports the budget under `indices.read_transient_budget`. (Refs #2427, #1122)- Opt-in lexical `judge` now matches CJK characters and adjacent pairs insideuninterrupted Chinese/Japanese/Korean text instead of treating the wholerun as one word (refs #2580). Single-character evidence is retained, pairsdiff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex c42faceed..7a91279b8 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -31677,6 +31677,17 @@ pub async fn nodes_stats(State(state): State<AppState>) -> impl IntoResponse {.unwrap_or_default().as_millis() as u64;+ let read_transient_budget = {+ let b = xerj_engine::read_budget::global().snapshot();+ json!({+ "limit_in_bytes": b.limit,+ "current_in_bytes": b.current,+ "peak_in_bytes": b.peak,+ "admitted": b.admitted,+ "waited": b.waited,+ "overruns": b.overruns,+ })+ };Json(json!({"_nodes": { "total": 1, "successful": 1, "failed": 0 },"cluster_name": "xerj",@@ -31741,6 +31752,9 @@ pub async fn nodes_stats(State(state): State<AppState>) -> impl IntoResponse {// bbq_hnsw/flat)."dense_vector": dense_vector_off_heap_stats(&state),"segment_hydration_cache": segment_hydration,+ // #2427/#1122: transient read-path builds the hydration+ // cache refused, bounded by their own budget.+ "read_transient_budget": read_transient_budget,},"transport": {"server_open": 1,diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex a3d61effc..52134eee2 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -222,6 +222,83 @@ pub fn set_allocator_purge_hook(hook: AllocatorPurgeHook) {*slot = Some(hook);}+/// Spaces breaker-driven cache releases (#2427/#1122). The first engaged+/// tick releases; later ticks within `interval` only flush and purge.+/// `XERJ_BREAKER_CACHE_RELEASE_SECS` sets the interval (default 30 s; `0`+/// releases on every tick, the pre-fix behaviour).+#[derive(Debug, Clone)]+pub struct PressureCacheReleaseGate {+ interval: std::time::Duration,+ last: Option<std::time::Instant>,+}++impl PressureCacheReleaseGate {+ pub const DEFAULT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);++ pub fn new(interval: std::time::Duration) -> Self {+ Self {+ interval,+ last: None,+ }+ }++ pub fn from_env() -> Self {+ let interval = std::env::var("XERJ_BREAKER_CACHE_RELEASE_SECS")+ .ok()+ .and_then(|v| v.trim().parse::<u64>().ok())+ .map(std::time::Duration::from_secs)+ .unwrap_or(Self::DEFAULT_INTERVAL);+ Self::new(interval)+ }++ pub fn interval(&self) -> std::time::Duration {+ self.interval+ }++ /// Whether a drain at `now` should release caches; records it if so.+ pub fn should_release(&mut self, now: std::time::Instant) -> bool {+ let due = self+ .last+ .is_none_or(|last| now.saturating_duration_since(last) >= self.interval);+ if due {+ self.last = Some(now);+ }+ due+ }+}++#[cfg(test)]+mod pressure_cache_release_gate_tests {+ use super::PressureCacheReleaseGate;+ use std::time::{Duration, Instant};++ /// #2427/#1122: the breaker requests a drain every sampler tick while+ /// engaged. Only the first tick in each interval may release caches.+ #[test]+ fn engaged_ticks_release_caches_at_most_once_per_interval() {+ let mut gate = PressureCacheReleaseGate::new(Duration::from_secs(30));+ let t0 = Instant::now();+ assert!(gate.should_release(t0), "the first engaged tick releases");+ for tick in 1..15u64 {+ assert!(+ !gate.should_release(t0 + Duration::from_secs(2 * tick)),+ "tick at {}s must only flush and purge",+ 2 * tick+ );+ }+ assert!(gate.should_release(t0 + Duration::from_secs(30)));+ assert!(!gate.should_release(t0 + Duration::from_secs(31)));+ }++ #[test]+ fn a_zero_interval_releases_on_every_tick() {+ let mut gate = PressureCacheReleaseGate::new(Duration::ZERO);+ let t0 = Instant::now();+ assert!(gate.should_release(t0));+ assert!(gate.should_release(t0));+ }+}+/// Run the installed allocator purge, if any. Never panics: a broken hook/// must not take the drain (or the process) down — purging is an/// optimisation for RSS return, not a correctness step.@@ -4457,6 +4534,29 @@ impl Engine {self.release_rebuildable_memory(&loaded).await}+ /// The breaker-driven drain (#2427/#1122): always flush memtables and+ /// purge the allocator, but release the rebuildable caches only when+ /// `release_caches` is set. The sampler requests a drain on every tick+ /// while the breaker is engaged; releasing every cache each time made a+ /// read-heavy node re-hydrate cold on every query, which refilled RSS and+ /// re-engaged the breaker — a ~2 s thrash loop that drained nothing+ /// durable. The worker spaces cache releases with+ /// [`PressureCacheReleaseGate`]; `_cache/clear` and the direct+ /// [`Self::drain_rebuildable_memory`] still release every time.+ pub async fn pressure_drain(&self, release_caches: bool) -> usize {+ let loaded = self.open_indices_largest_memtable_first();+ if release_caches {+ return self.release_rebuildable_memory(&loaded).await;+ }+ for idx in &loaded {+ if idx.memtable_bytes() > 0 && idx.flush().await.is_err() {+ tracing::warn!("memory pressure drain: flush failed");+ }+ }+ purge_allocator_pages();+ loaded.len()+ }+/// Open indices, largest live memtable first (#2321).////// The summed budget stays engaged until enough bytes are retired. DashMap@@ -4575,15 +4675,25 @@ impl Engine {// automatic wiring is inert.return;};+ let mut release_gate = PressureCacheReleaseGate::from_env();while let Ok(()) = drain_rx.recv() {let Some(engine) = weak_c.upgrade() else {return; // engine dropped};- let drained = handle.block_on(engine.drain_rebuildable_memory());- tracing::info!(+ let release_caches = release_gate.should_release(std::time::Instant::now());+ let drained = handle.block_on(engine.pressure_drain(release_caches));+ if release_caches {+ tracing::info!(indexes = drained,"memory breaker engaged — drained rebuildable state (flush + cache release + purge)");+ } else {+ tracing::debug!(+ indexes = drained,+ "memory breaker engaged — flushed and purged; caches kept (released within the last {:?})",+ release_gate.interval()+ );+ }}});// Publish the request channel only once a worker is draining it: withdiff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 5d14b6952..576b2d33b 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -37,6 +37,7 @@ use xerj_vector::hnsw::{HnswIndex, HnswParams};use xerj_vector::{Sq8CodeStore, Sq8Params};use crate::aggs::run_aggs_with_schema;+use crate::read_budget::{ReadPermit, ReadTransientBudget};use crate::segment_cache_budget::{CacheResident, SegmentCacheCategory, SegmentHydrationBudget, CATEGORY_COUNT,};@@ -62,6 +63,34 @@ fn index_segment_hydration_budget(_config: &Config) -> Arc<SegmentHydrationBudgepanic!("Index construction requires the process ResourceGovernor to be initialised");}+/// Reserve `bytes` of transient read memory unless the hydration cache has+/// room to keep the value (#2427/#1122). A value the cache admits is+/// accounted there; only one it will refuse is a per-query allocation.+fn transient_read_permit(+ hydration: &SegmentHydrationBudget,+ read_budget: &Arc<ReadTransientBudget>,+ bytes: u64,+) -> Option<ReadPermit> {+ if hydration.has_headroom(bytes) {+ None+ } else {+ Some(read_budget.acquire(bytes))+ }+}++/// Retained-size bound of a `StoredSlices` built from `raw` (the framing-only+/// estimator; a zstd frame without a content size falls back to 8x raw).+fn stored_slices_bound(raw: &[u8], expect_docs: u64) -> u64 {+ xerj_storage::stored_codec::stored_slices_retained_upper_bound(raw, expect_docs)+ .ok()+ .flatten()+ .unwrap_or_else(|| {+ (raw.len() as u64)+ .saturating_mul(8)+ .saturating_add(expect_docs.saturating_mul(8))+ })+}+/// Wait for a publication test barrier, on the runtime and bounded.////// A test must never park a tokio BLOCKING-pool thread to wait for a signal@@ -2015,6 +2044,74 @@ mod flush_publication_recovery_tests {assert_eq!(idx.search(&request).await.unwrap().hits[0].id, "survivor");}+ /// #2427/#1122: a stored-slices build the hydration cache cannot hold is+ /// transient, and it must hold a read-budget reservation for as long as a+ /// query holds the value — so concurrent cold queries are bounded by the+ /// read budget instead of stacking whole stored sections. A build the+ /// cache can hold reserves nothing (the cache accounts it).+ ///+ /// Fails before the fix (with `transient_read_permit` returning `None`):+ /// the read budget never sees the transient bytes.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn transient_stored_slices_hold_a_read_reservation_until_dropped() {+ let dir = tempfile::tempdir().unwrap();+ let engine = crate::Engine::new(topk_projection_config(dir.path())).unwrap();+ engine+ .create_index("read-budget", topk_projection_schema())+ .unwrap();+ let idx = engine.get_index("read-budget").unwrap();+ idx.abort_background_tasks();+ for i in 0..512u64 {+ idx.index_document(+ Some(format!("doc-{i}")),+ json!({"title": format!("title {i}"), "body": format!("body text {i}"), "blob": "x".repeat(64)}),+ )+ .await+ .unwrap();+ }+ idx.flush().await.unwrap();+ let seg = idx.store.snapshot().segments[0].clone();+ let read = ReadTransientBudget::new(1 << 30, std::time::Duration::from_secs(5));+ idx.release_memory();+ assert!(+ !idx.stored_slices_cache.contains_key(&seg.id),+ "precondition: cold"+ );++ // No room in the cache: transient, reserved until the value drops.+ let full = SegmentHydrationBudget::new(0);+ let slices = idx+ .stored_slices_for_with(&seg.id, seg.doc_count, &full, &read)+ .expect("slices build");+ assert!(!idx.stored_slices_cache.contains_key(&seg.id));+ let held = read.snapshot().current;+ assert!(+ held >= slices.bytes.len() as u64,+ "the transient build must reserve at least its decoded bytes ({} held, {} decoded)",+ held,+ slices.bytes.len()+ );+ let second = Arc::clone(&slices);+ drop(slices);+ assert_eq!(read.snapshot().current, held, "a live reader keeps it");+ drop(second);+ assert_eq!(read.snapshot().current, 0, "released with the last reader");++ // Room in the cache: admitted, nothing reserved.+ let roomy = SegmentHydrationBudget::new(u64::MAX);+ let cached = idx+ .stored_slices_for_with(&seg.id, seg.doc_count, &roomy, &read)+ .expect("slices build");+ assert!(idx.stored_slices_cache.contains_key(&seg.id));+ assert_eq!(read.snapshot().current, 0);+ assert_eq!(+ read.snapshot().admitted,+ 1,+ "only the transient build reserved"+ );+ drop(cached);+ }+fn topk_projection_config(dir: &std::path::Path) -> Config {let mut config = Config::default();config.server.data_dir = dir.to_string_lossy().into_owned();@@ -10445,6 +10542,9 @@ pub struct Index {self_ref: std::sync::OnceLock<std::sync::Weak<Index>>,/// Segments with a background stored-slices warm queued or running.cold_stored_warms: dashmap::DashMap<String, ()>,+ /// Background warms skipped because the hydration budget had no room+ /// for the slices they would build (#2427/#1122).+ cold_stored_warms_skipped: AtomicU64,/// Cold segments answered by row-selective hydration (observability and/// tests).cold_row_hydrations: AtomicU64,@@ -11150,6 +11250,7 @@ impl Index {schema_persist_failures: AtomicU64::new(0),self_ref: std::sync::OnceLock::new(),cold_stored_warms: dashmap::DashMap::new(),+ cold_stored_warms_skipped: AtomicU64::new(0),cold_row_hydrations: AtomicU64::new(0),});let _ = index.self_ref.set(Arc::downgrade(&index));@@ -11615,6 +11716,7 @@ impl Index {schema_persist_failures: AtomicU64::new(0),self_ref: std::sync::OnceLock::new(),cold_stored_warms: dashmap::DashMap::new(),+ cold_stored_warms_skipped: AtomicU64::new(0),cold_row_hydrations: AtomicU64::new(0),});let _ = index.self_ref.set(Arc::downgrade(&index));@@ -17992,7 +18094,19 @@ impl Index {}let segment_id = segment_id.to_string();runtime.spawn_blocking(move || {- if !index.stored_slices_cache.contains_key(segment_id.as_str()) {+ // #2427/#1122: a warm exists to leave the slices cached. When the+ // hydration budget has no room for them the cache would refuse+ // the result, so the warm would only decompress a whole stored+ // section to drop it — on every cold query, the transient spike+ // that kept a search-only node above the breaker watermark.+ let fits = index+ .stored_slices_bound_for(&segment_id, doc_count)+ .is_some_and(|bound| index.segment_hydration_budget.has_headroom(bound));+ if !fits {+ index+ .cold_stored_warms_skipped+ .fetch_add(1, Ordering::Relaxed);+ } else if !index.stored_slices_cache.contains_key(segment_id.as_str()) {let _ = index.stored_slices_for(&segment_id, doc_count);}index.cold_stored_warms.remove(segment_id.as_str());@@ -31598,6 +31712,32 @@ impl Index {value: T,budget: &Arc<SegmentHydrationBudget>,) -> Resident<T> {+ self.publish_current_permitted(map, segment_id, key, category, bytes, value, budget, None)+ }++ /// [`Self::publish_current_with_budget`] for a value built under a+ /// transient read reservation (#2427/#1122). Admitted to the cache, the+ /// reservation is returned at once — the hydration budget now accounts+ /// the bytes. Refused, the value is served uncached and keeps the+ /// reservation until its last reader drops it, so concurrent transient+ /// builds stay inside the read budget.+ #[allow(clippy::too_many_arguments)]+ fn publish_current_permitted<T>(+ &self,+ map: &dashmap::DashMap<String, Resident<T>>,+ segment_id: &str,+ key: String,+ category: SegmentCacheCategory,+ bytes: u64,+ value: T,+ budget: &Arc<SegmentHydrationBudget>,+ mut permit: Option<ReadPermit>,+ ) -> Resident<T> {+ // The permit was sized from a pre-build estimate; account the+ // measured size while the transient lives.+ if let Some(permit) = permit.as_mut() {+ permit.true_up(bytes);+ }#[cfg(test)]{let bit = 1_u64 << category as usize;@@ -31620,14 +31760,14 @@ impl Index {.iter().any(|meta| meta.id == segment_id){- return CacheResident::uncached(value);+ return CacheResident::uncached_with_permit(value, permit);}use dashmap::mapref::entry::Entry;match map.entry(key) {Entry::Occupied(entry) => Arc::clone(entry.get()),Entry::Vacant(entry) => {let Some(charge) = budget.try_charge(category, bytes) else {- return CacheResident::uncached(value);+ return CacheResident::uncached_with_permit(value, permit);};let resident = CacheResident::admitted(value, charge, category, bytes);entry.insert(Arc::clone(&resident));@@ -31710,15 +31850,23 @@ impl Index {return Some(Arc::clone(hit.value()));}+ // #2427/#1122: a reader the cache cannot hold is built per query and+ // decompresses every requested field's postings; reserve that in the+ // read budget before paying it (see `stored_slices_for_with`).+ let read_budget = crate::read_budget::global();+ let estimate = FtsIndexReader::estimate_open_bytes(fts_dir, segment_id, fields);+ let permit = transient_read_permit(&self.segment_hydration_budget, read_budget, estimate);let reader = Arc::new(FtsIndexReader::open(fts_dir, segment_id, fields).ok()?);let bytes = reader.retained_bytes();- Some(self.publish_current(+ Some(self.publish_current_permitted(&self.fts_reader_cache,segment_id,key,SegmentCacheCategory::FtsReader,bytes,reader,+ &self.segment_hydration_budget,+ permit,))}@@ -34880,6 +35028,29 @@ impl Index {/// malformed section (offsets incomplete) — caller falls back to the/// legacy scan.fn stored_slices_for(&self, seg_id: &str, expect_docs: u64) -> Option<Resident<StoredSlices>> {+ self.stored_slices_for_with(+ seg_id,+ expect_docs,+ &self.segment_hydration_budget,+ crate::read_budget::global(),+ )+ }++ /// [`Self::stored_slices_for`] against explicit budgets (tests pin them).+ ///+ /// #2427/#1122: when the hydration cache has no room for this segment the+ /// build is transient — decoded, used by this query, dropped. It reserves+ /// its retained-size bound in `read_budget` *before* the decompress and+ /// keeps the reservation until the last reader drops the value, so+ /// concurrent cold queries queue for read memory instead of stacking+ /// whole stored sections until the shared breaker engages.+ fn stored_slices_for_with(+ &self,+ seg_id: &str,+ expect_docs: u64,+ hydration: &Arc<SegmentHydrationBudget>,+ read_budget: &Arc<ReadTransientBudget>,+ ) -> Option<Resident<StoredSlices>> {if let Some(entry) = self.stored_slices_cache.get(seg_id) {return Some(Arc::clone(entry.value()));}@@ -34902,12 +35073,24 @@ impl Index {}// Reuse already-decompressed bytes from `decoded_stored_cache`// before paying a fresh open + decompress.- let bytes: Vec<u8> = if let Some(entry) = self.decoded_stored_cache.get(seg_id) {- entry.value().value().clone()+ let decoded = self+ .decoded_stored_cache+ .get(seg_id)+ .map(|entry| Arc::clone(entry.value()));+ let (bytes, permit): (Vec<u8>, Option<ReadPermit>) = if let Some(decoded) = decoded {+ let bound =+ (decoded.value().len() as u64).saturating_add(expect_docs.saturating_mul(8));+ let permit = transient_read_permit(hydration, read_budget, bound);+ (decoded.value().clone(), permit)} else {let reader = self.store.open_segment_arc(seg_id).ok()?;let raw = reader.section(SectionType::Stored).ok()??;- xerj_storage::stored_codec::decode_stored(raw).ok()?+ let permit = transient_read_permit(+ hydration,+ read_budget,+ stored_slices_bound(raw, expect_docs),+ );+ (xerj_storage::stored_codec::decode_stored(raw).ok()?, permit)};if bytes.len() > u32::MAX as usize {return None;@@ -34924,16 +35107,26 @@ impl Index {slices.bytes.capacity(),slices.offsets.capacity(),);- Some(self.publish_current(+ Some(self.publish_current_permitted(&self.stored_slices_cache,seg_id,key,SegmentCacheCategory::StoredSlices,bytes,slices,+ hydration,+ permit,))}+ /// Retained-size bound of this segment's `StoredSlices`, from framing+ /// only (no decompress). `None` when the segment or section is missing.+ fn stored_slices_bound_for(&self, seg_id: &str, expect_docs: u64) -> Option<u64> {+ let reader = self.store.open_segment_arc(seg_id).ok()?;+ let raw = reader.section(SectionType::Stored).ok()??;+ Some(stored_slices_bound(raw, expect_docs))+ }+/// `_id` of every row of a segment's columnar stored section, read from/// its identity columns only. `None` (legacy layout, read error, a row/// count other than `expect_docs`) sends the caller to the full decode.diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rsindex 3b93c1a44..3605ead92 100644--- a/engine/crates/xerj-engine/src/lib.rs+++ b/engine/crates/xerj-engine/src/lib.rs@@ -36,6 +36,7 @@ mod node_lock;pub mod painless;pub mod pools;pub mod rbac;+pub mod read_budget;pub mod secret_hash;pub mod segment_cache_budget;pub mod segment_cache_estimates;diff --git a/engine/crates/xerj-engine/src/read_budget.rs b/engine/crates/xerj-engine/src/read_budget.rsnew file mode 100644index 000000000..dae97e71b--- /dev/null+++ b/engine/crates/xerj-engine/src/read_budget.rs@@ -0,0 +1,323 @@+//! Process-wide budget for query-time *transient* hydration (#2427, #1122).+//!+//! A query that misses the segment hydration cache still builds the whole+//! value — a segment's decompressed stored section, or every requested+//! field's decompressed postings and norms — and, when the cache budget+//! refuses it, uses that value once and throws it away. Nothing bounded+//! those transient builds: a `{prefix}-*` `multi_match` fanned out over 49+//! indexes and every cold segment in them, the RSS sampler saw the spike,+//! the shared memory breaker engaged, and the write path answered 429 for+//! memory the *read* path had allocated.+//!+//! Every such build now reserves its estimated size here first. A+//! reservation that does not fit waits (back-pressure) until earlier+//! transient values are dropped. Two rules keep that wait safe:+//!+//! * **Progress.** A request is always admitted when nothing is held, so a+//! value larger than the whole budget is still served — alone.+//! * **No hold-and-wait.** A thread that already holds a reservation is+//! admitted without waiting. A query that holds a postings reader while it+//! asks for the same segment's stored bytes can therefore never deadlock+//! against another query doing the same.+//!+//! The wait is capped (`XERJ_READ_TRANSIENT_WAIT_MS`, default 5 s). A+//! reservation that times out is still admitted and counted in `overruns`,+//! so a slow holder delays a query rather than failing it, and the counter+//! shows an operator that the budget is too small for the workload.+//!+//! This is retained-payload accounting, like the hydration budget: decode+//! scratch and allocator slack are outside it.++use std::cell::Cell;+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::{Arc, Condvar, Mutex, OnceLock};+use std::time::{Duration, Instant};++const MIB: u64 = 1024 * 1024;+/// Fallback when no governor is installed (an `Index` built directly).+const DEFAULT_LIMIT_BYTES: u64 = 256 * MIB;+/// Share of the process memory limit given to transient read hydration.+const LIMIT_DIVISOR: u64 = 10;+const DEFAULT_MAX_WAIT: Duration = Duration::from_secs(5);++thread_local! {+ /// Reservations currently held by this thread (see "No hold-and-wait").+ static HELD_ON_THREAD: Cell<u32> = const { Cell::new(0) };+}++#[derive(Debug, Clone, Copy, Eq, PartialEq, serde::Serialize)]+pub struct ReadTransientBudgetSnapshot {+ pub limit: u64,+ pub current: u64,+ pub peak: u64,+ pub admitted: u64,+ pub waited: u64,+ pub overruns: u64,+}++pub struct ReadTransientBudget {+ limit: u64,+ max_wait: Duration,+ used: Mutex<u64>,+ released: Condvar,+ peak: AtomicU64,+ admitted: AtomicU64,+ waited: AtomicU64,+ overruns: AtomicU64,+}++impl ReadTransientBudget {+ pub fn new(limit: u64, max_wait: Duration) -> Arc<Self> {+ Arc::new(Self {+ limit,+ max_wait,+ used: Mutex::new(0),+ released: Condvar::new(),+ peak: AtomicU64::new(0),+ admitted: AtomicU64::new(0),+ waited: AtomicU64::new(0),+ overruns: AtomicU64::new(0),+ })+ }++ pub fn limit(&self) -> u64 {+ self.limit+ }++ /// Reserve `bytes` for one transient value, waiting for room if needed.+ /// The reservation is returned when the permit is dropped.+ pub fn acquire(self: &Arc<Self>, bytes: u64) -> ReadPermit {+ let reentrant = HELD_ON_THREAD.with(|held| held.get() > 0);+ let mut used = self.used.lock().unwrap_or_else(|p| p.into_inner());+ let fits = |used: u64| used == 0 || used.saturating_add(bytes) <= self.limit;+ if !reentrant && !fits(*used) {+ self.waited.fetch_add(1, Ordering::Relaxed);+ let deadline = Instant::now() + self.max_wait;+ while !fits(*used) {+ let now = Instant::now();+ if now >= deadline {+ self.overruns.fetch_add(1, Ordering::Relaxed);+ tracing::warn!(+ requested_bytes = bytes,+ in_use_bytes = *used,+ limit_bytes = self.limit,+ "read transient budget: waited {:?} for room; admitting over budget \+ (raise XERJ_READ_TRANSIENT_MB if this repeats)",+ self.max_wait+ );+ break;+ }+ used = self+ .released+ .wait_timeout(used, deadline - now)+ .unwrap_or_else(|p| p.into_inner())+ .0;+ }+ }+ *used = used.saturating_add(bytes);+ self.peak.fetch_max(*used, Ordering::Relaxed);+ self.admitted.fetch_add(1, Ordering::Relaxed);+ drop(used);+ HELD_ON_THREAD.with(|held| held.set(held.get().saturating_add(1)));+ ReadPermit {+ budget: Arc::clone(self),+ bytes,+ thread: std::thread::current().id(),+ }+ }++ pub fn snapshot(&self) -> ReadTransientBudgetSnapshot {+ let current = *self.used.lock().unwrap_or_else(|p| p.into_inner());+ ReadTransientBudgetSnapshot {+ limit: self.limit,+ current,+ peak: self.peak.load(Ordering::Relaxed),+ admitted: self.admitted.load(Ordering::Relaxed),+ waited: self.waited.load(Ordering::Relaxed),+ overruns: self.overruns.load(Ordering::Relaxed),+ }+ }++ fn release(&self, bytes: u64) {+ let mut used = self.used.lock().unwrap_or_else(|p| p.into_inner());+ *used = used.saturating_sub(bytes);+ drop(used);+ self.released.notify_all();+ }+}++/// One transient value's reservation. Travels with the value (inside its+/// `CacheResident`) and is returned when the last reference is dropped.+pub struct ReadPermit {+ budget: Arc<ReadTransientBudget>,+ bytes: u64,+ thread: std::thread::ThreadId,+}++impl ReadPermit {+ pub fn bytes(&self) -> u64 {+ self.bytes+ }++ /// Correct the reservation to the value's measured size once it is+ /// built. The reservation was taken from a pre-build estimate; growing it+ /// never waits (the memory is already allocated), it only makes later+ /// reservations see the true usage. Shrinking returns the excess.+ pub fn true_up(&mut self, actual: u64) {+ if actual == self.bytes {+ return;+ }+ let mut used = self.budget.used.lock().unwrap_or_else(|p| p.into_inner());+ if actual > self.bytes {+ *used = used.saturating_add(actual - self.bytes);+ self.budget.peak.fetch_max(*used, Ordering::Relaxed);+ drop(used);+ } else {+ *used = used.saturating_sub(self.bytes - actual);+ drop(used);+ self.budget.released.notify_all();+ }+ self.bytes = actual;+ }+}++impl Drop for ReadPermit {+ fn drop(&mut self) {+ // A permit dropped on another thread leaves the acquiring thread's+ // count raised: that thread then never waits again, which fails open+ // (no deadlock) rather than closed.+ if std::thread::current().id() == self.thread {+ HELD_ON_THREAD.with(|held| held.set(held.get().saturating_sub(1)));+ }+ self.budget.release(self.bytes);+ }+}++/// Resolve the budget from `XERJ_READ_TRANSIENT_MB` (`off`/`unlimited`, or a+/// positive MiB value), else a tenth of the governor's process memory limit.+fn resolve_limit(env: Option<&str>, process_limit: Option<u64>) -> u64 {+ match env.map(str::trim) {+ Some("off") | Some("unlimited") => return u64::MAX,+ Some(value) => match value.parse::<u64>() {+ Ok(mb) if mb > 0 => return mb.saturating_mul(MIB),+ _ => tracing::warn!(+ "invalid XERJ_READ_TRANSIENT_MB={value:?}; expected off, unlimited, or a \+ positive MiB value; using the default"+ ),+ },+ None => {}+ }+ match process_limit {+ Some(limit) if limit > 0 => (limit / LIMIT_DIVISOR).max(16 * MIB),+ _ => DEFAULT_LIMIT_BYTES,+ }+}++static GLOBAL: OnceLock<Arc<ReadTransientBudget>> = OnceLock::new();++/// The process-wide budget, created on first use.+pub fn global() -> &'static Arc<ReadTransientBudget> {+ GLOBAL.get_or_init(|| {+ let env = std::env::var("XERJ_READ_TRANSIENT_MB").ok();+ let process_limit = crate::governor::global().map(|g| g.process_memory_limit_bytes());+ let limit = resolve_limit(env.as_deref(), process_limit);+ let max_wait = std::env::var("XERJ_READ_TRANSIENT_WAIT_MS")+ .ok()+ .and_then(|v| v.trim().parse::<u64>().ok())+ .map(Duration::from_millis)+ .unwrap_or(DEFAULT_MAX_WAIT);+ ReadTransientBudget::new(limit, max_wait)+ })+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::sync::atomic::AtomicBool;++ #[test]+ fn reservations_over_the_limit_wait_for_a_release() {+ let budget = ReadTransientBudget::new(100, Duration::from_secs(10));+ let first = budget.acquire(80);+ let admitted = Arc::new(AtomicBool::new(false));+ let waiter = {+ let budget = Arc::clone(&budget);+ let admitted = Arc::clone(&admitted);+ std::thread::spawn(move || {+ let permit = budget.acquire(80);+ admitted.store(true, Ordering::SeqCst);+ drop(permit);+ })+ };+ std::thread::sleep(Duration::from_millis(100));+ assert!(+ !admitted.load(Ordering::SeqCst),+ "a second 80-byte reservation must wait while 80 of 100 bytes are held"+ );+ assert_eq!(budget.snapshot().current, 80);+ drop(first);+ waiter.join().unwrap();+ assert!(admitted.load(Ordering::SeqCst));+ let snap = budget.snapshot();+ assert_eq!(snap.current, 0);+ assert_eq!(snap.peak, 80, "the two reservations never overlapped");+ assert_eq!(snap.waited, 1);+ assert_eq!(snap.overruns, 0);+ }++ #[test]+ fn an_oversized_reservation_is_admitted_when_nothing_is_held() {+ let budget = ReadTransientBudget::new(10, Duration::from_secs(10));+ let permit = budget.acquire(1_000);+ assert_eq!(budget.snapshot().current, 1_000);+ drop(permit);+ assert_eq!(budget.snapshot().current, 0);+ }++ #[test]+ fn a_thread_that_holds_a_reservation_never_waits() {+ let budget = ReadTransientBudget::new(100, Duration::from_secs(10));+ let started = Instant::now();+ let a = budget.acquire(90);+ let b = budget.acquire(90);+ assert!(started.elapsed() < Duration::from_secs(1));+ assert_eq!(budget.snapshot().waited, 0);+ drop((a, b));+ assert_eq!(budget.snapshot().current, 0);+ }++ #[test]+ fn a_wait_past_the_cap_is_admitted_and_counted() {+ let budget = ReadTransientBudget::new(100, Duration::from_millis(50));+ let held = budget.acquire(100);+ let other = {+ let budget = Arc::clone(&budget);+ std::thread::spawn(move || drop(budget.acquire(10)))+ };+ other.join().unwrap();+ assert_eq!(budget.snapshot().overruns, 1);+ drop(held);+ }++ #[test]+ fn limit_resolution() {+ assert_eq!(resolve_limit(Some("64"), Some(1 << 40)), 64 * MIB);+ assert_eq!(resolve_limit(Some("off"), None), u64::MAX);+ assert_eq!(resolve_limit(None, Some(3072 * MIB)), 3072 * MIB / 10);+ assert_eq!(resolve_limit(Some("bogus"), None), DEFAULT_LIMIT_BYTES);+ assert_eq!(resolve_limit(None, Some(0)), DEFAULT_LIMIT_BYTES);+ }++ #[test]+ fn true_up_corrects_an_estimate_in_both_directions() {+ let budget = ReadTransientBudget::new(1_000, Duration::from_millis(50));+ let mut permit = budget.acquire(100);+ permit.true_up(400);+ assert_eq!(budget.snapshot().current, 400);+ assert_eq!(budget.snapshot().peak, 400);+ permit.true_up(50);+ assert_eq!(budget.snapshot().current, 50);+ drop(permit);+ assert_eq!(budget.snapshot().current, 0);+ }+}diff --git a/engine/crates/xerj-engine/src/segment_cache_budget.rs b/engine/crates/xerj-engine/src/segment_cache_budget.rsindex b2e06e719..91f002930 100644--- a/engine/crates/xerj-engine/src/segment_cache_budget.rs+++ b/engine/crates/xerj-engine/src/segment_cache_budget.rs@@ -9,6 +9,7 @@ use std::sync::atomic::{AtomicU64, Ordering};use std::sync::Arc;use crate::ingest_memory::{Category, Retained};+use crate::read_budget::ReadPermit;pub const CATEGORY_COUNT: usize = 9;@@ -95,6 +96,16 @@ impl SegmentHydrationBudget {self.limit}+ /// Whether a `bytes` charge would fit right now. Advisory (another thread+ /// may charge first): used to skip building a value the cache would only+ /// refuse, and to decide whether a build is transient.+ pub fn has_headroom(&self, bytes: u64) -> bool {+ self.used+ .load(Ordering::Acquire)+ .checked_add(bytes)+ .is_some_and(|next| next <= self.limit)+ }+pub fn try_charge(self: &Arc<Self>,category: SegmentCacheCategory,@@ -188,6 +199,9 @@ impl Drop for BudgetCharge {pub struct CacheResident<T> {value: T,_charge: Option<BudgetCharge>,+ /// A value the cache refused carries its transient read reservation+ /// (#2427/#1122), returned when the last reader drops it.+ _read_permit: Option<ReadPermit>,_trace_retained: Retained<'static>,}@@ -201,6 +215,7 @@ impl<T> CacheResident<T> {Arc::new(Self {value,_charge: Some(charge),+ _read_permit: None,_trace_retained: Retained::new(category.trace_category(),usize::try_from(bytes).unwrap_or(usize::MAX),@@ -209,9 +224,16 @@ impl<T> CacheResident<T> {}pub fn uncached(value: T) -> Arc<Self> {+ Self::uncached_with_permit(value, None)+ }++ /// An uncached value that keeps `permit` (its transient read reservation)+ /// until the last reference is dropped.+ pub fn uncached_with_permit(value: T, permit: Option<ReadPermit>) -> Arc<Self> {Arc::new(Self {value,_charge: None,+ _read_permit: permit,_trace_retained: Retained::disabled(),})}diff --git a/engine/crates/xerj-engine/tests/drain_releases_memory.rs b/engine/crates/xerj-engine/tests/drain_releases_memory.rsindex 9cd5f7365..79664db19 100644--- a/engine/crates/xerj-engine/tests/drain_releases_memory.rs+++ b/engine/crates/xerj-engine/tests/drain_releases_memory.rs@@ -184,3 +184,43 @@ async fn flush_prunes_published_storage_memtable_entries() {// readable, all durable.assert_eq!(count_all(&engine, "docs").await, 40);}++/// #2427/#1122: the breaker drain between cache releases flushes the+/// memtable (data safety, memory relief) but keeps the warm caches, so a+/// read-heavy node is not forced to re-hydrate cold on every query while the+/// breaker stays engaged — the thrash loop the gate showed.+#[tokio::test]+async fn pressure_drain_without_release_flushes_but_keeps_caches() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine, "docs", 50).await;+ engine.get_index("docs").unwrap().refresh().await.unwrap();+ hydrate(&engine, "docs").await;+ let cached_before = engine.get_index("docs").unwrap().total_cache_entries();+ assert!(cached_before > 0, "precondition: hydrated caches");++ // Unflushed docs must still be flushed by the keep-caches drain.+ let idx = engine.get_index("docs").unwrap();+ for i in 50..60 {+ idx.index_document(+ Some(format!("d{i}")),+ json!({ "body": format!("document number {i}") }),+ )+ .await+ .unwrap();+ }+ assert!(idx.memtable_bytes() > 0, "precondition: live memtable");++ let drained = engine.pressure_drain(false).await;+ assert_eq!(drained, 1);+ assert_eq!(idx.memtable_bytes(), 0, "the drain still flushes");+ assert!(+ idx.total_cache_entries() >= cached_before,+ "caches must survive a drain between releases (was {cached_before}, now {})",+ idx.total_cache_entries()+ );++ engine.pressure_drain(true).await;+ assert_eq!(idx.total_cache_entries(), 0, "a due release still releases");+ assert_eq!(count_all(&engine, "docs").await, 60, "lossless");+}diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rsindex d6bc389b3..0614063d5 100644--- a/engine/crates/xerj-fts/src/index.rs+++ b/engine/crates/xerj-fts/src/index.rs@@ -2496,6 +2496,104 @@ impl FtsIndexReader {Self::open_inner(segment_dir, segment_id, field_names, false)}+ /// Upper estimate of the heap a full [`Self::open`] of `field_names`+ /// would retain, read from file headers and sizes only (#2427/#1122).+ ///+ /// Lets the engine reserve read memory *before* it pays the postings+ /// decompress. Per field: the uncompressed postings length from the+ /// `ZPS1`/`ZPS2`/`ZPL1` envelope header (or the file length of a raw+ /// `.post`), the `.fst` file length, 8 bytes per dense norms entry, and+ /// the decoded `.meta` term records (`num_terms` from its header). A field whose files are+ /// missing or unreadable counts as zero — `open` will fail or skip it.+ pub fn estimate_open_bytes(+ segment_dir: impl AsRef<Path>,+ segment_id: &str,+ field_names: &[&str],+ ) -> u64 {+ use std::io::Read;+ let segment_dir = segment_dir.as_ref();+ let Ok(layout) = segment_filename_layout(segment_dir, segment_id) else {+ return 0;+ };+ let head = |path: &Path, n: usize| -> Option<Vec<u8>> {+ let mut buf = vec![0u8; n];+ let mut file = fs::File::open(path).ok()?;+ file.read_exact(&mut buf).ok()?;+ Some(buf)+ };+ let mut total: u64 = 0;+ for &field_name in field_names {+ let filename = |ext: &str| -> Option<PathBuf> {+ match layout {+ FilenameLayout::Legacy => {+ legacy_field_sidecar_path(segment_dir, segment_id, field_name, ext)+ }+ FilenameLayout::Encoded | FilenameLayout::CaseDistinguished => {+ Some(field_sidecar_path_for_layout(+ segment_dir,+ segment_id,+ field_name,+ ext,+ layout,+ ))+ }+ }+ };+ total = total.saturating_add(field_name.len() as u64);+ if let Some(post) = filename("post") {+ if let Some(h) = head(&post, 8) {+ let magic = &h[..4];+ if magic == POST_MAGIC_ZSTD_V2+ || magic == POST_MAGIC_ZSTD+ || magic == POST_MAGIC_LZ4+ {+ let len = u32::from_le_bytes([h[4], h[5], h[6], h[7]]);+ total = total.saturating_add(u64::from(len));+ } else {+ // Raw postings: mmap'd or read whole; count the file.+ let len = fs::metadata(&post).map(|m| m.len()).unwrap_or(0);+ total = total.saturating_add(len);+ }+ }+ }+ if let Some(fst) = filename("fst") {+ // An FST that cannot be mmap'd is read whole.+ let len = fs::metadata(&fst).map(|m| m.len()).unwrap_or(0);+ total = total.saturating_add(len);+ }+ if let Some(norms) = filename("norms") {+ let entries = match head(&norms, 9) {+ Some(h) if &h[..4] == NORMS_MAGIC => {+ u64::from(u32::from_le_bytes([h[5], h[6], h[7], h[8]]))+ }+ _ => fs::metadata(&norms).map(|m| m.len()).unwrap_or(0),+ };+ total = total.saturating_add(entries.saturating_mul(8));+ }+ if let Some(meta) = filename("meta") {+ // `num_terms` sits at byte 20 of every ZFM header.+ // ZFM3/4/5 expand to 24-byte flat records; legacy ZFM1/2+ // keep a term map (`retained_bytes` counts 64 B per term)+ // plus the term strings, which the file length covers.+ let len = fs::metadata(&meta).map(|m| m.len()).unwrap_or(0);+ let add = match head(&meta, 24) {+ Some(h) => {+ let terms = u64::from(u32::from_le_bytes([h[20], h[21], h[22], h[23]]));+ let magic = &h[..4];+ if magic == META_MAGIC_V1 || magic == META_MAGIC_V2 {+ terms.saturating_mul(64).saturating_add(len)+ } else {+ terms.saturating_mul(ZFM3_RECORD_LEN as u64)+ }+ }+ None => len.saturating_mul(4),+ };+ total = total.saturating_add(add);+ }+ }+ total+ }+/// Open ONLY what BM25 collection statistics need: the FST term/// dictionary (mmap'd) and the small `.meta` side-car.///@@ -3186,6 +3284,35 @@ mod tests {assert!(segment_recorded_analyzers(dir2.path(), "seg0").is_none());}+ /// #2427/#1122: the engine reserves read memory from this estimate before+ /// a full open, so it must not undercount what the open retains.+ #[test]+ fn estimate_open_bytes_covers_the_retained_postings_and_norms() {+ let dir = TempDir::new().unwrap();+ let mut writer = FtsIndexWriter::new(dir.path(), "seg-est", make_registry());+ for i in 0..2_000u32 {+ let text = format!("doc {i} alpha beta gamma term{} other{}", i % 97, i % 13);+ let doc: HashMap<String, FieldValues> = [("body".to_owned(), FieldValues::from(text))]+ .into_iter()+ .collect();+ writer.add_document(i, &doc);+ }+ writer.finish().unwrap();+ let reader = FtsIndexReader::open(dir.path(), "seg-est", &["body"]).unwrap();+ let estimate = FtsIndexReader::estimate_open_bytes(dir.path(), "seg-est", &["body"]);+ let retained = reader.retained_bytes();+ assert!(retained > 0);+ assert!(+ estimate >= retained,+ "estimate {estimate} must cover the {retained} bytes the open retains"+ );+ assert_eq!(+ FtsIndexReader::estimate_open_bytes(dir.path(), "seg-missing", &["body"]),+ "body".len() as u64,+ "a segment without side-cars costs (almost) nothing"+ );+ }+#[test]fn write_and_read_single_field() {let dir = TempDir::new().unwrap();