AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(docvalues): reserve read budget for uncached doc_values decodes (Refs #2427) (#2778)
xerj-orgf855d39db5ae
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 1c3f3e545..7324366b4 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -79,6 +79,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0longer take a node from below the watermark into the parent breaker band andturn ingest into HTTP 429s. The relief for a search node resting between themerge-pool ceiling and the watermark is kept (#2427).+- A search that decodes a segment's doc-values sidecar the segment cache has+ no room to keep now reserves the decoded size in the read transient budget+ before decoding, and holds it until the query drops the columns. Before,+ such decodes were neither cached nor reserved, so concurrent searches during+ heavy ingest could stack decoded sidecars (about 100 MB late in the+ cap-2048 run) that no memory budget counted (Refs #2427).- **A memory-pressure drain no longer spikes RSS past the cap while itflushes** (#2427). The breaker drain and the cache-release pathflushed each index with every non-empty memtable shard building itsdiff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 97dcbff03..85790598b 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -2273,6 +2273,83 @@ mod flush_publication_recovery_tests {drop(cached);}+ /// #2427: a doc-values decode the hydration cache refuses is rebuilt by+ /// every query, and it must hold a read-budget reservation for as long+ /// as a query holds it, the same as a transient stored-slices build. A+ /// decode the cache keeps reserves nothing.+ ///+ /// Fails before the fix: the refused decode was served uncached with no+ /// reservation, so the read budget saw 0 bytes.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn transient_doc_values_hold_a_read_reservation_until_dropped_2427() {+ let dir = tempfile::tempdir().unwrap();+ let engine = crate::Engine::new(topk_projection_config(dir.path())).unwrap();+ let mut schema = Schema::empty();+ schema+ .add_field(FieldConfig::new("status", FieldType::Keyword))+ .unwrap();+ schema+ .add_field(FieldConfig::new("bytes", FieldType::Long))+ .unwrap();+ engine.create_index("dv-read-budget", schema).unwrap();+ let idx = engine.get_index("dv-read-budget").unwrap();+ idx.abort_background_tasks();+ for i in 0..2048u64 {+ idx.index_document(+ Some(format!("doc-{i}")),+ json!({"status": format!("s{}", i % 37), "bytes": i * 7}),+ )+ .await+ .unwrap();+ }+ idx.flush().await.unwrap();+ let seg = idx.store.snapshot().segments[0].clone();+ let segments_dir = idx.data_dir.join("segments");+ assert!(+ segments_dir.join(format!("{}.dv", seg.id)).exists(),+ "precondition: the segment has a doc-values sidecar"+ );+ let read = ReadTransientBudget::new(1 << 30, std::time::Duration::from_secs(5));+ idx.release_memory();+ assert!(!idx.dv_cache.contains_key(&seg.id), "precondition: cold");++ // No room in the cache: transient, reserved until the value drops.+ let full = SegmentHydrationBudget::new(0);+ let cols = idx+ .dv_columns_for_with(&segments_dir, &seg.id, &full, &read)+ .expect("doc values decode");+ assert!(!idx.dv_cache.contains_key(&seg.id));+ let decoded: u64 = cols+ .value()+ .values()+ .map(|c| c.estimated_retained_bytes())+ .sum();+ let held = read.snapshot().current;+ assert!(+ decoded > 0 && held >= decoded,+ "the transient decode must reserve at least its decoded bytes ({held} held, {decoded} decoded)"+ );+ let second = Arc::clone(&cols);+ drop(cols);+ assert_eq!(read.snapshot().current, held, "a live reader keeps it");+ drop(second);+ assert_eq!(read.snapshot().current, 0, "released with the last reader");++ // Room in the cache: admitted, nothing reserved.+ let roomy = SegmentHydrationBudget::new(u64::MAX);+ let cached = idx+ .dv_columns_for_with(&segments_dir, &seg.id, &roomy, &read)+ .expect("doc values decode");+ assert!(idx.dv_cache.contains_key(&seg.id));+ assert_eq!(read.snapshot().current, 0);+ assert_eq!(+ read.snapshot().admitted,+ 1,+ "only the transient decode reserved"+ );+ drop(cached);+ }+fn topk_projection_config(dir: &std::path::Path) -> Config {let mut config = Config::default();config.server.data_dir = dir.to_string_lossy().into_owned();@@ -32686,11 +32763,46 @@ impl Index {&self,segments_dir: &std::path::Path,segment_id: &str,+ ) -> Option<Resident<DocValueMap>> {+ self.dv_columns_for_with(+ segments_dir,+ segment_id,+ &self.segment_hydration_budget,+ crate::read_budget::global(),+ )+ }++ /// [`Self::dv_columns_for`] against explicit budgets (#2427). A decode+ /// the hydration cache cannot keep is rebuilt by every query that needs+ /// it; it reserves its size in the read budget before decoding and+ /// holds the reservation until its last reader drops it, like the+ /// stored-slices and FTS-reader builds. Before this, a refused decode+ /// was neither cached nor reserved, so concurrent searches stacked+ /// whole decoded sidecars (~100 MB late in the cap-2048 ingest run)+ /// that no budget saw.+ fn dv_columns_for_with(+ &self,+ segments_dir: &std::path::Path,+ segment_id: &str,+ hydration: &Arc<SegmentHydrationBudget>,+ read_budget: &Arc<ReadTransientBudget>,) -> Option<Resident<DocValueMap>> {if let Some(entry) = self.dv_cache.get(segment_id) {return Some(Arc::clone(entry.value()));}- let cols = read_doc_values_sidecar(segments_dir, segment_id);+ let raw = std::fs::read(segments_dir.join(format!("{segment_id}.dv"))).ok()?;+ let doc_count = self+ .store+ .snapshot()+ .segments+ .iter()+ .find(|meta| meta.id == segment_id)+ .map_or(0, |meta| meta.doc_count);+ let estimate = xerj_storage::doc_values::estimate_decoded_bytes(&raw, doc_count)+ .saturating_add(raw.len() as u64);+ let permit = transient_read_permit(hydration, read_budget, estimate);+ let cols = xerj_storage::doc_values::decode_columns(&raw).unwrap_or_default();+ drop(raw);if cols.is_empty() {return None;}@@ -32700,13 +32812,15 @@ impl Index {std::mem::size_of::<CacheResident<DocValueMap>>(),&cols,);- Some(self.publish_current(+ Some(self.publish_current_permitted(&self.dv_cache,segment_id,key,SegmentCacheCategory::DocValues,bytes,cols,+ hydration,+ permit,))}diff --git a/engine/crates/xerj-storage/src/doc_values.rs b/engine/crates/xerj-storage/src/doc_values.rsindex f18c65045..937e4764c 100644--- a/engine/crates/xerj-storage/src/doc_values.rs+++ b/engine/crates/xerj-storage/src/doc_values.rs@@ -1119,6 +1119,68 @@ pub fn decode_columns(bytes: &[u8]) -> Result<BTreeMap<String, Column>> {Ok(out)}+/// Upper estimate of the heap [`decode_columns`] retains for a sidecar of+/// `doc_count` rows, read from the column headers alone (no payload is+/// decompressed). Lets a caller reserve memory before paying the decode+/// (#2427): a numeric column decodes to 24 B/row (value plus sorted+/// index) whatever its packed size, a keyword column to its ordinals and+/// per-term counts plus up to 3x its decompressed dictionary (each term+/// becomes an owned `String`). Header damage stops the walk; the columns+/// counted so far are returned.+pub fn estimate_decoded_bytes(bytes: &[u8], doc_count: u64) -> u64 {+ fn le_u32(bytes: &[u8], at: usize) -> Option<u32> {+ Some(u32::from_le_bytes(bytes.get(at..at + 4)?.try_into().ok()?))+ }+ fn le_u64(bytes: &[u8], at: usize) -> Option<u64> {+ Some(u64::from_le_bytes(bytes.get(at..at + 8)?.try_into().ok()?))+ }+ if le_u32(bytes, 0) != Some(MAGIC) {+ return 0;+ }+ let Some(num_columns) = le_u32(bytes, 4) else {+ return 0;+ };+ let mut at = 8usize;+ let mut total = 0u64;+ for _ in 0..num_columns {+ let Some(&kind_byte) = bytes.get(at) else {+ break;+ };+ let Some(name_len) = le_u32(bytes, at + 1) else {+ break;+ };+ let payload_at = at + 5 + name_len as usize;+ let Some(payload_len) = le_u64(bytes, payload_at) else {+ break;+ };+ let start = payload_at + 8;+ let Some(payload) = usize::try_from(payload_len)+ .ok()+ .and_then(|len| bytes.get(start..start.checked_add(len)?))+ else {+ break;+ };+ at = start + payload.len();+ let decompressed = if kind_byte & KIND_FLAG_ZSTD != 0 {+ zstd::zstd_safe::get_frame_content_size(payload)+ .ok()+ .flatten()+ .unwrap_or_else(|| payload_len.saturating_mul(8))+ } else {+ le_u32(payload, 0).map_or(0, u64::from)+ };+ total = total.saturating_add(match kind_byte & KIND_MASK {+ KIND_NUMERIC => doc_count.saturating_mul(24).saturating_add(decompressed),+ KIND_KEYWORD => doc_count+ .saturating_mul(8)+ .saturating_add(decompressed.saturating_mul(3)),+ KIND_PRESENCE => decompressed.saturating_mul(2),+ _ => 0,+ });+ }+ total+}+fn io_to_storage(e: std::io::Error) -> StorageError {StorageError::Other(anyhow::anyhow!("doc-values io: {e}"))}@@ -1127,6 +1189,51 @@ fn io_to_storage(e: std::io::Error) -> StorageError {mod tests {use super::*;+ /// #2427: the header-only estimate must cover what the decode retains,+ /// so a reservation taken from it before decoding is not an undercount.+ #[test]+ fn estimate_decoded_bytes_covers_the_decoded_columns_2427() {+ let n = 50_000u64;+ let mut columns = BTreeMap::new();+ columns.insert(+ "bytes".to_string(),+ Column::Numeric(NumericColumn::from_iter(+ (0..n).map(|i| Some((i * 7919) as i64)),+ )),+ );+ columns.insert(+ "status".to_string(),+ Column::Keyword(+ KeywordColumn::from_iter((0..n).map(|i| Some(format!("term-{}", i % 4093))))+ .unwrap(),+ ),+ );+ columns.insert(+ "ids".to_string(),+ Column::Keyword(+ KeywordColumn::from_iter((0..n).map(|i| Some(format!("doc-{i:012}")))).unwrap(),+ ),+ );+ let bytes = encode_columns(&columns);+ let decoded: u64 = decode_columns(&bytes)+ .unwrap()+ .values()+ .map(Column::estimated_retained_bytes)+ .sum();+ let estimate = estimate_decoded_bytes(&bytes, n);+ assert!(+ estimate >= decoded,+ "estimate {estimate} B < decoded {decoded} B"+ );+ assert!(+ estimate <= decoded.saturating_mul(4),+ "estimate {estimate} B is far above decoded {decoded} B"+ );+ assert_eq!(estimate_decoded_bytes(b"not a sidecar", n), 0);+ // A torn sidecar counts only the columns whose payload is whole.+ assert!(estimate_decoded_bytes(&bytes[..bytes.len() - 1], n) < estimate);+ }+#[test]fn numeric_roundtrip() {let col = NumericColumn::from_iter(vec![Some(1), Some(2), None, Some(4), Some(5)]);