AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(storage): estimate retained segment cache capacity
buger7eb0801cc0dc
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rsindex d692ee240..52b252f59 100644--- a/engine/crates/xerj-engine/src/lib.rs+++ b/engine/crates/xerj-engine/src/lib.rs@@ -26,6 +26,7 @@ pub mod ingest_memory;pub mod memtable;pub mod painless;pub mod rbac;+pub mod segment_cache_estimates;pub mod slow_query;pub mod sql;pub mod turbo_ingest;diff --git a/engine/crates/xerj-engine/src/segment_cache_estimates.rs b/engine/crates/xerj-engine/src/segment_cache_estimates.rsnew file mode 100644index 000000000..9c02ff991--- /dev/null+++ b/engine/crates/xerj-engine/src/segment_cache_estimates.rs@@ -0,0 +1,238 @@+//! Deterministic retained-capacity estimates for immutable segment caches.+//!+//! These estimates are intentionally conservative and saturating. They are+//! neither allocator identities nor RSS measurements.++use serde_json::Value;++fn usize_u64(value: usize) -> u64 {+ u64::try_from(value).unwrap_or(u64::MAX)+}++fn capacity_bytes<T>(capacity: usize) -> u64 {+ usize_u64(capacity).saturating_mul(usize_u64(std::mem::size_of::<T>()))+}++/// Key, resident wrapper, Arc counters, and a conservative DashMap+/// slot/control allowance. `resident_inline_size` must be+/// `size_of::<CacheResident<T>>()` for the map's exact resident type.+pub fn common_entry_bytes(key: &String, resident_inline_size: usize) -> u64 {+ usize_u64(resident_inline_size)+ .saturating_add(usize_u64(2 * std::mem::size_of::<usize>()))+ .saturating_add(usize_u64(std::mem::size_of::<String>()))+ .saturating_add(usize_u64(key.capacity()))+ .saturating_add(usize_u64(6 * std::mem::size_of::<usize>()))+}++pub fn stored_slices_bytes(+ key: &String,+ resident_inline_size: usize,+ bytes_capacity: usize,+ offsets_capacity: usize,+) -> u64 {+ common_entry_bytes(key, resident_inline_size)+ .saturating_add(usize_u64(bytes_capacity))+ .saturating_add(capacity_bytes::<(u32, u32)>(offsets_capacity))+}++pub fn decoded_stored_bytes(+ key: &String,+ resident_inline_size: usize,+ bytes_capacity: usize,+) -> u64 {+ common_entry_bytes(key, resident_inline_size).saturating_add(usize_u64(bytes_capacity))+}++pub fn row_sequences_bytes(key: &String, resident_inline_size: usize, capacity: usize) -> u64 {+ common_entry_bytes(key, resident_inline_size).saturating_add(capacity_bytes::<u64>(capacity))+}++pub fn sort_shadow_bytes(+ key: &String,+ resident_inline_size: usize,+ capacity: Option<usize>,+) -> u64 {+ common_entry_bytes(key, resident_inline_size).saturating_add(+ capacity+ .map(|capacity| {+ usize_u64(std::mem::size_of::<Vec<(i64, u32)>>())+ .saturating_add(usize_u64(2 * std::mem::size_of::<usize>()))+ .saturating_add(capacity_bytes::<(i64, u32)>(capacity))+ })+ .unwrap_or_default(),+ )+}++pub fn id_positions_bytes(+ key: &String,+ resident_inline_size: usize,+ map: &std::collections::HashMap<String, u32>,+) -> u64 {+ let slots = usize_u64(map.capacity()).saturating_mul(+ usize_u64(std::mem::size_of::<(String, u32)>())+ .saturating_add(usize_u64(std::mem::size_of::<usize>())),+ );+ common_entry_bytes(key, resident_inline_size)+ .saturating_add(slots)+ .saturating_add(map.keys().fold(0_u64, |total, value| {+ total.saturating_add(usize_u64(value.capacity()))+ }))+}++pub fn doc_values_bytes(+ key: &String,+ resident_inline_size: usize,+ columns: &std::collections::BTreeMap<String, xerj_storage::doc_values::Column>,+) -> u64 {+ let node_allowance = usize_u64(std::mem::size_of::<(+ String,+ xerj_storage::doc_values::Column,+ [usize; 3],+ )>());+ common_entry_bytes(key, resident_inline_size).saturating_add(columns.iter().fold(+ 0_u64,+ |total, (field, column)| {+ total+ .saturating_add(usize_u64(field.capacity()))+ .saturating_add(node_allowance)+ .saturating_add(column.estimated_retained_bytes())+ },+ ))+}++/// Conservative retained estimate for values produced by serde_json+/// `from_slice` and then left insert-only. All stored-value cache callers meet+/// that precondition; arbitrary remove/reserve/shrink mutation would require a+/// different capacity proof.+pub fn stored_values_bytes(key: &String, resident_inline_size: usize, values: &Vec<Value>) -> u64 {+ common_entry_bytes(key, resident_inline_size)+ .saturating_add(capacity_bytes::<Value>(values.capacity()))+ .saturating_add(values.iter().fold(0_u64, |total, value| {+ total.saturating_add(value_owned_bytes(value))+ }))+}++fn value_owned_bytes(value: &Value) -> u64 {+ match value {+ Value::String(value) => usize_u64(value.capacity()),+ Value::Array(values) => capacity_bytes::<Value>(values.capacity()).saturating_add(+ values.iter().fold(0_u64, |total, value| {+ total.saturating_add(value_owned_bytes(value))+ }),+ ),+ Value::Object(values) => {+ // Pinned serde_json 1.0.149 preserve_order → IndexMap 2.14 →+ // hashbrown 0.16. For from_slice/insert-only maps, growth leaves+ // dense entry capacity <= max(3, 2*n). The raw index table is+ // power-of-two and bounded by max(4, 2*entry_capacity). Charge a+ // full usize index plus one control byte per bucket even though+ // IndexMap uses narrower adaptive indices, plus fixed+ // control/alignment slack.+ let len = values.len();+ let entry_capacity = if len == 0 {+ 0+ } else {+ len.saturating_mul(2).max(3)+ };+ let bucket_capacity = if entry_capacity == 0 {+ 0+ } else {+ entry_capacity.saturating_mul(2).max(4)+ };+ let dense = capacity_bytes::<(u64, String, Value)>(entry_capacity);+ let hash_index = usize_u64(bucket_capacity)+ .saturating_mul(usize_u64(std::mem::size_of::<usize>() + 1))+ .saturating_add(if bucket_capacity == 0 { 0 } else { 64 });+ dense+ .saturating_add(hash_index)+ .saturating_add(values.iter().fold(0_u64, |total, (key, value)| {+ total+ .saturating_add(usize_u64(key.capacity()))+ .saturating_add(value_owned_bytes(value))+ }))+ }+ Value::Null | Value::Bool(_) | Value::Number(_) => 0,+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn capacity_and_nested_json_are_counted() {+ let key = String::with_capacity(64);+ assert!(+ decoded_stored_bytes(&key, 32, 128) > decoded_stored_bytes(&String::from("x"), 32, 1)+ );++ let mut nested = Vec::with_capacity(8);+ nested.push(Value::String(String::from("owned")));+ let values = vec![Value::Array(nested)];+ let segment = String::from("seg");+ let estimate = stored_values_bytes(&segment, 64, &values);+ assert!(estimate > common_entry_bytes(&segment, 64));++ let shadow_key = String::from("seg\u{1}field");+ let none = sort_shadow_bytes(&shadow_key, 32, None);+ let some_empty = sort_shadow_bytes(&shadow_key, 32, Some(0));+ assert!(none > 0);+ assert!(+ some_empty+ >= none+ .saturating_add(std::mem::size_of::<Vec<(i64, u32)>>() as u64)+ .saturating_add((2 * std::mem::size_of::<usize>()) as u64)+ );++ let mut object = serde_json::Map::new();+ object.insert(+ String::with_capacity(48),+ Value::Object(serde_json::Map::with_capacity(16)),+ );+ let object_value = Value::Object(object);+ assert!(value_owned_bytes(&object_value) > 0);+ }++ #[test]+ fn insert_only_indexmap_bound_is_monotonic_at_growth_edges() {+ let mut previous = 0;+ for len in [1_usize, 3, 4, 7, 8, 15] {+ let mut object = serde_json::Map::new();+ for i in 0..len {+ object.insert(format!("key-{i}"), Value::Number(i.into()));+ }+ let estimate = value_owned_bytes(&Value::Object(object));+ let entry_capacity = len.saturating_mul(2).max(3);+ let bucket_capacity = entry_capacity.saturating_mul(2).max(4);+ let fixed = capacity_bytes::<(u64, String, Value)>(entry_capacity)+ .saturating_add(+ usize_u64(bucket_capacity)+ .saturating_mul(usize_u64(std::mem::size_of::<usize>() + 1)),+ )+ .saturating_add(64);+ assert!(estimate >= fixed);+ assert!(estimate >= previous);+ previous = estimate;+ }+ }++ #[test]+ fn formulas_saturate_instead_of_wrapping() {+ let key = String::from("seg");+ assert_eq!(+ decoded_stored_bytes(&key, usize::MAX, usize::MAX),+ u64::try_from(usize::MAX)+ .unwrap_or(u64::MAX)+ .saturating_add(u64::try_from(usize::MAX).unwrap_or(u64::MAX))+ .saturating_add(+ u64::try_from(+ 2 * std::mem::size_of::<usize>()+ + std::mem::size_of::<String>()+ + key.capacity()+ + 6 * std::mem::size_of::<usize>(),+ )+ .unwrap_or(u64::MAX),+ )+ );+ }+}diff --git a/engine/crates/xerj-storage/src/doc_values.rs b/engine/crates/xerj-storage/src/doc_values.rsindex d9eb9a2d6..35cc6d43c 100644--- a/engine/crates/xerj-storage/src/doc_values.rs+++ b/engine/crates/xerj-storage/src/doc_values.rs@@ -569,6 +569,52 @@ impl Column {Column::Keyword(k) => k.doc_count,}}++ /// Conservative deterministic estimate of heap allocations retained by+ /// this decoded column. This is capacity accounting, not allocator/RSS+ /// identity. The inline `Column` enum itself is owned by the caller's+ /// map and is intentionally not counted here.+ pub fn estimated_retained_bytes(&self) -> u64 {+ match self {+ Column::Numeric(column) => capacity_bytes::<i64>(column.data.capacity())+ .saturating_add(capacity_bytes::<(i64, u32)>(column.sorted.capacity()))+ .saturating_add(roaring_retained_bytes(&column.null_bitmap)),+ Column::Keyword(column) => {+ let term_storage = capacity_bytes::<String>(column.terms.capacity())+ .saturating_add(column.terms.iter().fold(0_u64, |total, term| {+ total.saturating_add(usize_to_u64(term.capacity()))+ }));+ term_storage+ .saturating_add(usize_to_u64(column.fst_bytes.capacity()))+ .saturating_add(capacity_bytes::<u32>(column.ords.capacity()))+ .saturating_add(capacity_bytes::<u32>(column.per_ord_count.capacity()))+ .saturating_add(roaring_retained_bytes(&column.null_bitmap))+ }+ }+ }+}++fn usize_to_u64(value: usize) -> u64 {+ u64::try_from(value).unwrap_or(u64::MAX)+}++fn capacity_bytes<T>(capacity: usize) -> u64 {+ usize_to_u64(capacity).saturating_mul(usize_to_u64(std::mem::size_of::<T>()))+}++fn roaring_retained_bytes(bitmap: &RoaringBitmap) -> u64 {+ let statistics = bitmap.statistics();+ statistics+ .n_bytes_array_containers+ .saturating_add(statistics.n_bytes_bitset_containers)+ .saturating_add(statistics.n_bytes_run_containers)+ // roaring 0.10 does not expose the container Vec capacity. One+ // conservative three-word allowance per live container covers the+ // key/store slot without pretending serialized size is heap size.+ .saturating_add(+ u64::from(statistics.n_containers)+ .saturating_mul(usize_to_u64(3 * std::mem::size_of::<usize>())),+ )}/// Encode a `field → Column` map into the binary `Columns` section payload.@@ -777,4 +823,43 @@ mod tests {panic!("expected numeric");}}++ #[test]+ fn retained_estimates_count_numeric_keyword_capacity_and_roaring() {+ let mut numeric = NumericColumn::from_iter(vec![Some(1), None, Some(3)]);+ numeric.data.reserve(128);+ numeric.sorted.reserve(64);+ let numeric = Column::Numeric(numeric);+ let numeric_estimate = numeric.estimated_retained_bytes();+ let Column::Numeric(numeric_column) = &numeric else {+ unreachable!()+ };+ assert!(+ numeric_estimate+ >= (numeric_column.data.capacity() * std::mem::size_of::<i64>()+ + numeric_column.sorted.capacity() * std::mem::size_of::<(i64, u32)>())+ as u64+ );++ let mut keyword = KeywordColumn::from_iter(vec![+ Some(String::from("alpha")),+ None,+ Some(String::from("beta")),+ ])+ .unwrap();+ keyword.terms.reserve(32);+ keyword.fst_bytes.reserve(256);+ keyword.ords.reserve(128);+ keyword.per_ord_count.reserve(64);+ let keyword = Column::Keyword(keyword);+ let keyword_estimate = keyword.estimated_retained_bytes();+ let Column::Keyword(keyword_column) = &keyword else {+ unreachable!()+ };+ let vector_floor = keyword_column.terms.capacity() * std::mem::size_of::<String>()+ + keyword_column.fst_bytes.capacity()+ + keyword_column.ords.capacity() * std::mem::size_of::<u32>()+ + keyword_column.per_ord_count.capacity() * std::mem::size_of::<u32>();+ assert!(keyword_estimate >= vector_floor as u64);+ }}