Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(storage): estimate retained segment cache capacity

buger7eb0801cc0dc
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index d692ee240..52b252f59 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -26,6 +26,7 @@ pub mod ingest_memory;
pub mod memtable;
pub mod painless;
pub mod rbac;
+pub mod segment_cache_estimates;
pub mod slow_query;
pub mod sql;
pub mod turbo_ingest;
diff --git a/engine/crates/xerj-engine/src/segment_cache_estimates.rs b/engine/crates/xerj-engine/src/segment_cache_estimates.rs
new file mode 100644
index 000000000..9c02ff991
--- /dev/null
+++ b/engine/crates/xerj-engine/src/segment_cache_estimates.rs
@@ -0,0 +1,238 @@
+//! Deterministic retained-capacity estimates for immutable segment caches.
+//!
+//! These estimates are intentionally conservative and saturating. They are
+//! neither allocator identities nor RSS measurements.
+
+use serde_json::Value;
+
+fn usize_u64(value: usize) -> u64 {
+ u64::try_from(value).unwrap_or(u64::MAX)
+}
+
+fn capacity_bytes<T>(capacity: usize) -> u64 {
+ usize_u64(capacity).saturating_mul(usize_u64(std::mem::size_of::<T>()))
+}
+
+/// Key, resident wrapper, Arc counters, and a conservative DashMap
+/// slot/control allowance. `resident_inline_size` must be
+/// `size_of::<CacheResident<T>>()` for the map's exact resident type.
+pub fn common_entry_bytes(key: &String, resident_inline_size: usize) -> u64 {
+ usize_u64(resident_inline_size)
+ .saturating_add(usize_u64(2 * std::mem::size_of::<usize>()))
+ .saturating_add(usize_u64(std::mem::size_of::<String>()))
+ .saturating_add(usize_u64(key.capacity()))
+ .saturating_add(usize_u64(6 * std::mem::size_of::<usize>()))
+}
+
+pub fn stored_slices_bytes(
+ key: &String,
+ resident_inline_size: usize,
+ bytes_capacity: usize,
+ offsets_capacity: usize,
+) -> u64 {
+ common_entry_bytes(key, resident_inline_size)
+ .saturating_add(usize_u64(bytes_capacity))
+ .saturating_add(capacity_bytes::<(u32, u32)>(offsets_capacity))
+}
+
+pub fn decoded_stored_bytes(
+ key: &String,
+ resident_inline_size: usize,
+ bytes_capacity: usize,
+) -> u64 {
+ common_entry_bytes(key, resident_inline_size).saturating_add(usize_u64(bytes_capacity))
+}
+
+pub fn row_sequences_bytes(key: &String, resident_inline_size: usize, capacity: usize) -> u64 {
+ common_entry_bytes(key, resident_inline_size).saturating_add(capacity_bytes::<u64>(capacity))
+}
+
+pub fn sort_shadow_bytes(
+ key: &String,
+ resident_inline_size: usize,
+ capacity: Option<usize>,
+) -> u64 {
+ common_entry_bytes(key, resident_inline_size).saturating_add(
+ capacity
+ .map(|capacity| {
+ usize_u64(std::mem::size_of::<Vec<(i64, u32)>>())
+ .saturating_add(usize_u64(2 * std::mem::size_of::<usize>()))
+ .saturating_add(capacity_bytes::<(i64, u32)>(capacity))
+ })
+ .unwrap_or_default(),
+ )
+}
+
+pub fn id_positions_bytes(
+ key: &String,
+ resident_inline_size: usize,
+ map: &std::collections::HashMap<String, u32>,
+) -> u64 {
+ let slots = usize_u64(map.capacity()).saturating_mul(
+ usize_u64(std::mem::size_of::<(String, u32)>())
+ .saturating_add(usize_u64(std::mem::size_of::<usize>())),
+ );
+ common_entry_bytes(key, resident_inline_size)
+ .saturating_add(slots)
+ .saturating_add(map.keys().fold(0_u64, |total, value| {
+ total.saturating_add(usize_u64(value.capacity()))
+ }))
+}
+
+pub fn doc_values_bytes(
+ key: &String,
+ resident_inline_size: usize,
+ columns: &std::collections::BTreeMap<String, xerj_storage::doc_values::Column>,
+) -> u64 {
+ let node_allowance = usize_u64(std::mem::size_of::<(
+ String,
+ xerj_storage::doc_values::Column,
+ [usize; 3],
+ )>());
+ common_entry_bytes(key, resident_inline_size).saturating_add(columns.iter().fold(
+ 0_u64,
+ |total, (field, column)| {
+ total
+ .saturating_add(usize_u64(field.capacity()))
+ .saturating_add(node_allowance)
+ .saturating_add(column.estimated_retained_bytes())
+ },
+ ))
+}
+
+/// Conservative retained estimate for values produced by serde_json
+/// `from_slice` and then left insert-only. All stored-value cache callers meet
+/// that precondition; arbitrary remove/reserve/shrink mutation would require a
+/// different capacity proof.
+pub fn stored_values_bytes(key: &String, resident_inline_size: usize, values: &Vec<Value>) -> u64 {
+ common_entry_bytes(key, resident_inline_size)
+ .saturating_add(capacity_bytes::<Value>(values.capacity()))
+ .saturating_add(values.iter().fold(0_u64, |total, value| {
+ total.saturating_add(value_owned_bytes(value))
+ }))
+}
+
+fn value_owned_bytes(value: &Value) -> u64 {
+ match value {
+ Value::String(value) => usize_u64(value.capacity()),
+ Value::Array(values) => capacity_bytes::<Value>(values.capacity()).saturating_add(
+ values.iter().fold(0_u64, |total, value| {
+ total.saturating_add(value_owned_bytes(value))
+ }),
+ ),
+ Value::Object(values) => {
+ // Pinned serde_json 1.0.149 preserve_order → IndexMap 2.14 →
+ // hashbrown 0.16. For from_slice/insert-only maps, growth leaves
+ // dense entry capacity <= max(3, 2*n). The raw index table is
+ // power-of-two and bounded by max(4, 2*entry_capacity). Charge a
+ // full usize index plus one control byte per bucket even though
+ // IndexMap uses narrower adaptive indices, plus fixed
+ // control/alignment slack.
+ let len = values.len();
+ let entry_capacity = if len == 0 {
+ 0
+ } else {
+ len.saturating_mul(2).max(3)
+ };
+ let bucket_capacity = if entry_capacity == 0 {
+ 0
+ } else {
+ entry_capacity.saturating_mul(2).max(4)
+ };
+ let dense = capacity_bytes::<(u64, String, Value)>(entry_capacity);
+ let hash_index = usize_u64(bucket_capacity)
+ .saturating_mul(usize_u64(std::mem::size_of::<usize>() + 1))
+ .saturating_add(if bucket_capacity == 0 { 0 } else { 64 });
+ dense
+ .saturating_add(hash_index)
+ .saturating_add(values.iter().fold(0_u64, |total, (key, value)| {
+ total
+ .saturating_add(usize_u64(key.capacity()))
+ .saturating_add(value_owned_bytes(value))
+ }))
+ }
+ Value::Null | Value::Bool(_) | Value::Number(_) => 0,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn capacity_and_nested_json_are_counted() {
+ let key = String::with_capacity(64);
+ assert!(
+ decoded_stored_bytes(&key, 32, 128) > decoded_stored_bytes(&String::from("x"), 32, 1)
+ );
+
+ let mut nested = Vec::with_capacity(8);
+ nested.push(Value::String(String::from("owned")));
+ let values = vec![Value::Array(nested)];
+ let segment = String::from("seg");
+ let estimate = stored_values_bytes(&segment, 64, &values);
+ assert!(estimate > common_entry_bytes(&segment, 64));
+
+ let shadow_key = String::from("seg\u{1}field");
+ let none = sort_shadow_bytes(&shadow_key, 32, None);
+ let some_empty = sort_shadow_bytes(&shadow_key, 32, Some(0));
+ assert!(none > 0);
+ assert!(
+ some_empty
+ >= none
+ .saturating_add(std::mem::size_of::<Vec<(i64, u32)>>() as u64)
+ .saturating_add((2 * std::mem::size_of::<usize>()) as u64)
+ );
+
+ let mut object = serde_json::Map::new();
+ object.insert(
+ String::with_capacity(48),
+ Value::Object(serde_json::Map::with_capacity(16)),
+ );
+ let object_value = Value::Object(object);
+ assert!(value_owned_bytes(&object_value) > 0);
+ }
+
+ #[test]
+ fn insert_only_indexmap_bound_is_monotonic_at_growth_edges() {
+ let mut previous = 0;
+ for len in [1_usize, 3, 4, 7, 8, 15] {
+ let mut object = serde_json::Map::new();
+ for i in 0..len {
+ object.insert(format!("key-{i}"), Value::Number(i.into()));
+ }
+ let estimate = value_owned_bytes(&Value::Object(object));
+ let entry_capacity = len.saturating_mul(2).max(3);
+ let bucket_capacity = entry_capacity.saturating_mul(2).max(4);
+ let fixed = capacity_bytes::<(u64, String, Value)>(entry_capacity)
+ .saturating_add(
+ usize_u64(bucket_capacity)
+ .saturating_mul(usize_u64(std::mem::size_of::<usize>() + 1)),
+ )
+ .saturating_add(64);
+ assert!(estimate >= fixed);
+ assert!(estimate >= previous);
+ previous = estimate;
+ }
+ }
+
+ #[test]
+ fn formulas_saturate_instead_of_wrapping() {
+ let key = String::from("seg");
+ assert_eq!(
+ decoded_stored_bytes(&key, usize::MAX, usize::MAX),
+ u64::try_from(usize::MAX)
+ .unwrap_or(u64::MAX)
+ .saturating_add(u64::try_from(usize::MAX).unwrap_or(u64::MAX))
+ .saturating_add(
+ u64::try_from(
+ 2 * std::mem::size_of::<usize>()
+ + std::mem::size_of::<String>()
+ + key.capacity()
+ + 6 * std::mem::size_of::<usize>(),
+ )
+ .unwrap_or(u64::MAX),
+ )
+ );
+ }
+}
diff --git a/engine/crates/xerj-storage/src/doc_values.rs b/engine/crates/xerj-storage/src/doc_values.rs
index d9eb9a2d6..35cc6d43c 100644
--- a/engine/crates/xerj-storage/src/doc_values.rs
+++ b/engine/crates/xerj-storage/src/doc_values.rs
@@ -569,6 +569,52 @@ impl Column {
Column::Keyword(k) => k.doc_count,
}
}
+
+ /// Conservative deterministic estimate of heap allocations retained by
+ /// this decoded column. This is capacity accounting, not allocator/RSS
+ /// identity. The inline `Column` enum itself is owned by the caller's
+ /// map and is intentionally not counted here.
+ pub fn estimated_retained_bytes(&self) -> u64 {
+ match self {
+ Column::Numeric(column) => capacity_bytes::<i64>(column.data.capacity())
+ .saturating_add(capacity_bytes::<(i64, u32)>(column.sorted.capacity()))
+ .saturating_add(roaring_retained_bytes(&column.null_bitmap)),
+ Column::Keyword(column) => {
+ let term_storage = capacity_bytes::<String>(column.terms.capacity())
+ .saturating_add(column.terms.iter().fold(0_u64, |total, term| {
+ total.saturating_add(usize_to_u64(term.capacity()))
+ }));
+ term_storage
+ .saturating_add(usize_to_u64(column.fst_bytes.capacity()))
+ .saturating_add(capacity_bytes::<u32>(column.ords.capacity()))
+ .saturating_add(capacity_bytes::<u32>(column.per_ord_count.capacity()))
+ .saturating_add(roaring_retained_bytes(&column.null_bitmap))
+ }
+ }
+ }
+}
+
+fn usize_to_u64(value: usize) -> u64 {
+ u64::try_from(value).unwrap_or(u64::MAX)
+}
+
+fn capacity_bytes<T>(capacity: usize) -> u64 {
+ usize_to_u64(capacity).saturating_mul(usize_to_u64(std::mem::size_of::<T>()))
+}
+
+fn roaring_retained_bytes(bitmap: &RoaringBitmap) -> u64 {
+ let statistics = bitmap.statistics();
+ statistics
+ .n_bytes_array_containers
+ .saturating_add(statistics.n_bytes_bitset_containers)
+ .saturating_add(statistics.n_bytes_run_containers)
+ // roaring 0.10 does not expose the container Vec capacity. One
+ // conservative three-word allowance per live container covers the
+ // key/store slot without pretending serialized size is heap size.
+ .saturating_add(
+ u64::from(statistics.n_containers)
+ .saturating_mul(usize_to_u64(3 * std::mem::size_of::<usize>())),
+ )
}
/// Encode a `field → Column` map into the binary `Columns` section payload.
@@ -777,4 +823,43 @@ mod tests {
panic!("expected numeric");
}
}
+
+ #[test]
+ fn retained_estimates_count_numeric_keyword_capacity_and_roaring() {
+ let mut numeric = NumericColumn::from_iter(vec![Some(1), None, Some(3)]);
+ numeric.data.reserve(128);
+ numeric.sorted.reserve(64);
+ let numeric = Column::Numeric(numeric);
+ let numeric_estimate = numeric.estimated_retained_bytes();
+ let Column::Numeric(numeric_column) = &numeric else {
+ unreachable!()
+ };
+ assert!(
+ numeric_estimate
+ >= (numeric_column.data.capacity() * std::mem::size_of::<i64>()
+ + numeric_column.sorted.capacity() * std::mem::size_of::<(i64, u32)>())
+ as u64
+ );
+
+ let mut keyword = KeywordColumn::from_iter(vec![
+ Some(String::from("alpha")),
+ None,
+ Some(String::from("beta")),
+ ])
+ .unwrap();
+ keyword.terms.reserve(32);
+ keyword.fst_bytes.reserve(256);
+ keyword.ords.reserve(128);
+ keyword.per_ord_count.reserve(64);
+ let keyword = Column::Keyword(keyword);
+ let keyword_estimate = keyword.estimated_retained_bytes();
+ let Column::Keyword(keyword_column) = &keyword else {
+ unreachable!()
+ };
+ let vector_floor = keyword_column.terms.capacity() * std::mem::size_of::<String>()
+ + keyword_column.fst_bytes.capacity()
+ + keyword_column.ords.capacity() * std::mem::size_of::<u32>()
+ + keyword_column.per_ord_count.capacity() * std::mem::size_of::<u32>();
+ assert!(keyword_estimate >= vector_floor as u64);
+ }
}