Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): bound all segment hydration cache owners

buger6b5c8d88c02a
diff --git a/demo/usecases/autoindex/scale/bounded/analyze.py b/demo/usecases/autoindex/scale/bounded/analyze.py
index b8ab2e4cd..c9267e122 100644
--- a/demo/usecases/autoindex/scale/bounded/analyze.py
+++ b/demo/usecases/autoindex/scale/bounded/analyze.py
@@ -51,13 +51,13 @@ MEASUREMENTS = {
"merge_json_buffer": "unavailable",
"merge_parsed": "unavailable",
"merge_encoded": "unavailable",
- "cache_stored_slices": "unavailable",
- "cache_doc_values": "unavailable",
- "cache_stored_values": "unavailable",
- "cache_sort_shadow": "unavailable",
- "cache_id_positions": "unavailable",
- "cache_row_sequences": "unavailable",
- "cache_decoded_stored": "unavailable",
+ "cache_stored_slices": "estimated",
+ "cache_doc_values": "estimated",
+ "cache_stored_values": "estimated",
+ "cache_sort_shadow": "estimated",
+ "cache_id_positions": "estimated",
+ "cache_row_sequences": "estimated",
+ "cache_decoded_stored": "estimated",
}
@@ -86,6 +86,7 @@ def read_ndjson(path: Path) -> list[dict]:
def validate_trace(path: Path) -> dict:
rows = read_ndjson(path)
problems = []
+ hydration_snapshots = []
if any(row.get("schema") != TRACE_SCHEMA for row in rows):
problems.append("schema mismatch")
seq = [row.get("seq") for row in rows]
@@ -103,6 +104,31 @@ def validate_trace(path: Path) -> dict:
if max(int(row.get("dropped_events", 0)) for row in rows) != 0:
problems.append("dropped_events is nonzero")
for position, row in enumerate(rows, 1):
+ hydration = row.get("segment_hydration_cache")
+ required_hydration_fields = (
+ "limit",
+ "current",
+ "peak",
+ "refused",
+ "accounting_errors",
+ )
+ if not isinstance(hydration, dict) or any(
+ type(hydration.get(field)) is not int or hydration[field] < 0
+ for field in required_hydration_fields
+ ):
+ problems.append(
+ f"record {position}: missing or invalid segment hydration cache snapshot"
+ )
+ else:
+ hydration_snapshots.append(hydration)
+ if hydration["current"] > hydration["peak"] or hydration["peak"] > hydration["limit"]:
+ problems.append(
+ f"record {position}: invalid segment hydration cache current/peak/limit"
+ )
+ if hydration["accounting_errors"] != 0:
+ problems.append(
+ f"record {position}: segment hydration cache accounting_errors is nonzero"
+ )
gauges = row.get("logical")
if not isinstance(gauges, list) or [gauge.get("category") for gauge in gauges] != CATEGORIES:
problems.append(f"record {position}: logical category vocabulary/order mismatch")
@@ -178,6 +204,19 @@ def validate_trace(path: Path) -> dict:
}
if not any("logical category vocabulary/order mismatch" in problem for problem in problems)
else None,
+ "segment_hydration_cache": {
+ "limit_bytes": hydration_snapshots[-1]["limit"],
+ "max_current_bytes": max(snapshot["current"] for snapshot in hydration_snapshots),
+ "peak_bytes": max(snapshot["peak"] for snapshot in hydration_snapshots),
+ "admission_refusals": max(
+ snapshot["refused"] for snapshot in hydration_snapshots
+ ),
+ "accounting_errors": max(
+ snapshot["accounting_errors"] for snapshot in hydration_snapshots
+ ),
+ }
+ if hydration_snapshots
+ else None,
"problems": problems,
}
diff --git a/demo/usecases/autoindex/scale/bounded/tests/test_suite.py b/demo/usecases/autoindex/scale/bounded/tests/test_suite.py
index a1018b450..6f4b15326 100644
--- a/demo/usecases/autoindex/scale/bounded/tests/test_suite.py
+++ b/demo/usecases/autoindex/scale/bounded/tests/test_suite.py
@@ -42,14 +42,42 @@ class BoundedSuiteTests(unittest.TestCase):
self.assertEqual(parsed["data_dir"], value)
def test_trace_validator_rejects_gap_and_accounting_error(self):
+ # Literal engine-emitted vocabulary: keep this fixture independent of
+ # the analyzer's expected mapping so a stale expectation cannot make
+ # the test pass tautologically.
+ engine_measurements = [
+ "serialized_size",
+ "exact_capacity",
+ "serialized_size",
+ "estimated",
+ "exact_capacity",
+ "estimated",
+ "exact_capacity",
+ "estimated",
+ "estimated",
+ "unavailable",
+ "unavailable",
+ "unavailable",
+ "unavailable",
+ "unavailable",
+ "estimated",
+ "estimated",
+ "estimated",
+ "estimated",
+ "estimated",
+ "estimated",
+ "estimated",
+ ]
gauges = [
{
"category": category,
"current": 0,
"peak": 1,
- "measurement": analyze.MEASUREMENTS[category],
+ "measurement": measurement,
}
- for category in analyze.CATEGORIES
+ for category, measurement in zip(
+ analyze.CATEGORIES, engine_measurements, strict=True
+ )
]
event = {
"schema": analyze.TRACE_SCHEMA,
@@ -57,6 +85,15 @@ class BoundedSuiteTests(unittest.TestCase):
"event": "trace_start",
"accounting_errors": 0,
"dropped_events": 0,
+ "segment_hydration_cache": {
+ "limit": 1024,
+ "current": 0,
+ "peak": 128,
+ "refused": 2,
+ "accounting_errors": 0,
+ "category_current": [0] * 7,
+ "category_peak": [128, 0, 0, 0, 0, 0, 0],
+ },
"logical": gauges,
"logical_total_current": 0,
"allocator": {"allocated": 1, "active": 2, "resident": 3, "epoch_ok": True},
@@ -79,6 +116,16 @@ class BoundedSuiteTests(unittest.TestCase):
result = analyze.validate_trace(path)
self.assertIn("sequence is not contiguous from 1", result["problems"])
self.assertIn("accounting_errors is nonzero", result["problems"])
+ self.assertEqual(
+ result["segment_hydration_cache"],
+ {
+ "limit_bytes": 1024,
+ "max_current_bytes": 0,
+ "peak_bytes": 128,
+ "admission_refusals": 2,
+ "accounting_errors": 0,
+ },
+ )
def test_trace_validator_rejects_category_and_total_corruption(self):
gauges = [
@@ -94,6 +141,15 @@ class BoundedSuiteTests(unittest.TestCase):
"schema": analyze.TRACE_SCHEMA,
"accounting_errors": 0,
"dropped_events": 0,
+ "segment_hydration_cache": {
+ "limit": 1024,
+ "current": 0,
+ "peak": 0,
+ "refused": 0,
+ "accounting_errors": 0,
+ "category_current": [0] * 7,
+ "category_peak": [0] * 7,
+ },
"logical": gauges,
"logical_total_current": 9,
"allocator": {
@@ -128,6 +184,55 @@ class BoundedSuiteTests(unittest.TestCase):
any("category vocabulary/order" in problem for problem in result["problems"])
)
+ def test_trace_validator_requires_segment_hydration_cache_evidence(self):
+ gauges = [
+ {
+ "category": category,
+ "current": 0,
+ "peak": 0,
+ "measurement": analyze.MEASUREMENTS[category],
+ }
+ for category in analyze.CATEGORIES
+ ]
+ base = {
+ "schema": analyze.TRACE_SCHEMA,
+ "accounting_errors": 0,
+ "dropped_events": 0,
+ "logical": gauges,
+ "logical_total_current": 0,
+ "allocator": {
+ "allocated": None,
+ "active": None,
+ "resident": None,
+ "epoch_ok": False,
+ },
+ "process": {
+ "rss": None,
+ "rss_ok": False,
+ "cpu_time_ns": None,
+ "cpu_time_ok": False,
+ },
+ "flags": {
+ "sampled": False,
+ "logical_exact": False,
+ "allocator_supported": False,
+ },
+ }
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "trace.ndjson"
+ start = {**base, "seq": 1, "event": "trace_start"}
+ stop = {**base, "seq": 2, "event": "trace_stop"}
+ path.write_text(json.dumps(start) + "\n" + json.dumps(stop) + "\n")
+ result = analyze.validate_trace(path)
+ self.assertEqual(result["segment_hydration_cache"], None)
+ self.assertEqual(
+ sum(
+ "missing or invalid segment hydration cache snapshot" in problem
+ for problem in result["problems"]
+ ),
+ 2,
+ )
+
def test_budget_enforces_absolute_and_relational_ceilings(self):
result = {
"cells": [
diff --git a/docs/recipes/production-deployment.md b/docs/recipes/production-deployment.md
index 614c0c1d3..ce368a7cb 100644
--- a/docs/recipes/production-deployment.md
+++ b/docs/recipes/production-deployment.md
@@ -209,6 +209,34 @@ transport encryption; your platform provides the rest.
| Restrictive CORS by default (no cross-origin reads unless allow-listed) | ✅ (`[cors]`) |
| Secrets written `0600` (admin key, auto-generated TLS key) | ✅ |
+### Bound retained segment hydration
+
+Every index shares one process-wide admission authority for the seven
+immutable segment hydration caches (stored bytes/slices, decoded doc values,
+parsed stored JSON, sort shadows, id positions, row sequences). By default it
+retains at most 20% of the effective cgroup memory limit:
+
+```toml
+[limits]
+# 0 = automatic: effective cgroup/system memory / 5, with no minimum floor.
+max_segment_hydration_cache_mb = 0
+```
+
+An explicit value is clamped to 50% of the effective limit. For a diagnostic
+run, `XERJ_SEGMENT_HYDRATION_CACHE_MB=off|auto|<positive MiB>` overrides the
+file. Admission refusal does not change query results: the current query uses
+the decoded value without retaining it, while publish-time warming drops an
+unadmitted value.
+
+`GET /_nodes/stats` reports
+`indices.segment_hydration_cache.{limit_in_bytes,current_in_bytes,peak_in_bytes,admission_refusals}` and
+per-category gauges/table capacities. These are conservative retained-payload
+and key estimates, **not an RSS limit**. Query-owned materialization, decode
+scratch, mmaps, allocator fragmentation, FTS state, and other caches remain
+outside this ceiling. A single highly compressed segment can therefore create
+a transient decode peak before post-build admission; bounded/streaming decode
+is a separate follow-up.
+
**Delegated to your environment (not engine-level in this build):**
| Control | Do it with |
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 384502028..8bb37ca85 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -17229,6 +17229,29 @@ pub async fn nodes_stats(State(state): State<AppState>) -> impl IntoResponse {
let free_pct = 100u64.saturating_sub(used_pct);
let heap_used_pct = (rss_bytes * 100).checked_div(mem_total).unwrap_or(0);
let cpu_pct = sample_cpu_percent().await;
+ let segment_hydration = xerj_engine::governor::global().map(|governor| {
+ let snapshot = governor.snapshot();
+ let gauge = snapshot.segment_hydration;
+ let capacities = state.engine.segment_hydration_cache_capacities();
+ json!({
+ "accounting": "estimated_retained_payload_and_keys_not_rss",
+ "source": format!("{:?}", snapshot.segment_hydration_source).to_ascii_lowercase(),
+ "limit_in_bytes": gauge.limit,
+ "current_in_bytes": gauge.current,
+ "peak_in_bytes": gauge.peak,
+ "admission_refusals": gauge.refused,
+ "accounting_errors": gauge.accounting_errors,
+ "categories": {
+ "stored_slices": { "current_in_bytes": gauge.category_current[0], "peak_in_bytes": gauge.category_peak[0], "map_capacity": capacities[0] },
+ "doc_values": { "current_in_bytes": gauge.category_current[1], "peak_in_bytes": gauge.category_peak[1], "map_capacity": capacities[1] },
+ "stored_values": { "current_in_bytes": gauge.category_current[2], "peak_in_bytes": gauge.category_peak[2], "map_capacity": capacities[2] },
+ "sort_shadow": { "current_in_bytes": gauge.category_current[3], "peak_in_bytes": gauge.category_peak[3], "map_capacity": capacities[3] },
+ "id_positions": { "current_in_bytes": gauge.category_current[4], "peak_in_bytes": gauge.category_peak[4], "map_capacity": capacities[4] },
+ "row_sequences": { "current_in_bytes": gauge.category_current[5], "peak_in_bytes": gauge.category_peak[5], "map_capacity": capacities[5] },
+ "decoded_stored": { "current_in_bytes": gauge.category_current[6], "peak_in_bytes": gauge.category_peak[6], "map_capacity": capacities[6] },
+ }
+ })
+ });
let node_id = state.engine.node_id.as_str();
let now_ms = std::time::SystemTime::now()
@@ -17299,6 +17322,7 @@ pub async fn nodes_stats(State(state): State<AppState>) -> impl IntoResponse {
// hold (vec/cenivf/clivf for bbq_disk; veb for
// bbq_hnsw/flat).
"dense_vector": dense_vector_off_heap_stats(&state),
+ "segment_hydration_cache": segment_hydration,
},
"transport": {
"server_open": 1,
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 8ceabebc2..5146577aa 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -338,6 +338,17 @@ pub struct Engine {
}
impl Engine {
+ pub fn segment_hydration_cache_capacities(&self) -> [usize; 7] {
+ let mut total = [0_usize; 7];
+ for index in self.indices.iter() {
+ let capacities = index.value().segment_hydration_cache_capacities();
+ for (sum, capacity) in total.iter_mut().zip(capacities) {
+ *sum = sum.saturating_add(capacity);
+ }
+ }
+ total
+ }
+
/// Create a new engine, opening any existing indices from disk.
pub fn new(config: Config) -> Result<Self> {
let data_dir = PathBuf::from(&config.server.data_dir);
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index 88ea58da3..a53e17775 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -170,7 +170,7 @@ fn fast_aggs_disabled() -> bool {
#[derive(Clone)]
struct SegEntry {
id: String,
- cols: std::sync::Arc<std::collections::BTreeMap<String, Column>>,
+ cols: super::Resident<super::DocValueMap>,
docs: u32,
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 9d72c3f99..d4189f907 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -34,6 +34,28 @@ use xerj_vector::hnsw::{HnswIndex, HnswParams};
use xerj_vector::Sq8Params;
use crate::aggs::run_aggs_with_all;
+use crate::segment_cache_budget::{CacheResident, SegmentCacheCategory, SegmentHydrationBudget};
+use crate::segment_cache_estimates as cache_estimates;
+
+pub(crate) type Resident<T> = Arc<CacheResident<T>>;
+pub(crate) type DocValueMap = std::collections::BTreeMap<String, xerj_storage::doc_values::Column>;
+
+fn index_segment_hydration_budget(_config: &Config) -> Arc<SegmentHydrationBudget> {
+ if let Some(governor) = crate::governor::global() {
+ return governor.segment_hydration_budget();
+ }
+ #[cfg(test)]
+ {
+ // Direct unit-test Index construction bypasses Engine::new. Keep one
+ // explicit process-wide test authority instead of silently creating a
+ // fresh per-index budget.
+ static TEST_BUDGET: std::sync::OnceLock<Arc<SegmentHydrationBudget>> =
+ std::sync::OnceLock::new();
+ Arc::clone(TEST_BUDGET.get_or_init(|| SegmentHydrationBudget::new(u64::MAX)))
+ }
+ #[cfg(not(test))]
+ panic!("Index construction requires the process ResourceGovernor to be initialised");
+}
/// Clears an index's `merge_in_progress` flag on every exit path of the
/// merge holder (background pass or forcemerge), including panics.
@@ -300,7 +322,7 @@ mod semantic_deadline_regression_tests {
.apply_merge(std::slice::from_ref(&old_meta.id), replacement)
.unwrap();
{
- let _publication = idx.stored_value_cache_lifecycle.write();
+ let _publication = idx.segment_cache_lifecycle.write();
idx.stored_value_cache.remove(&old_meta.id);
}
idx.test_stored_value_load_pause_before_publish
@@ -316,6 +338,480 @@ mod semantic_deadline_regression_tests {
);
}
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn all_hydration_publishers_refuse_retired_segment_resurrection() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("all-cache-retire", Schema::empty())
+ .unwrap();
+ let idx = engine.get_index("all-cache-retire").unwrap();
+ idx.index_document(Some("v1".into()), serde_json::json!({"value": 1}))
+ .await
+ .unwrap();
+ idx.flush().await.unwrap();
+ let old_meta = idx.store.snapshot().segments[0].clone();
+ {
+ let _publication = idx.segment_cache_lifecycle.write();
+ idx.remove_segment_hydration_entries(&old_meta.id);
+ }
+
+ let all = (1_u64 << 7) - 1;
+ idx.test_segment_cache_publish_ready_mask
+ .store(0, Ordering::Release);
+ idx.test_segment_cache_publish_pause_mask
+ .store(all, Ordering::Release);
+
+ let mut publishers = Vec::new();
+ for category in SegmentCacheCategory::ALL {
+ let idx = Arc::clone(&idx);
+ let old_id = old_meta.id.clone();
+ publishers.push(std::thread::spawn(move || match category {
+ SegmentCacheCategory::StoredSlices => {
+ let key = old_id.clone();
+ let _ = idx.publish_current(
+ &idx.stored_slices_cache,
+ &old_id,
+ key,
+ category,
+ 1,
+ StoredSlices {
+ bytes: vec![b'{', b'}'],
+ offsets: vec![(0, 2)],
+ },
+ );
+ }
+ SegmentCacheCategory::DocValues => {
+ let key = old_id.clone();
+ let _ = idx.publish_current(
+ &idx.dv_cache,
+ &old_id,
+ key,
+ category,
+ 1,
+ DocValueMap::new(),
+ );
+ }
+ SegmentCacheCategory::StoredValues => {
+ let key = old_id.clone();
+ let _ = idx.publish_current(
+ &idx.stored_value_cache,
+ &old_id,
+ key,
+ category,
+ 1,
+ vec![serde_json::json!({"_id": "v1"})],
+ );
+ }
+ SegmentCacheCategory::SortShadow => {
+ let key = format!("{old_id}\u{1}field");
+ let _ = idx.publish_current(
+ &idx.sort_shadow_cache,
+ &old_id,
+ key,
+ category,
+ 1,
+ Some(Arc::new(vec![(1, 0)])),
+ );
+ }
+ SegmentCacheCategory::IdPositions => {
+ let key = old_id.clone();
+ let _ = idx.publish_current(
+ &idx.id_pos_cache,
+ &old_id,
+ key,
+ category,
+ 1,
+ std::collections::HashMap::from([(String::from("v1"), 0)]),
+ );
+ }
+ SegmentCacheCategory::RowSequences => {
+ let key = old_id.clone();
+ let _ =
+ idx.publish_current(&idx.row_seq_cache, &old_id, key, category, 1, vec![1]);
+ }
+ SegmentCacheCategory::DecodedStored => {
+ let key = old_id.clone();
+ let _ = idx.publish_current(
+ &idx.decoded_stored_cache,
+ &old_id,
+ key,
+ category,
+ 1,
+ vec![b'{', b'}'],
+ );
+ }
+ }));
+ }
+
+ tokio::time::timeout(std::time::Duration::from_secs(2), async {
+ while idx
+ .test_segment_cache_publish_ready_mask
+ .load(Ordering::Acquire)
+ != all
+ {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("all publishers did not reach the deterministic gate");
+
+ let mut replacement = old_meta.clone();
+ replacement.id = Uuid::new_v4().to_string();
+ replacement.seg_path = format!("segments/{}.seg", replacement.id);
+ replacement.sidx_path = format!("segments/{}.sidx", replacement.id);
+ idx.store
+ .apply_merge(std::slice::from_ref(&old_meta.id), replacement)
+ .unwrap();
+ {
+ let _publication = idx.segment_cache_lifecycle.write();
+ idx.remove_segment_hydration_entries(&old_meta.id);
+ }
+ idx.test_segment_cache_publish_pause_mask
+ .store(0, Ordering::Release);
+ for publisher in publishers {
+ publisher.join().unwrap();
+ }
+
+ assert!(!idx.stored_slices_cache.contains_key(&old_meta.id));
+ assert!(!idx.dv_cache.contains_key(&old_meta.id));
+ assert!(!idx.stored_value_cache.contains_key(&old_meta.id));
+ assert!(!idx.id_pos_cache.contains_key(&old_meta.id));
+ assert!(!idx.row_seq_cache.contains_key(&old_meta.id));
+ assert!(!idx.decoded_stored_cache.contains_key(&old_meta.id));
+ let prefix = format!("{}\u{1}", old_meta.id);
+ assert!(!idx
+ .sort_shadow_cache
+ .iter()
+ .any(|entry| entry.key().starts_with(&prefix)));
+ }
+
+ #[tokio::test]
+ async fn failed_output_cleanup_removes_every_hydration_owner() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("failed-output-cache", Schema::empty())
+ .unwrap();
+ let idx = engine.get_index("failed-output-cache").unwrap();
+ idx.index_document(Some("v1".into()), serde_json::json!({"value": 1}))
+ .await
+ .unwrap();
+ idx.flush().await.unwrap();
+ let segment_id = idx.store.snapshot().segments[0].id.clone();
+ {
+ let _publication = idx.segment_cache_lifecycle.write();
+ idx.remove_segment_hydration_entries(&segment_id);
+ }
+
+ let _ = idx.publish_current(
+ &idx.stored_slices_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::StoredSlices,
+ 1,
+ StoredSlices {
+ bytes: vec![b'{', b'}'],
+ offsets: vec![(0, 2)],
+ },
+ );
+ let _ = idx.publish_current(
+ &idx.dv_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::DocValues,
+ 1,
+ DocValueMap::new(),
+ );
+ let _ = idx.publish_current(
+ &idx.stored_value_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::StoredValues,
+ 1,
+ vec![serde_json::json!({"_id": "v1"})],
+ );
+ let shadow_key = format!("{segment_id}\u{1}field");
+ let _ = idx.publish_current(
+ &idx.sort_shadow_cache,
+ &segment_id,
+ shadow_key,
+ SegmentCacheCategory::SortShadow,
+ 1,
+ Some(Arc::new(vec![(1, 0)])),
+ );
+ let _ = idx.publish_current(
+ &idx.id_pos_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::IdPositions,
+ 1,
+ std::collections::HashMap::from([(String::from("v1"), 0)]),
+ );
+ let _ = idx.publish_current(
+ &idx.row_seq_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::RowSequences,
+ 1,
+ vec![1],
+ );
+ let _ = idx.publish_current(
+ &idx.decoded_stored_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::DecodedStored,
+ 1,
+ vec![b'{', b'}'],
+ );
+
+ idx.publish_warm_caches().remove_failed_output(&segment_id);
+ assert!(!idx.stored_slices_cache.contains_key(&segment_id));
+ assert!(!idx.dv_cache.contains_key(&segment_id));
+ assert!(!idx.stored_value_cache.contains_key(&segment_id));
+ assert!(!idx.id_pos_cache.contains_key(&segment_id));
+ assert!(!idx.row_seq_cache.contains_key(&segment_id));
+ assert!(!idx.decoded_stored_cache.contains_key(&segment_id));
+ let prefix = format!("{segment_id}\u{1}");
+ assert!(!idx
+ .sort_shadow_cache
+ .iter()
+ .any(|entry| entry.key().starts_with(&prefix)));
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn prepublication_lease_cleans_after_cancelled_owner_and_last_worker() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("lease-cancel", Schema::empty())
+ .unwrap();
+ let idx = engine.get_index("lease-cancel").unwrap();
+ let mut caches = idx.publish_warm_caches();
+ let budget = SegmentHydrationBudget::new(1024);
+ caches.budget = Arc::clone(&budget);
+ let segment_id = "detached-warm".to_string();
+ let lease = PrePublicationLease::known(caches.clone(), segment_id.clone());
+ let observed_cache = Arc::clone(&caches.decoded_stored);
+ let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(0);
+ let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0);
+ let (done_tx, done_rx) = std::sync::mpsc::sync_channel(0);
+ let owner = tokio::spawn(async move {
+ let worker_lease = lease.clone();
+ tokio::task::spawn_blocking(move || {
+ ready_tx.send(()).unwrap();
+ release_rx.recv().unwrap();
+ let capability = PrePublication(());
+ let _ = publish_prepublication(
+ &capability,
+ &caches,
+ &caches.decoded_stored,
+ segment_id,
+ SegmentCacheCategory::DecodedStored,
+ 64,
+ vec![1_u8; 64],
+ );
+ drop(worker_lease);
+ done_tx.send(()).unwrap();
+ })
+ .await
+ .unwrap();
+ lease.commit();
+ });
+ ready_rx
+ .recv_timeout(std::time::Duration::from_secs(2))
+ .unwrap();
+ owner.abort();
+ assert!(owner.await.unwrap_err().is_cancelled());
+ release_tx.send(()).unwrap();
+ done_rx
+ .recv_timeout(std::time::Duration::from_secs(2))
+ .unwrap();
+
+ assert!(!observed_cache.contains_key("detached-warm"));
+ assert_eq!(budget.snapshot().current, 0);
+ }
+
+ #[tokio::test]
+ async fn prepublication_lease_cleans_after_worker_panic() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine.create_index("lease-panic", Schema::empty()).unwrap();
+ let idx = engine.get_index("lease-panic").unwrap();
+ let mut caches = idx.publish_warm_caches();
+ let budget = SegmentHydrationBudget::new(1024);
+ caches.budget = Arc::clone(&budget);
+ let segment_id = "panicked-warm".to_string();
+ let lease = PrePublicationLease::known(caches.clone(), segment_id.clone());
+ let worker = lease.clone();
+ let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ let capability = PrePublication(());
+ let _ = publish_prepublication(
+ &capability,
+ &caches,
+ &caches.row_sequences,
+ segment_id.clone(),
+ SegmentCacheCategory::RowSequences,
+ 8,
+ vec![1],
+ );
+ drop(worker);
+ panic!("injected warm panic");
+ }));
+ assert!(result.is_err());
+ drop(lease);
+ assert!(!caches.row_sequences.contains_key(&segment_id));
+ assert_eq!(budget.snapshot().current, 0);
+ }
+
+ #[tokio::test]
+ async fn committed_prepublication_lease_retains_published_owner() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("lease-commit", Schema::empty())
+ .unwrap();
+ let idx = engine.get_index("lease-commit").unwrap();
+ let mut caches = idx.publish_warm_caches();
+ let budget = SegmentHydrationBudget::new(1024);
+ caches.budget = Arc::clone(&budget);
+ let segment_id = "published-warm".to_string();
+ let lease = PrePublicationLease::known(caches.clone(), segment_id.clone());
+ let capability = PrePublication(());
+ let _ = publish_prepublication(
+ &capability,
+ &caches,
+ &caches.id_positions,
+ segment_id.clone(),
+ SegmentCacheCategory::IdPositions,
+ 16,
+ std::collections::HashMap::new(),
+ );
+ lease.commit();
+ drop(lease);
+ assert!(caches.id_positions.contains_key(&segment_id));
+ caches.remove_failed_output(&segment_id);
+ // Cleanup is idempotent and cannot underflow the retained budget.
+ caches.remove_failed_output(&segment_id);
+ assert_eq!(budget.snapshot().current, 0);
+ assert_eq!(budget.snapshot().accounting_errors, 0);
+ }
+
+ #[tokio::test]
+ async fn every_hydration_refusal_returns_the_local_value_without_publication() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("cache-refusal", Schema::empty())
+ .unwrap();
+ let idx = engine.get_index("cache-refusal").unwrap();
+ idx.index_document(Some("v1".into()), serde_json::json!({"value": 1}))
+ .await
+ .unwrap();
+ idx.flush().await.unwrap();
+ let segment_id = idx.store.snapshot().segments[0].id.clone();
+ {
+ let _publication = idx.segment_cache_lifecycle.write();
+ idx.remove_segment_hydration_entries(&segment_id);
+ }
+ let off = SegmentHydrationBudget::new(0);
+
+ let slices = idx.publish_current_with_budget(
+ &idx.stored_slices_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::StoredSlices,
+ 1,
+ StoredSlices {
+ bytes: vec![b'{', b'}'],
+ offsets: vec![(0, 2)],
+ },
+ &off,
+ );
+ assert_eq!(slices.bytes, vec![b'{', b'}']);
+
+ let dv = idx.publish_current_with_budget(
+ &idx.dv_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::DocValues,
+ 1,
+ DocValueMap::new(),
+ &off,
+ );
+ assert!(dv.is_empty());
+
+ let values = idx.publish_current_with_budget(
+ &idx.stored_value_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::StoredValues,
+ 1,
+ vec![serde_json::json!({"_id": "v1"})],
+ &off,
+ );
+ assert_eq!(values[0]["_id"], "v1");
+
+ let shadow_key = format!("{segment_id}\u{1}field");
+ let shadow = idx.publish_current_with_budget(
+ &idx.sort_shadow_cache,
+ &segment_id,
+ shadow_key,
+ SegmentCacheCategory::SortShadow,
+ 1,
+ Some(Arc::new(vec![(7, 0)])),
+ &off,
+ );
+ assert_eq!(shadow.value().as_ref().unwrap()[0], (7, 0));
+
+ let ids = idx.publish_current_with_budget(
+ &idx.id_pos_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::IdPositions,
+ 1,
+ std::collections::HashMap::from([(String::from("v1"), 0)]),
+ &off,
+ );
+ assert_eq!(ids.get("v1"), Some(&0));
+
+ let rows = idx.publish_current_with_budget(
+ &idx.row_seq_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::RowSequences,
+ 1,
+ vec![42],
+ &off,
+ );
+ assert_eq!(rows[0], 42);
+
+ let decoded = idx.publish_current_with_budget(
+ &idx.decoded_stored_cache,
+ &segment_id,
+ segment_id.clone(),
+ SegmentCacheCategory::DecodedStored,
+ 1,
+ vec![1, 2, 3],
+ &off,
+ );
+ assert_eq!(&decoded[..], &[1, 2, 3]);
+
+ assert_eq!(off.snapshot().current, 0);
+ assert_eq!(off.snapshot().refused, 7);
+ assert!(!idx.stored_slices_cache.contains_key(&segment_id));
+ assert!(!idx.dv_cache.contains_key(&segment_id));
+ assert!(!idx.stored_value_cache.contains_key(&segment_id));
+ assert!(!idx.id_pos_cache.contains_key(&segment_id));
+ assert!(!idx.row_seq_cache.contains_key(&segment_id));
+ assert!(!idx.decoded_stored_cache.contains_key(&segment_id));
+ let prefix = format!("{segment_id}\u{1}");
+ assert!(!idx
+ .sort_shadow_cache
+ .iter()
+ .any(|entry| entry.key().starts_with(&prefix)));
+ }
+
#[tokio::test]
async fn failed_cold_load_closes_followers_and_clears_singleflight() {
let dir = TempDir::new().unwrap();
@@ -1219,6 +1715,10 @@ pub struct Index {
test_stored_value_load_pause_before_publish: Arc<std::sync::atomic::AtomicBool>,
#[cfg(test)]
test_stored_value_load_ready_to_publish: Arc<std::sync::atomic::AtomicBool>,
+ #[cfg(test)]
+ test_segment_cache_publish_pause_mask: Arc<AtomicU64>,
+ #[cfg(test)]
+ test_segment_cache_publish_ready_mask: Arc<AtomicU64>,
// ── Per-index enrich lookup tables ────────────────────────────────────────
/// Named enrich tables: each table maps a key to a JSON object of extra
/// fields to merge into matching documents at ingest time.
@@ -1287,12 +1787,7 @@ pub struct Index {
/// valid for the lifetime of the segment. First agg against a segment
/// decodes the `.dv` sidecar; subsequent queries hit the cache and
/// skip I/O + LZ4 decompress + per-column decode entirely.
- dv_cache: Arc<
- dashmap::DashMap<
- String,
- Arc<std::collections::BTreeMap<String, xerj_storage::doc_values::Column>>,
- >,
- >,
+ dv_cache: Arc<dashmap::DashMap<String, Resident<DocValueMap>>>,
/// Per-(segment, field) sorted sort-key shadow: `(f64-bits, doc_pos)`
/// ascending by value, used by the field-sort candidate prefilter.
@@ -1305,7 +1800,7 @@ pub struct Index {
/// per query. Keyed by `"{segment_id}\u{1}{field}"`; same
/// immutable-segment lifecycle as `dv_cache` (never invalidated,
/// retired segment ids are simply never queried again).
- sort_shadow_cache: Arc<dashmap::DashMap<String, Option<Arc<Vec<(i64, u32)>>>>>,
+ sort_shadow_cache: Arc<dashmap::DashMap<String, Resident<Option<Arc<Vec<(i64, u32)>>>>>>,
/// Fields that have EVER been requested through `sorted_shadow_for`
/// on this index (bounded at 16) — the publish-time warm pre-builds
/// these fields' shadows for every new segment so the first sorted
@@ -1339,7 +1834,7 @@ pub struct Index {
/// `StoredSlices` offset index, so it is O(#ids) and FLAT vs corpus
/// size. Segments are immutable → the map never goes stale; evicted by
/// segment id at the merge-completion site alongside the other caches.
- id_pos_cache: Arc<dashmap::DashMap<String, Arc<std::collections::HashMap<String, u32>>>>,
+ id_pos_cache: Arc<dashmap::DashMap<String, Resident<std::collections::HashMap<String, u32>>>>,
/// Per-segment `stored-position → seq_no` map for the scored-family
/// columnar fast path. Segment ROW order is NOT insertion order (the
@@ -1349,7 +1844,7 @@ pub struct Index {
/// per segment from the (cached) `id_pos_map_for` + version map; only
/// consulted under `!deletes_present` (seq of a live id is then stable),
/// and evicted by segment id at the merge-completion site.
- row_seq_cache: Arc<dashmap::DashMap<String, Arc<Vec<u64>>>>,
+ row_seq_cache: Arc<dashmap::DashMap<String, Resident<Vec<u64>>>>,
/// Per-segment cache of decoded `Vec<Value>` from the stored section.
/// KNN search and segment-scan get-document paths used to call
@@ -1358,15 +1853,16 @@ pub struct Index {
/// vector query re-paid ~10 GB of decompress + parse work. Segments
/// are immutable once flushed, so this cache is correct without
/// invalidation. Same `Left unbounded for now` caveat as `dv_cache`.
- stored_value_cache: Arc<dashmap::DashMap<String, Arc<Vec<Value>>>>,
+ stored_value_cache: Arc<dashmap::DashMap<String, Resident<Vec<Value>>>>,
/// Serializes the short "is this segment still published? + cache insert"
/// transaction against merge eviction. Segment parsing never holds it.
- stored_value_cache_lifecycle: Arc<parking_lot::RwLock<()>>,
+ segment_cache_lifecycle: Arc<parking_lot::RwLock<()>>,
+ segment_hydration_budget: Arc<SegmentHydrationBudget>,
/// Per-segment cancellation-safe cold-load single-flight. The sender
/// publishes the immutable parsed section to every concurrent vector
/// query; the detached producer survives cancellation of any requester.
stored_value_loads:
- Arc<dashmap::DashMap<String, tokio::sync::watch::Sender<Option<Arc<Vec<Value>>>>>>,
+ Arc<dashmap::DashMap<String, tokio::sync::watch::Sender<Option<Resident<Vec<Value>>>>>>,
/// Bounds simultaneous whole-segment decompression/parsing across distinct
/// cold segments. Per-segment single-flight removes duplicate work; this
/// bound prevents a many-segment cold storm from multiplying peak memory.
@@ -1379,15 +1875,12 @@ pub struct Index {
/// lookups + a per-candidate `simd_json` parse — no per-query
/// zstd/LZ4 decompress and no O(segment-bytes) brace re-scan.
///
- /// Unlike `stored_value_cache` (unbounded parsed `Value`s, ~3-6× raw
- /// bytes) this holds the raw bytes once per segment and is BUDGETED:
- /// `stored_slices_cache_bytes` tracks retained size and inserts stop
- /// at `STORED_SLICES_CACHE_BUDGET` (miss → per-query decompress path,
- /// exactly the pre-cache behaviour). Segments are immutable, so
+ /// This holds raw bytes once per segment and shares the process-wide
+ /// segment hydration budget with all other immutable cache owners.
+ /// Refusal keeps the built value local to the current query. Segments are immutable, so
/// entries stay valid until a merge drops the segment id (evicted at
/// the merge-completion site alongside dv_cache/stored_value_cache).
- stored_slices_cache: Arc<dashmap::DashMap<String, Arc<StoredSlices>>>,
- stored_slices_cache_bytes: Arc<AtomicU64>,
+ stored_slices_cache: Arc<dashmap::DashMap<String, Resident<StoredSlices>>>,
/// Per-segment cache of the DECOMPRESSED stored section bytes for the
/// UNSORTED scan path (`scan_stored_section_into`). The bounded
/// collector already stops the scan O(from+size) docs in, but every
@@ -1397,8 +1890,7 @@ pub struct Index {
/// misses) it WAS the match_all/bool/range read-under-write tail.
/// Budgeted like `stored_slices_cache`; entries evicted by id at the
/// merge-completion site. Segments are immutable → no invalidation.
- decoded_stored_cache: Arc<dashmap::DashMap<String, Arc<Vec<u8>>>>,
- decoded_stored_cache_bytes: Arc<AtomicU64>,
+ decoded_stored_cache: Arc<dashmap::DashMap<String, Resident<Vec<u8>>>>,
/// Per-(segment, query-shape) match-count cache for the
/// `try_shortcut_count` Bool intersection arm. The fused columnar
/// walk is O(anchor-predicate matches) per segment PER QUERY — for a
@@ -1501,6 +1993,21 @@ pub struct Index {
}
impl Index {
+ /// Aggregate DashMap table capacities for the seven hydration families.
+ /// These are diagnostic table slots, separate from the refundable
+ /// retained-payload/key budget.
+ pub fn segment_hydration_cache_capacities(&self) -> [usize; 7] {
+ [
+ self.stored_slices_cache.capacity(),
+ self.dv_cache.capacity(),
+ self.stored_value_cache.capacity(),
+ self.sort_shadow_cache.capacity(),
+ self.id_pos_cache.capacity(),
+ self.row_seq_cache.capacity(),
+ self.decoded_stored_cache.capacity(),
+ ]
+ }
+
#[cfg(test)]
fn publication_test_point(&self, id: &str, point: PublicationTestPoint) {
// Clone the callback while holding the hook registry lock, then invoke
@@ -1621,6 +2128,7 @@ impl Index {
.fields()
.iter()
.any(|f| matches!(f.field_type, FieldType::Date));
+ let segment_hydration_budget = index_segment_hydration_budget(config);
let index = Arc::new(Self {
name,
schema: Arc::new(RwLock::new(managed)),
@@ -1669,6 +2177,10 @@ impl Index {
test_stored_value_load_ready_to_publish: Arc::new(std::sync::atomic::AtomicBool::new(
false,
)),
+ #[cfg(test)]
+ test_segment_cache_publish_pause_mask: Arc::new(AtomicU64::new(0)),
+ #[cfg(test)]
+ test_segment_cache_publish_ready_mask: Arc::new(AtomicU64::new(0)),
enrichments: Arc::new(RwLock::new(HashMap::new())),
hnsw: Arc::new(RwLock::new(None)),
hnsw_id_map: Arc::new(RwLock::new(HashMap::new())),
@@ -1703,13 +2215,12 @@ impl Index {
id_pos_cache: Arc::new(dashmap::DashMap::new()),
row_seq_cache: Arc::new(dashmap::DashMap::new()),
stored_value_cache: Arc::new(dashmap::DashMap::new()),
- stored_value_cache_lifecycle: Arc::new(parking_lot::RwLock::new(())),
+ segment_cache_lifecycle: Arc::new(parking_lot::RwLock::new(())),
+ segment_hydration_budget,
stored_value_loads: Arc::new(dashmap::DashMap::new()),
stored_value_load_semaphore: Arc::new(Semaphore::new(4)),
stored_slices_cache: Arc::new(dashmap::DashMap::new()),
- stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),
decoded_stored_cache: Arc::new(dashmap::DashMap::new()),
- decoded_stored_cache_bytes: Arc::new(AtomicU64::new(0)),
shortcut_count_cache: Arc::new(dashmap::DashMap::new()),
ghost_positions_cache: Arc::new(dashmap::DashMap::new()),
regexp_expand_cache: Arc::new(dashmap::DashMap::new()),
@@ -1902,6 +2413,7 @@ impl Index {
.fields()
.iter()
.any(|f| matches!(f.field_type, FieldType::Date));
+ let segment_hydration_budget = index_segment_hydration_budget(config);
let index = Arc::new(Self {
name,
schema: Arc::new(RwLock::new(schema)),
@@ -1925,6 +2437,10 @@ impl Index {
test_stored_value_load_ready_to_publish: Arc::new(std::sync::atomic::AtomicBool::new(
false,
)),
+ #[cfg(test)]
+ test_segment_cache_publish_pause_mask: Arc::new(AtomicU64::new(0)),
+ #[cfg(test)]
+ test_segment_cache_publish_ready_mask: Arc::new(AtomicU64::new(0)),
enrichments: Arc::new(RwLock::new(HashMap::new())),
store,
memtable: Arc::new(memtable),
@@ -1979,13 +2495,12 @@ impl Index {
id_pos_cache: Arc::new(dashmap::DashMap::new()),
row_seq_cache: Arc::new(dashmap::DashMap::new()),
stored_value_cache: Arc::new(dashmap::DashMap::new()),
- stored_value_cache_lifecycle: Arc::new(parking_lot::RwLock::new(())),
+ segment_cache_lifecycle: Arc::new(parking_lot::RwLock::new(())),
+ segment_hydration_budget,
stored_value_loads: Arc::new(dashmap::DashMap::new()),
stored_value_load_semaphore: Arc::new(Semaphore::new(4)),
stored_slices_cache: Arc::new(dashmap::DashMap::new()),
- stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),
decoded_stored_cache: Arc::new(dashmap::DashMap::new()),
- decoded_stored_cache_bytes: Arc::new(AtomicU64::new(0)),
shortcut_count_cache: Arc::new(dashmap::DashMap::new()),
ghost_positions_cache: Arc::new(dashmap::DashMap::new()),
regexp_expand_cache: Arc::new(dashmap::DashMap::new()),
@@ -4482,17 +4997,34 @@ impl Index {
// completion under the mixed read/write bench). On the
// blocking pool — the decompress is CPU-bound and must
// not stall the async merge driver.
- {
+ let prepublication_lease = {
let w_store = Arc::clone(&self.store);
let w_caches = self.publish_warm_caches();
let w_dir = self.data_dir.join("segments");
let w_id = merged_meta.id.clone();
let w_docs = merged_meta.doc_count;
- let _ = tokio::task::spawn_blocking(move || {
+ let lease = PrePublicationLease::known(w_caches.clone(), w_id.clone());
+ let worker_lease = lease.clone();
+ let warm_join = tokio::task::spawn_blocking(move || {
warm_segment_at_publish(&w_store, &w_dir, &w_caches, &w_id, w_docs);
+ drop(worker_lease);
})
.await;
- }
+ if let Err(error) = warm_join {
+ tracing::warn!(
+ merged_id = merged_meta.id.as_str(),
+ "merge: pre-publication cache warm failed: {error}"
+ );
+ // Warming is optional. Remove any partial entries
+ // left before the panic, then publish the valid
+ // merge output cold.
+ self.publish_warm_caches()
+ .remove_failed_output(merged_meta.id.as_str());
+ }
+ // Keep the parent lease armed across all work between
+ // warm and publication.
+ lease
+ };
// Mixed-RUW: seed the merged segment's SHORTCUT-COUNT
// cache from its inputs, BEFORE the swap makes it
// visible. Per-segment shortcut counts are PHYSICAL
@@ -4547,19 +5079,17 @@ impl Index {
}
if let Err(e) = self.store.apply_merge(&batch_slice, merged_meta.clone()) {
tracing::warn!("merge: apply_merge failed: {e}");
- // The pre-warmed slices belong to a segment that
- // never became visible — release them.
- if let Some((_, slices)) =
- self.stored_slices_cache.remove(merged_meta.id.as_str())
- {
- self.stored_slices_cache_bytes
- .fetch_sub(slices.retained_bytes(), Ordering::Relaxed);
- }
// Seeded shortcut counts for a segment that never
- // became visible — drop them.
+ // became visible — drop them. The armed lease removes
+ // all seven hydration owners when this scope exits.
self.shortcut_count_cache
.retain(|(seg, _), _| seg != merged_meta.id.as_str());
- } else {
+ continue;
+ }
+ // `apply_merge` has made the output authoritative. No
+ // await is permitted between publication and this commit.
+ prepublication_lease.commit();
+ {
// Write the merge output's `.ids` side-car AFTER the
// merge is committed: if we crashed before
// `apply_merge`, an output segment carrying a valid
@@ -4654,9 +5184,9 @@ impl Index {
// this writer and be removed here; one that arrives after this
// writer sees the retired snapshot and cannot publish.
{
- let _publication = self.stored_value_cache_lifecycle.write();
+ let _publication = self.segment_cache_lifecycle.write();
for id in &dropped_seg_ids {
- self.stored_value_cache.remove(id.as_str());
+ self.remove_segment_hydration_entries(id.as_str());
}
}
// Do not remove `stored_value_loads` here. Callers holding the
@@ -4665,17 +5195,6 @@ impl Index {
// key on completion, and the lifecycle transaction above
// prevents it from retaining the retired value in the cache.
for id in &dropped_seg_ids {
- self.dv_cache.remove(id.as_str());
- self.id_pos_cache.remove(id.as_str());
- self.row_seq_cache.remove(id.as_str());
- if let Some((_, slices)) = self.stored_slices_cache.remove(id.as_str()) {
- self.stored_slices_cache_bytes
- .fetch_sub(slices.retained_bytes(), Ordering::Relaxed);
- }
- if let Some((_, bytes)) = self.decoded_stored_cache.remove(id.as_str()) {
- self.decoded_stored_cache_bytes
- .fetch_sub(bytes.len() as u64, Ordering::Relaxed);
- }
self.fast_date_cache
.retain(|(seg, _), _| seg != id.as_str());
self.fast_date_sorted_cache
@@ -4685,8 +5204,6 @@ impl Index {
// merged-away segment leaked its shadows for the process
// lifetime under a sustained writer.
let shadow_prefix = format!("{}\u{1}", id.as_str());
- self.sort_shadow_cache
- .retain(|k, _| !k.starts_with(&shadow_prefix));
self.shortcut_count_cache
.retain(|(seg, _), _| seg != id.as_str());
self.range_prefilter_cache
@@ -10422,7 +10939,7 @@ impl Index {
let cached_bytes: Option<Vec<u8>> = self
.decoded_stored_cache
.get(seg_id.as_str())
- .map(|e| e.value().as_ref().clone())
+ .map(|e| e.value().value().clone())
.or_else(|| {
self.stored_slices_cache
.get(seg_id.as_str())
@@ -10478,40 +10995,39 @@ impl Index {
bytes: stored_bytes,
offsets,
};
- let sz = slices.retained_bytes();
- if self
- .stored_slices_cache_bytes
- .load(Ordering::Relaxed)
- .saturating_add(sz)
- <= stored_slices_cache_budget()
- && self
- .stored_slices_cache
- .insert(seg_id.clone(), Arc::new(slices))
- .is_none()
- {
- self.stored_slices_cache_bytes
- .fetch_add(sz, Ordering::Relaxed);
- }
+ let bytes = cache_estimates::stored_slices_bytes(
+ &seg_id,
+ std::mem::size_of::<CacheResident<StoredSlices>>(),
+ slices.bytes.capacity(),
+ slices.offsets.capacity(),
+ );
+ let _ = self.publish_current(
+ &self.stored_slices_cache,
+ &seg_id,
+ seg_id.clone(),
+ SegmentCacheCategory::StoredSlices,
+ bytes,
+ slices,
+ );
} else if !want_cache {
// Unsorted path: retain the decompressed
// section so the NEXT query on this
// segment skips the decompress (the
// read-under-write match_all/bool/range
// tail fix — see `decoded_stored_cache`).
- let sz = stored_bytes.len() as u64;
- if self
- .decoded_stored_cache_bytes
- .load(Ordering::Relaxed)
- .saturating_add(sz)
- <= DECODED_STORED_CACHE_BUDGET
- && self
- .decoded_stored_cache
- .insert(seg_id.clone(), Arc::new(stored_bytes))
- .is_none()
- {
- self.decoded_stored_cache_bytes
- .fetch_add(sz, Ordering::Relaxed);
- }
+ let bytes = cache_estimates::decoded_stored_bytes(
+ &seg_id,
+ std::mem::size_of::<CacheResident<Vec<u8>>>(),
+ stored_bytes.capacity(),
+ );
+ let _ = self.publish_current(
+ &self.decoded_stored_cache,
+ &seg_id,
+ seg_id.clone(),
+ SegmentCacheCategory::DecodedStored,
+ bytes,
+ stored_bytes,
+ );
}
}
}
@@ -13020,6 +13536,7 @@ impl Index {
field_str,
meta.doc_count,
)?;
+ let shadow = shadow.value().as_ref()?;
let (slo, slo_incl, shi, shi_incl) = shadow_range_bounds(
gte.as_ref(),
gt.as_ref(),
@@ -13027,7 +13544,7 @@ impl Index {
lt.as_ref(),
)?;
seg_matches = seg_matches.saturating_add(shadow_range_count(
- &shadow, slo, shi, slo_incl, shi_incl,
+ shadow, slo, shi, slo_incl, shi_incl,
));
}
None => return None, // abandon — field absent in this segment
@@ -13395,10 +13912,124 @@ impl Index {
fn publish_warm_caches(&self) -> PublishWarmCaches {
PublishWarmCaches {
slices: Arc::clone(&self.stored_slices_cache),
- slices_bytes: Arc::clone(&self.stored_slices_cache_bytes),
dv: Arc::clone(&self.dv_cache),
shadow: Arc::clone(&self.sort_shadow_cache),
shadow_fields: Arc::clone(&self.sort_shadow_fields),
+ lifecycle: Arc::clone(&self.segment_cache_lifecycle),
+ budget: Arc::clone(&self.segment_hydration_budget),
+ stored_values: Arc::clone(&self.stored_value_cache),
+ id_positions: Arc::clone(&self.id_pos_cache),
+ row_sequences: Arc::clone(&self.row_seq_cache),
+ decoded_stored: Arc::clone(&self.decoded_stored_cache),
+ }
+ }
+
+ /// Publish one already-built immutable value if its segment is still
+ /// current. Build and retained-size estimation happen before this short
+ /// transaction. Lock order: build mutex → lifecycle → DashMap shard.
+ fn publish_current<T>(
+ &self,
+ map: &dashmap::DashMap<String, Resident<T>>,
+ segment_id: &str,
+ key: String,
+ category: SegmentCacheCategory,
+ bytes: u64,
+ value: T,
+ ) -> Resident<T> {
+ self.publish_current_with_budget(
+ map,
+ segment_id,
+ key,
+ category,
+ bytes,
+ value,
+ &self.segment_hydration_budget,
+ )
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ fn publish_current_with_budget<T>(
+ &self,
+ map: &dashmap::DashMap<String, Resident<T>>,
+ segment_id: &str,
+ key: String,
+ category: SegmentCacheCategory,
+ bytes: u64,
+ value: T,
+ budget: &Arc<SegmentHydrationBudget>,
+ ) -> Resident<T> {
+ #[cfg(test)]
+ {
+ let bit = 1_u64 << category as usize;
+ self.test_segment_cache_publish_ready_mask
+ .fetch_or(bit, Ordering::Release);
+ while self
+ .test_segment_cache_publish_pause_mask
+ .load(Ordering::Acquire)
+ & bit
+ != 0
+ {
+ std::thread::yield_now();
+ }
+ }
+ let _publication = self.segment_cache_lifecycle.read();
+ if !self
+ .store
+ .snapshot()
+ .segments
+ .iter()
+ .any(|meta| meta.id == segment_id)
+ {
+ return CacheResident::uncached(value);
+ }
+ use dashmap::mapref::entry::Entry;
+ match map.entry(key) {
+ Entry::Occupied(entry) => Arc::clone(entry.get()),
+ Entry::Vacant(entry) => {
+ let Some(charge) = budget.try_charge(category, bytes) else {
+ return CacheResident::uncached(value);
+ };
+ let resident = CacheResident::admitted(value, charge, category, bytes);
+ entry.insert(Arc::clone(&resident));
+ resident
+ }
+ }
+ }
+
+ /// Caller holds `segment_cache_lifecycle.write()`.
+ fn remove_segment_hydration_entries(&self, segment_id: &str) {
+ self.stored_value_cache.remove(segment_id);
+ self.dv_cache.remove(segment_id);
+ self.id_pos_cache.remove(segment_id);
+ self.row_seq_cache.remove(segment_id);
+ self.stored_slices_cache.remove(segment_id);
+ self.decoded_stored_cache.remove(segment_id);
+ let shadow_prefix = format!("{segment_id}\u{1}");
+ self.sort_shadow_cache
+ .retain(|key, _| !key.starts_with(&shadow_prefix));
+
+ // DashMap retains shard capacity after logical removals. Shrinking
+ // only an entirely empty map avoids locking every shard on each merge.
+ if self.stored_value_cache.is_empty() {
+ self.stored_value_cache.shrink_to_fit();
+ }
+ if self.dv_cache.is_empty() {
+ self.dv_cache.shrink_to_fit();
+ }
+ if self.id_pos_cache.is_empty() {
+ self.id_pos_cache.shrink_to_fit();
+ }
+ if self.row_seq_cache.is_empty() {
+ self.row_seq_cache.shrink_to_fit();
+ }
+ if self.stored_slices_cache.is_empty() {
+ self.stored_slices_cache.shrink_to_fit();
+ }
+ if self.decoded_stored_cache.is_empty() {
+ self.decoded_stored_cache.shrink_to_fit();
+ }
+ if self.sort_shadow_cache.is_empty() {
+ self.sort_shadow_cache.shrink_to_fit();
}
}
@@ -13406,7 +14037,7 @@ impl Index {
&self,
segments_dir: &std::path::Path,
segment_id: &str,
- ) -> Option<Arc<std::collections::BTreeMap<String, xerj_storage::doc_values::Column>>> {
+ ) -> Option<Resident<DocValueMap>> {
if let Some(entry) = self.dv_cache.get(segment_id) {
return Some(Arc::clone(entry.value()));
}
@@ -13414,10 +14045,20 @@ impl Index {
if cols.is_empty() {
return None;
}
- let arc = Arc::new(cols);
- self.dv_cache
- .insert(segment_id.to_string(), Arc::clone(&arc));
- Some(arc)
+ let key = segment_id.to_string();
+ let bytes = cache_estimates::doc_values_bytes(
+ &key,
+ std::mem::size_of::<CacheResident<DocValueMap>>(),
+ &cols,
+ );
+ Some(self.publish_current(
+ &self.dv_cache,
+ segment_id,
+ key,
+ SegmentCacheCategory::DocValues,
+ bytes,
+ cols,
+ ))
}
/// Columnar `function_score { match_all, field_value_factor }` fast path.
@@ -13807,7 +14448,7 @@ impl Index {
// gate forces it to equal Σ segment doc_count (bench mapping has no
// nulls, so this coincides with ES's field docCount).
let mut seg_cols = Vec::with_capacity(segments.len());
- let mut seg_seqs: Vec<Arc<Vec<u64>>> = Vec::with_capacity(segments.len());
+ let mut seg_seqs: Vec<Resident<Vec<u64>>> = Vec::with_capacity(segments.len());
let mut seg_ghosts: Vec<Option<Arc<Vec<u64>>>> = Vec::with_capacity(segments.len());
for meta in segments {
seg_cols.push(self.dv_columns_for(segments_dir, &meta.id)?);
@@ -15031,7 +15672,7 @@ impl Index {
/// catch that. serde_json handles every variant correctly and the
/// per-doc parse cost is amortised once across the whole cache
/// lifetime of a segment.
- fn stored_values_for(&self, segment_id: &str) -> Option<Arc<Vec<Value>>> {
+ fn stored_values_for(&self, segment_id: &str) -> Option<Resident<Vec<Value>>> {
if let Some(entry) = self.stored_value_cache.get(segment_id) {
return Some(Arc::clone(entry.value()));
}
@@ -15039,21 +15680,20 @@ impl Index {
let stored_bytes_raw = reader.section(SectionType::Stored).ok()??;
let stored_bytes = xerj_storage::stored_codec::decode_stored(stored_bytes_raw).ok()?;
let docs: Vec<Value> = serde_json::from_slice(&stored_bytes).ok()?;
- let arc = Arc::new(docs);
- {
- let _publication = self.stored_value_cache_lifecycle.read();
- if self
- .store
- .snapshot()
- .segments
- .iter()
- .any(|meta| meta.id == segment_id)
- {
- self.stored_value_cache
- .insert(segment_id.to_string(), Arc::clone(&arc));
- }
- }
- Some(arc)
+ let key = segment_id.to_string();
+ let bytes = cache_estimates::stored_values_bytes(
+ &key,
+ std::mem::size_of::<CacheResident<Vec<Value>>>(),
+ &docs,
+ );
+ Some(self.publish_current(
+ &self.stored_value_cache,
+ segment_id,
+ key,
+ SegmentCacheCategory::StoredValues,
+ bytes,
+ docs,
+ ))
}
/// Async boundary for a cold stored-value load.
@@ -15069,7 +15709,7 @@ impl Index {
/// is useful: it warms the immutable segment for the next request. The
/// segment-current check prevents such a detached completion from
/// resurrecting a cache entry that a concurrent merge already retired.
- async fn stored_values_for_async(&self, segment_id: &str) -> Option<Arc<Vec<Value>>> {
+ async fn stored_values_for_async(&self, segment_id: &str) -> Option<Resident<Vec<Value>>> {
if let Some(entry) = self.stored_value_cache.get(segment_id) {
return Some(Arc::clone(entry.value()));
}
@@ -15085,7 +15725,8 @@ impl Index {
let store = Arc::clone(&self.store);
let cache = Arc::clone(&self.stored_value_cache);
- let lifecycle = Arc::clone(&self.stored_value_cache_lifecycle);
+ let lifecycle = Arc::clone(&self.segment_cache_lifecycle);
+ let budget = Arc::clone(&self.segment_hydration_budget);
let loads = Arc::clone(&self.stored_value_loads);
let load_semaphore = Arc::clone(&self.stored_value_load_semaphore);
let load_segment_id = segment_id.clone();
@@ -15118,7 +15759,12 @@ impl Index {
let stored_bytes =
xerj_storage::stored_codec::decode_stored(stored_bytes_raw).ok()?;
let docs: Vec<Value> = serde_json::from_slice(&stored_bytes).ok()?;
- let arc = Arc::new(docs);
+ let key = load_segment_id.clone();
+ let bytes = cache_estimates::stored_values_bytes(
+ &key,
+ std::mem::size_of::<CacheResident<Vec<Value>>>(),
+ &docs,
+ );
#[cfg(test)]
{
ready_to_publish.store(true, Ordering::Release);
@@ -15126,18 +15772,40 @@ impl Index {
std::thread::sleep(std::time::Duration::from_millis(1));
}
}
- {
+ let resident = {
let _publication = lifecycle.read();
- if store
+ if !store
.snapshot()
.segments
.iter()
.any(|meta| meta.id == load_segment_id)
{
- cache.insert(load_segment_id, Arc::clone(&arc));
+ CacheResident::uncached(docs)
+ } else {
+ use dashmap::mapref::entry::Entry;
+ match cache.entry(key) {
+ Entry::Occupied(entry) => Arc::clone(entry.get()),
+ Entry::Vacant(entry) => {
+ if let Some(charge) = budget.try_charge(
+ SegmentCacheCategory::StoredValues,
+ bytes,
+ ) {
+ let resident = CacheResident::admitted(
+ docs,
+ charge,
+ SegmentCacheCategory::StoredValues,
+ bytes,
+ );
+ entry.insert(Arc::clone(&resident));
+ resident
+ } else {
+ CacheResident::uncached(docs)
+ }
+ }
+ }
}
- }
- Some(arc)
+ };
+ Some(resident)
});
if let Ok(Some(value)) = blocking.await {
let _ = sender.send(Some(value));
@@ -15280,12 +15948,13 @@ impl Index {
let seg_doc_count = k.doc_count as u64;
let shadow =
self.sorted_shadow_for(segments_dir, segment_id, field, seg_doc_count)?;
+ let shadow = shadow.value().as_ref()?;
let (slo, slo_incl, shi, shi_incl) = shadow_range_bounds(gte, gt, lte, lt)?;
// Same selectivity precheck on the shadow scale.
- if shadow_range_count(&shadow, slo, shi, slo_incl, shi_incl) as usize > cap {
+ if shadow_range_count(shadow, slo, shi, slo_incl, shi_incl) as usize > cap {
return None;
}
- shadow_range_positions(&shadow, slo, shi, slo_incl, shi_incl)
+ shadow_range_positions(shadow, slo, shi, slo_incl, shi_incl)
}
None => return None,
};
@@ -15391,7 +16060,7 @@ impl Index {
&self,
seg_id: &str,
expect_docs: u64,
- ) -> Option<Arc<std::collections::HashMap<String, u32>>> {
+ ) -> Option<Resident<std::collections::HashMap<String, u32>>> {
if let Some(entry) = self.id_pos_cache.get(seg_id) {
return Some(Arc::clone(entry.value()));
}
@@ -15423,10 +16092,20 @@ impl Index {
if map.len() as u64 != expect_docs {
return None;
}
- let arc = Arc::new(map);
- self.id_pos_cache
- .insert(seg_id.to_string(), Arc::clone(&arc));
- Some(arc)
+ let key = seg_id.to_string();
+ let bytes = cache_estimates::id_positions_bytes(
+ &key,
+ std::mem::size_of::<CacheResident<std::collections::HashMap<String, u32>>>(),
+ &map,
+ );
+ Some(self.publish_current(
+ &self.id_pos_cache,
+ seg_id,
+ key,
+ SegmentCacheCategory::IdPositions,
+ bytes,
+ map,
+ ))
}
/// Lazily-built, cached `stored-position → seq_no` array for a segment
@@ -15445,7 +16124,7 @@ impl Index {
/// go BACK to live in place (a re-index lands in the memtable, which
/// keeps the columnar path off entirely), so the sentinel is never
/// observable in results.
- fn row_seqs_for(&self, seg_id: &str, expect_docs: u64) -> Option<Arc<Vec<u64>>> {
+ fn row_seqs_for(&self, seg_id: &str, expect_docs: u64) -> Option<Resident<Vec<u64>>> {
if let Some(entry) = self.row_seq_cache.get(seg_id) {
return Some(Arc::clone(entry.value()));
}
@@ -15458,10 +16137,20 @@ impl Index {
None => return None,
}
}
- let arc = Arc::new(seqs);
- self.row_seq_cache
- .insert(seg_id.to_string(), Arc::clone(&arc));
- Some(arc)
+ let key = seg_id.to_string();
+ let bytes = cache_estimates::row_sequences_bytes(
+ &key,
+ std::mem::size_of::<CacheResident<Vec<u64>>>(),
+ seqs.capacity(),
+ );
+ Some(self.publish_current(
+ &self.row_seq_cache,
+ seg_id,
+ key,
+ SegmentCacheCategory::RowSequences,
+ bytes,
+ seqs,
+ ))
}
/// Position pre-filter for an `ids` query: resolve each requested id to its
@@ -15659,7 +16348,7 @@ impl Index {
segment_id: &str,
field: &str,
seg_doc_count: u64,
- ) -> Option<Arc<Vec<(i64, u32)>>> {
+ ) -> Option<Resident<Option<Arc<Vec<(i64, u32)>>>>> {
// Register the field so the publish-time warm pre-builds this
// shadow for every FUTURE segment (bounded registry).
if self.sort_shadow_fields.len() < 16 && !self.sort_shadow_fields.contains_key(field) {
@@ -15667,7 +16356,7 @@ impl Index {
}
let key = format!("{segment_id}\u{1}{field}");
if let Some(entry) = self.sort_shadow_cache.get(&key) {
- return entry.value().clone();
+ return Some(Arc::clone(entry.value()));
}
// A missing/unreadable dv sidecar is a TRANSIENT state (a segment can
// become visible a beat before its sidecar read succeeds under heavy
@@ -15679,8 +16368,19 @@ impl Index {
// sorted read full-scanned it until a merge retired it.
let cols = self.dv_columns_for(segments_dir, segment_id)?;
let built = build_sort_shadow(&cols, field, seg_doc_count);
- self.sort_shadow_cache.insert(key, built.clone());
- built
+ let bytes = cache_estimates::sort_shadow_bytes(
+ &key,
+ std::mem::size_of::<CacheResident<Option<Arc<Vec<(i64, u32)>>>>>(),
+ built.as_ref().map(|shadow| shadow.capacity()),
+ );
+ Some(self.publish_current(
+ &self.sort_shadow_cache,
+ segment_id,
+ key,
+ SegmentCacheCategory::SortShadow,
+ bytes,
+ built,
+ ))
}
fn build_sort_candidates_prefilter(
@@ -15698,6 +16398,7 @@ impl Index {
return None;
}
let shadow = self.sorted_shadow_for(segments_dir, segment_id, &sf.field, seg_doc_count)?;
+ let shadow = shadow.value().as_ref()?;
let sorted: &[(i64, u32)] = shadow.as_slice();
if sorted.is_empty() {
return None;
@@ -15911,6 +16612,7 @@ impl Index {
return None;
}
let shadow = self.sorted_shadow_for(segments_dir, segment_id, &sf.field, seg_doc_count)?;
+ let shadow = shadow.value().as_ref()?;
let sorted: &[(i64, u32)] = shadow.as_slice();
if sorted.is_empty() {
return None;
@@ -15983,7 +16685,7 @@ impl Index {
/// never a per-doc parse). `None` on open/decode failure or a
/// malformed section (offsets incomplete) — caller falls back to the
/// legacy scan.
- fn stored_slices_for(&self, seg_id: &str, expect_docs: u64) -> Option<Arc<StoredSlices>> {
+ fn stored_slices_for(&self, seg_id: &str, expect_docs: u64) -> Option<Resident<StoredSlices>> {
if let Some(entry) = self.stored_slices_cache.get(seg_id) {
return Some(Arc::clone(entry.value()));
}
@@ -16007,7 +16709,7 @@ impl Index {
// Reuse already-decompressed bytes from `decoded_stored_cache`
// before paying a fresh open + decompress.
let bytes: Vec<u8> = if let Some(entry) = self.decoded_stored_cache.get(seg_id) {
- entry.value().as_ref().clone()
+ entry.value().value().clone()
} else {
let reader = self.store.open_segment_arc(seg_id).ok()?;
let raw = reader.section(SectionType::Stored).ok()??;
@@ -16020,22 +16722,22 @@ impl Index {
if offsets.len() as u64 != expect_docs {
return None;
}
- let slices = Arc::new(StoredSlices { bytes, offsets });
- let sz = slices.retained_bytes();
- if self
- .stored_slices_cache_bytes
- .load(Ordering::Relaxed)
- .saturating_add(sz)
- <= stored_slices_cache_budget()
- && self
- .stored_slices_cache
- .insert(seg_id.to_string(), Arc::clone(&slices))
- .is_none()
- {
- self.stored_slices_cache_bytes
- .fetch_add(sz, Ordering::Relaxed);
- }
- Some(slices)
+ let slices = StoredSlices { bytes, offsets };
+ let key = seg_id.to_string();
+ let bytes = cache_estimates::stored_slices_bytes(
+ &key,
+ std::mem::size_of::<CacheResident<StoredSlices>>(),
+ slices.bytes.capacity(),
+ slices.offsets.capacity(),
+ );
+ Some(self.publish_current(
+ &self.stored_slices_cache,
+ seg_id,
+ key,
+ SegmentCacheCategory::StoredSlices,
+ bytes,
+ slices,
+ ))
}
/// Per-segment GHOST-position bitmap (b7 DEFECT 1c): bit `pos` set ⇔
@@ -16884,6 +17586,7 @@ impl Drop for FlushDurationTimer {
#[derive(Clone)]
struct FlushPublisherTestHook {
callback: Arc<dyn Fn() -> bool + Send + Sync>,
+ after_warm: Option<Arc<dyn Fn(&str) -> bool + Send + Sync>>,
ledger: &'static crate::ingest_memory::Ledger,
target_memtable: usize,
}
@@ -17104,6 +17807,9 @@ async fn do_flush_shard(
// build them. The cost (~700 ns / doc) is irrelevant next to the
// 50-100 ms segment write that follows, and 100K-doc/s ingest
// bench numbers are unaffected.
+ let failed_warm_caches = warm_caches.clone();
+ let warmed_segment_id = Arc::new(parking_lot::Mutex::new(None::<String>));
+ let callback_warmed_segment_id = Arc::clone(&warmed_segment_id);
let build_fts = move |meta: &xerj_storage::segment::SegmentMeta| -> xerj_storage::Result<()> {
#[cfg(test)]
let test_callback = test_hook
@@ -17111,6 +17817,11 @@ async fn do_flush_shard(
.filter(|hook| hook.target_memtable == test_target)
.map(|hook| Arc::clone(&hook.callback));
#[cfg(test)]
+ let after_warm_callback = test_hook
+ .as_ref()
+ .filter(|hook| hook.target_memtable == test_target)
+ .and_then(|hook| hook.after_warm.as_ref().map(Arc::clone));
+ #[cfg(test)]
if let Some(callback) = test_callback {
if callback() {
return Err(std::io::Error::other("injected flush publisher failure").into());
@@ -17119,7 +17830,7 @@ async fn do_flush_shard(
// Whole side-car build runs on the dedicated ingest pool so a
// flush burst can't queue search/agg par_iters behind it on the
// global rayon pool (read-under-write; see `crate::ingest_pool`).
- crate::background_pool().install(|| {
+ crate::background_pool().install(|| -> xerj_storage::Result<()> {
// ── Doc-values side-car (always built) ────────────────────
let t_dv = std::time::Instant::now();
{
@@ -17172,6 +17883,7 @@ async fn do_flush_shard(
// warm left a ~250 ms visible-but-cold window per flush storm;
// 64 racing queries each decompressed all 16 fresh segments
// inside it — the dec≈900 ms stall episodes.)
+ *callback_warmed_segment_id.lock() = Some(meta.id.clone());
warm_segment_at_publish(
&store_for_warm,
&segments_dir_for_dv,
@@ -17179,7 +17891,16 @@ async fn do_flush_shard(
meta.id.as_str(),
meta.doc_count,
);
- });
+ #[cfg(test)]
+ if let Some(callback) = after_warm_callback {
+ if callback(meta.id.as_str()) {
+ return Err(
+ std::io::Error::other("injected post-warm publisher failure").into(),
+ );
+ }
+ }
+ Ok(())
+ })?;
let _ = &segments_dir;
let _ = &drained_fts;
@@ -17202,8 +17923,18 @@ async fn do_flush_shard(
let t_finalize = std::time::Instant::now();
let _fin_permit = crate::flush_finalize_gate().acquire().await.ok();
let finalize_join = tokio::task::spawn_blocking(move || {
- crate::background_pool()
- .install(|| store.finalize_flush_with_publisher(storage_drained, build_fts))
+ // This lease lives in the blocking worker, not the async waiter:
+ // aborting an async task cannot cancel `spawn_blocking`.
+ let lease =
+ PrePublicationLease::tracked(failed_warm_caches, Arc::clone(&warmed_segment_id));
+ let result = crate::background_pool()
+ .install(|| store.finalize_flush_with_publisher(storage_drained, build_fts));
+ if matches!(result, Ok(Some(_))) {
+ // finalize_flush_with_publisher publishes the snapshot before it
+ // returns Some. Commit while still in the worker, with no await.
+ lease.commit();
+ }
+ result
})
.await;
let meta = match finalize_join {
@@ -17212,7 +17943,9 @@ async fn do_flush_shard(
tracing::warn!("storage finalize returned None — unexpected");
return Ok(());
}
- Ok(Err(e)) => return Err(e.into()),
+ Ok(Err(e)) => {
+ return Err(e.into());
+ }
Err(join_e) => {
tracing::warn!("finalize spawn_blocking join failed: {join_e}");
return Ok(());
@@ -23817,12 +24550,6 @@ struct StoredSlices {
offsets: Vec<(u32, u32)>,
}
-impl StoredSlices {
- fn retained_bytes(&self) -> u64 {
- (self.bytes.len() + self.offsets.len() * 8 + 64) as u64
- }
-}
-
/// Segment sort-key shadow builder — shared by the query-path
/// `Index::sorted_shadow_for` and the publish-time warm. See
/// `sorted_shadow_for` for the eligibility contract (no nulls, full
@@ -23989,16 +24716,105 @@ fn shadow_range_count(
/// handle. All fields are the Index's own Arcs.
#[derive(Clone)]
struct PublishWarmCaches {
- slices: Arc<dashmap::DashMap<String, Arc<StoredSlices>>>,
- slices_bytes: Arc<AtomicU64>,
- dv: Arc<
- dashmap::DashMap<
- String,
- Arc<std::collections::BTreeMap<String, xerj_storage::doc_values::Column>>,
- >,
- >,
- shadow: Arc<dashmap::DashMap<String, Option<Arc<Vec<(i64, u32)>>>>>,
+ slices: Arc<dashmap::DashMap<String, Resident<StoredSlices>>>,
+ dv: Arc<dashmap::DashMap<String, Resident<DocValueMap>>>,
+ shadow: Arc<dashmap::DashMap<String, Resident<Option<Arc<Vec<(i64, u32)>>>>>>,
shadow_fields: Arc<dashmap::DashMap<String, ()>>,
+ lifecycle: Arc<parking_lot::RwLock<()>>,
+ budget: Arc<SegmentHydrationBudget>,
+ stored_values: Arc<dashmap::DashMap<String, Resident<Vec<Value>>>>,
+ id_positions: Arc<dashmap::DashMap<String, Resident<std::collections::HashMap<String, u32>>>>,
+ row_sequences: Arc<dashmap::DashMap<String, Resident<Vec<u64>>>>,
+ decoded_stored: Arc<dashmap::DashMap<String, Resident<Vec<u8>>>>,
+}
+
+/// Capability documenting the sole path allowed to bypass the authoritative
+/// snapshot-current check: flush/merge output warming immediately before
+/// publication. It is private to this module and carries no ambient authority.
+struct PrePublication(());
+
+impl PublishWarmCaches {
+ fn remove_failed_output(&self, segment_id: &str) {
+ let _publication = self.lifecycle.write();
+ self.slices.remove(segment_id);
+ self.dv.remove(segment_id);
+ self.stored_values.remove(segment_id);
+ self.id_positions.remove(segment_id);
+ self.row_sequences.remove(segment_id);
+ self.decoded_stored.remove(segment_id);
+ let prefix = format!("{segment_id}\u{1}");
+ self.shadow.retain(|key, _| !key.starts_with(&prefix));
+ }
+}
+
+/// Removes cache owners for an output that did not reach authoritative
+/// publication. The guard must live inside the blocking worker that performs
+/// the warm: aborting the async waiter does not cancel `spawn_blocking`, so an
+/// outer guard could clean too early and allow the detached worker to reinsert.
+struct PrePublicationLeaseInner {
+ caches: PublishWarmCaches,
+ segment_id: Arc<parking_lot::Mutex<Option<String>>>,
+ committed: AtomicBool,
+}
+
+impl Drop for PrePublicationLeaseInner {
+ fn drop(&mut self) {
+ if !self.committed.load(Ordering::Acquire) {
+ if let Some(segment_id) = self.segment_id.lock().take() {
+ self.caches.remove_failed_output(&segment_id);
+ }
+ }
+ }
+}
+
+#[derive(Clone)]
+struct PrePublicationLease(Arc<PrePublicationLeaseInner>);
+
+impl PrePublicationLease {
+ fn known(caches: PublishWarmCaches, segment_id: String) -> Self {
+ Self(Arc::new(PrePublicationLeaseInner {
+ caches,
+ segment_id: Arc::new(parking_lot::Mutex::new(Some(segment_id))),
+ committed: AtomicBool::new(false),
+ }))
+ }
+
+ fn tracked(
+ caches: PublishWarmCaches,
+ segment_id: Arc<parking_lot::Mutex<Option<String>>>,
+ ) -> Self {
+ Self(Arc::new(PrePublicationLeaseInner {
+ caches,
+ segment_id,
+ committed: AtomicBool::new(false),
+ }))
+ }
+
+ fn commit(&self) {
+ self.0.committed.store(true, Ordering::Release);
+ }
+}
+
+fn publish_prepublication<T>(
+ _capability: &PrePublication,
+ caches: &PublishWarmCaches,
+ map: &dashmap::DashMap<String, Resident<T>>,
+ key: String,
+ category: SegmentCacheCategory,
+ bytes: u64,
+ value: T,
+) -> Option<Resident<T>> {
+ let _publication = caches.lifecycle.read();
+ use dashmap::mapref::entry::Entry;
+ match map.entry(key) {
+ Entry::Occupied(entry) => Some(Arc::clone(entry.get())),
+ Entry::Vacant(entry) => {
+ let charge = caches.budget.try_charge(category, bytes)?;
+ let resident = CacheResident::admitted(value, charge, category, bytes);
+ entry.insert(Arc::clone(&resident));
+ Some(resident)
+ }
+ }
}
/// Warm one just-written segment's read-path caches OUTSIDE the query
@@ -24027,6 +24843,7 @@ fn warm_segment_at_publish(
seg_id: &str,
expect_docs: u64,
) {
+ let prepublication = PrePublication(());
// Sustained-ingest escape hatch: XERJ_NO_PUBLISH_WARM=1 skips ALL
// publish-time cache warming (stored slices + dv columns + sort
// shadows). Queries fall back to per-query decode-on-miss — the
@@ -24044,7 +24861,7 @@ fn warm_segment_at_publish(
}
// 1. Stored slices.
if !caches.slices.contains_key(seg_id) {
- let built: Option<Arc<StoredSlices>> = (|| {
+ let built: Option<StoredSlices> = (|| {
let reader = store.open_segment_arc(seg_id).ok()?;
let raw = reader.section(SectionType::Stored).ok()??;
let bytes = xerj_storage::stored_codec::decode_stored(raw).ok()?;
@@ -24055,35 +24872,52 @@ fn warm_segment_at_publish(
if offsets.len() as u64 != expect_docs {
return None;
}
- Some(Arc::new(StoredSlices { bytes, offsets }))
+ Some(StoredSlices { bytes, offsets })
})();
if let Some(slices) = built {
- let sz = slices.retained_bytes();
- if caches
- .slices_bytes
- .load(Ordering::Relaxed)
- .saturating_add(sz)
- <= stored_slices_cache_budget()
- && caches.slices.insert(seg_id.to_string(), slices).is_none()
- {
- caches.slices_bytes.fetch_add(sz, Ordering::Relaxed);
- }
+ let key = seg_id.to_string();
+ let bytes = cache_estimates::stored_slices_bytes(
+ &key,
+ std::mem::size_of::<CacheResident<StoredSlices>>(),
+ slices.bytes.capacity(),
+ slices.offsets.capacity(),
+ );
+ let _ = publish_prepublication(
+ &prepublication,
+ caches,
+ &caches.slices,
+ key,
+ SegmentCacheCategory::StoredSlices,
+ bytes,
+ slices,
+ );
}
}
// 2. dv columns (mirrors `Index::dv_columns_for`'s miss arm).
- let cols: Option<Arc<std::collections::BTreeMap<String, xerj_storage::doc_values::Column>>> =
- if let Some(entry) = caches.dv.get(seg_id) {
- Some(Arc::clone(entry.value()))
+ let cols: Option<Resident<DocValueMap>> = if let Some(entry) = caches.dv.get(seg_id) {
+ Some(Arc::clone(entry.value()))
+ } else {
+ let cols = read_doc_values_sidecar(segments_dir, seg_id);
+ if cols.is_empty() {
+ None
} else {
- let cols = read_doc_values_sidecar(segments_dir, seg_id);
- if cols.is_empty() {
- None
- } else {
- let arc = Arc::new(cols);
- caches.dv.insert(seg_id.to_string(), Arc::clone(&arc));
- Some(arc)
- }
- };
+ let key = seg_id.to_string();
+ let bytes = cache_estimates::doc_values_bytes(
+ &key,
+ std::mem::size_of::<CacheResident<DocValueMap>>(),
+ &cols,
+ );
+ publish_prepublication(
+ &prepublication,
+ caches,
+ &caches.dv,
+ key,
+ SegmentCacheCategory::DocValues,
+ bytes,
+ cols,
+ )
+ }
+ };
// 3. Sort shadows for every historically-sorted field.
if let Some(cols) = cols {
for e in caches.shadow_fields.iter() {
@@ -24091,7 +24925,20 @@ fn warm_segment_at_publish(
let key = format!("{seg_id}\u{1}{field}");
if !caches.shadow.contains_key(&key) {
let built = build_sort_shadow(&cols, field, expect_docs);
- caches.shadow.insert(key, built);
+ let bytes = cache_estimates::sort_shadow_bytes(
+ &key,
+ std::mem::size_of::<CacheResident<Option<Arc<Vec<(i64, u32)>>>>>(),
+ built.as_ref().map(|shadow| shadow.capacity()),
+ );
+ let _ = publish_prepublication(
+ &prepublication,
+ caches,
+ &caches.shadow,
+ key,
+ SegmentCacheCategory::SortShadow,
+ bytes,
+ built,
+ );
}
}
}
@@ -24188,42 +25035,6 @@ fn staggered_per_shard_threshold(base_total: usize, shard_idx: usize, n_shards:
((base as f64) * (1.0 + frac * pos)).max(1.0) as usize
}
-/// Retained-memory budget for `stored_slices_cache`. Inserts stop once the
-/// budget is reached (queries then fall back to the per-query decompress
-/// path); merge eviction returns the dropped segment's bytes to the budget.
-/// 20% of host RAM, ≥2 GB floor (was a flat 3 GB, then a host-tuned 24 GB):
-/// a sustained bulk writer grows the corpus past a small flat budget DURING
-/// the read window; once the budget is full every over-budget segment is
-/// decompressed PER QUERY (insert stops, nothing evicts) — measured
-/// dec=28–30 s per match_all against a 21 M-doc corpus even fully quiesced,
-/// because with corpus-wide primary-key ties (cycled telemetry timestamps)
-/// the per-segment best-candidate rejection never strictly loses, so NO
-/// segment is skipped. Merge eviction returns merged-away segments' bytes.
-/// The durable fix — hydrating only GLOBAL winners after a shadow-merge of
-/// per-segment candidate keys — is tracked as follow-up.
-fn stored_slices_cache_budget() -> u64 {
- static BUDGET: std::sync::LazyLock<u64> = std::sync::LazyLock::new(|| {
- const FLOOR: u64 = 2 * 1024 * 1024 * 1024;
- let ram_total: Option<u64> = std::fs::read_to_string("/proc/meminfo").ok().and_then(|s| {
- s.lines()
- .find(|l| l.starts_with("MemTotal:"))
- .and_then(|l| {
- l.split_whitespace()
- .nth(1)
- .and_then(|kb| kb.parse::<u64>().ok())
- .map(|kb| kb * 1024)
- })
- });
- ram_total.map(|t| (t / 5).max(FLOOR)).unwrap_or(FLOOR)
- });
- *BUDGET
-}
-
-/// Budget for `decoded_stored_cache` (raw decompressed stored sections,
-/// ~1× the corpus JSON size). Overflow → inserts are refused and the
-/// query falls back to the per-query decompress (pre-cache behaviour).
-const DECODED_STORED_CACHE_BUDGET: u64 = 4 * 1024 * 1024 * 1024;
-
/// Entry cap for `shortcut_count_cache` (cleared wholesale when exceeded).
const SHORTCUT_COUNT_CACHE_MAX: usize = 65_536;
@@ -29194,6 +30005,7 @@ mod flush_memory_integration_tests {
let _ = release_rx.lock().take().unwrap().recv();
inject_error
}),
+ after_warm: None,
ledger,
target_memtable: Arc::as_ptr(&idx.memtable) as usize,
};
@@ -29223,6 +30035,85 @@ mod flush_memory_integration_tests {
);
}
+ async fn run_post_warm_failure_case(name: &'static str, panic_after_warm: bool) {
+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ engine.create_index(name, Schema::empty()).unwrap();
+ let idx = engine.get_index(name).unwrap();
+ let doc_id = format!("{name}-doc");
+ idx.index_document(Some(doc_id.clone()), json!({"body": "retained", "rank": 7}))
+ .await
+ .unwrap();
+
+ let shard = idx.memtable.shard_for_dynamic(&doc_id);
+ let (field_configs, excluded_fts_fields) = {
+ let schema = idx.schema.read().await;
+ (
+ build_fts_field_configs(&schema.schema),
+ crate::memtable::semantic_derived_vector_fields(&schema.schema),
+ )
+ };
+ let mut warm_caches = idx.publish_warm_caches();
+ let budget = SegmentHydrationBudget::new(16 * 1024 * 1024);
+ warm_caches.budget = Arc::clone(&budget);
+ let observed = warm_caches.clone();
+ let warmed_id = Arc::new(parking_lot::Mutex::new(None::<String>));
+ let callback_id = Arc::clone(&warmed_id);
+ let hook = FlushPublisherTestHook {
+ callback: Arc::new(|| false),
+ after_warm: Some(Arc::new(move |segment_id| {
+ *callback_id.lock() = Some(segment_id.to_string());
+ assert!(
+ observed.slices.contains_key(segment_id)
+ || observed.dv.contains_key(segment_id),
+ "real publisher did not warm a cache family"
+ );
+ if panic_after_warm {
+ panic!("injected post-warm panic");
+ }
+ true
+ })),
+ ledger,
+ target_memtable: Arc::as_ptr(&idx.memtable) as usize,
+ };
+
+ let result = do_flush_shard(
+ shard,
+ Arc::clone(&idx.store),
+ Arc::clone(&idx.memtable),
+ Arc::clone(&idx.registry),
+ idx.data_dir.clone(),
+ field_configs,
+ excluded_fts_fields,
+ || {},
+ warm_caches,
+ Some(hook),
+ )
+ .await;
+ if panic_after_warm {
+ assert!(result.is_ok(), "join panic is logged and kept non-fatal");
+ } else {
+ assert!(result.is_err());
+ }
+ let segment_id = warmed_id.lock().clone().expect("warm hook did not run");
+ assert!(!idx.stored_slices_cache.contains_key(&segment_id));
+ assert!(!idx.dv_cache.contains_key(&segment_id));
+ assert!(!idx.stored_value_cache.contains_key(&segment_id));
+ assert!(!idx.id_pos_cache.contains_key(&segment_id));
+ assert!(!idx.row_seq_cache.contains_key(&segment_id));
+ assert!(!idx.decoded_stored_cache.contains_key(&segment_id));
+ let prefix = format!("{segment_id}\u{1}");
+ assert!(!idx
+ .sort_shadow_cache
+ .iter()
+ .any(|entry| entry.key().starts_with(&prefix)));
+ assert_eq!(budget.snapshot().current, 0);
+ assert_eq!(budget.snapshot().accounting_errors, 0);
+ }
+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn parallel_real_flushes_route_isolated_ledgers_through_success_and_error() {
let (success, failure) = tokio::join!(
@@ -29231,4 +30122,10 @@ mod flush_memory_integration_tests {
);
assert_eq!((success, failure), ((), ()));
}
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+ async fn real_flush_cleans_warmed_owners_on_post_warm_error_and_panic() {
+ run_post_warm_failure_case("flush-post-warm-error", false).await;
+ run_post_warm_failure_case("flush-post-warm-panic", true).await;
+ }
}