Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(engine): add opt-in ingest memory attribution

buger43397f03344d
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 6f0e1b58c..59b1e5064 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -6452,6 +6452,7 @@ dependencies = [
"chrono",
"flate2",
"jemalloc_pprof",
+ "libc",
"memchr",
"memmap2",
"pprof",
@@ -6465,6 +6466,7 @@ dependencies = [
"serde",
"serde_json",
"tempfile",
+ "tikv-jemalloc-ctl",
"tikv-jemallocator",
"tokio",
"tonic",
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index aa9ee9f93..384502028 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -12130,6 +12130,13 @@ async fn process_bulk_body(
_started: Instant,
opts: xerj_engine::bulk::BulkOpts,
) -> axum::response::Response {
+ // The request body remains owned by this handler through parsing,
+ // grouping, and publication. The opt-in guard is a single branch and
+ // performs no atomic work when tracing is disabled.
+ let _http_body_memory = xerj_engine::ingest_memory::Retained::new(
+ xerj_engine::ingest_memory::Category::HttpBody,
+ body.len(),
+ );
let text = match std::str::from_utf8(body) {
Ok(t) => t,
Err(_) => {
@@ -12151,9 +12158,16 @@ async fn process_bulk_body(
// dimensions and the instant are identical, matching ES. Returns `None`
// when no action targets a TSDB index, so normal bulk bodies are unchanged.
let ts_rewritten = rewrite_bulk_time_series_ids(state, default_index, text);
+ let _ts_rewritten_memory = ts_rewritten.as_ref().map(|replacement| {
+ xerj_engine::ingest_memory::Retained::new(
+ xerj_engine::ingest_memory::Category::HttpRewriteBuffer,
+ replacement.capacity(),
+ )
+ });
let base_text: &str = ts_rewritten.as_deref().unwrap_or(text);
let rewritten_body;
+ let mut rewritten_capacity = None;
// P2.2 — the ignore_malformed rewrite parses + re-serialises EVERY
// source line serially (client-blocking). `apply_ignore_malformed`
// can only modify a doc when some field in the target mapping carries
@@ -12170,8 +12184,15 @@ async fn process_bulk_body(
base_text
} else {
rewritten_body = rewrite_bulk_ignore_malformed(state, default_index, base_text);
+ rewritten_capacity = Some(rewritten_body.capacity());
&rewritten_body
};
+ let _ignore_rewritten_memory = rewritten_capacity.map(|capacity| {
+ xerj_engine::ingest_memory::Retained::new(
+ xerj_engine::ingest_memory::Category::HttpRewriteBuffer,
+ capacity,
+ )
+ });
let result =
xerj_engine::bulk::process_bulk_with_opts(&state.engine, default_index, text_ref, opts)
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index 822b9f209..d3d1ca908 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -15,6 +15,103 @@ use crate::engine::Engine;
use crate::index::IndexResponse;
use crate::{EngineError, Result};
+#[cfg(test)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum SemanticLifecyclePoint {
+ Grouped,
+ Parsed,
+ Prepared,
+ EmbeddingInside,
+ EmbeddingItemInside,
+ BeforePublication,
+ PublicationInside,
+ AfterPublication,
+}
+
+#[cfg(test)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum SemanticLifecycleFault {
+ None,
+ Embedding,
+ Publication,
+}
+
+#[cfg(test)]
+type SemanticLifecycleHook = std::sync::Arc<
+ dyn Fn(&str, SemanticLifecyclePoint) -> SemanticLifecycleFault + Send + Sync + 'static,
+>;
+
+#[cfg(test)]
+tokio::task_local! {
+ static SEMANTIC_LIFECYCLE_HOOK: (String, SemanticLifecycleHook);
+ static DISABLED_SEMANTIC_COUNTERS: std::sync::Arc<DisabledSemanticCounters>;
+}
+
+#[cfg(test)]
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+struct DisabledSemanticInstrumentation {
+ schema_reads: usize,
+ guard_vectors: usize,
+ sizing_walks: usize,
+}
+
+#[cfg(test)]
+#[derive(Default)]
+struct DisabledSemanticCounters {
+ schema_reads: std::sync::atomic::AtomicUsize,
+ guard_vectors: std::sync::atomic::AtomicUsize,
+ sizing_walks: std::sync::atomic::AtomicUsize,
+}
+
+#[cfg(test)]
+impl DisabledSemanticCounters {
+ fn snapshot(&self) -> DisabledSemanticInstrumentation {
+ use std::sync::atomic::Ordering;
+ DisabledSemanticInstrumentation {
+ schema_reads: self.schema_reads.load(Ordering::Relaxed),
+ guard_vectors: self.guard_vectors.load(Ordering::Relaxed),
+ sizing_walks: self.sizing_walks.load(Ordering::Relaxed),
+ }
+ }
+}
+
+#[cfg(test)]
+fn count_disabled_semantic_access(
+ select: fn(&DisabledSemanticCounters) -> &std::sync::atomic::AtomicUsize,
+) {
+ use std::sync::atomic::Ordering;
+ let _ = DISABLED_SEMANTIC_COUNTERS.try_with(|counters| {
+ select(counters).fetch_add(1, Ordering::Relaxed);
+ });
+}
+
+#[cfg(test)]
+pub(crate) fn semantic_lifecycle_hook(
+ target: &str,
+ point: SemanticLifecyclePoint,
+) -> SemanticLifecycleFault {
+ SEMANTIC_LIFECYCLE_HOOK
+ .try_with(|(hook_target, hook)| {
+ if hook_target == "*" || hook_target == target {
+ hook(target, point)
+ } else {
+ SemanticLifecycleFault::None
+ }
+ })
+ .unwrap_or(SemanticLifecycleFault::None)
+}
+
+#[cfg(test)]
+async fn with_semantic_lifecycle_hook<F: std::future::Future>(
+ target: impl Into<String>,
+ hook: SemanticLifecycleHook,
+ future: F,
+) -> F::Output {
+ SEMANTIC_LIFECYCLE_HOOK
+ .scope((target.into(), hook), future)
+ .await
+}
+
// ── Types ─────────────────────────────────────────────────────────────────────
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -124,6 +221,30 @@ pub async fn process_bulk(engine: &Engine, default_index: Option<&str>, body: &s
process_bulk_with_opts(engine, default_index, body, BulkOpts::default()).await
}
+fn release_raw_owner(
+ owners: &mut std::collections::HashMap<usize, (usize, crate::ingest_memory::Retained<'static>)>,
+ source: std::sync::Arc<[u8]>,
+) {
+ let owner = std::sync::Arc::as_ptr(&source) as *const () as usize;
+ // The logical owner must outlive every Arc clone. Dropping the ledger
+ // token first would publish a false zero while this clone still retains
+ // the backing allocation.
+ drop(source);
+ release_raw_owner_by_id(owners, owner);
+}
+
+fn release_raw_owner_by_id(
+ owners: &mut std::collections::HashMap<usize, (usize, crate::ingest_memory::Retained<'static>)>,
+ owner: usize,
+) {
+ if let Some((remaining, _)) = owners.get_mut(&owner) {
+ *remaining -= 1;
+ if *remaining == 0 {
+ owners.remove(&owner);
+ }
+ }
+}
+
#[derive(Clone, Default)]
pub struct BulkOpts {
pub require_alias: Option<bool>,
@@ -1014,11 +1135,40 @@ pub async fn process_bulk_with_opts(
// still used after preparation; it observes the supplied companion vector
// and therefore preserves WAL, overwrite status, response ordering, and
// item-level errors.
+ let mut semantic_raw_memory = crate::ingest_memory::enabled().then(
+ std::collections::HashMap::<usize, (usize, crate::ingest_memory::Retained<'static>)>::new,
+ );
+ if let Some(raw_memory) = semantic_raw_memory.as_mut() {
+ for batch in semantic_index_batches.values() {
+ for (_, _, source) in batch {
+ let owner = std::sync::Arc::as_ptr(source) as *const () as usize;
+ raw_memory
+ .entry(owner)
+ .and_modify(|(remaining, _)| *remaining += 1)
+ .or_insert_with(|| {
+ (
+ 1usize,
+ crate::ingest_memory::Retained::new(
+ crate::ingest_memory::Category::RawSource,
+ source.len(),
+ ),
+ )
+ });
+ }
+ }
+ }
for (index_name, batch) in semantic_index_batches {
+ #[cfg(test)]
+ let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::Grouped);
let idx = match engine.get_or_create_index(&index_name) {
Ok(i) => i,
Err(e) => {
- for (item_idx, id, _) in batch {
+ for (item_idx, id, source) in batch {
+ if let Some(raw_memory) = semantic_raw_memory.as_mut() {
+ release_raw_owner(raw_memory, source);
+ } else {
+ drop(source);
+ }
items[item_idx] = Some(BulkItemResult {
action: "index".into(),
index: index_name.clone(),
@@ -1033,16 +1183,55 @@ pub async fn process_bulk_with_opts(
continue;
}
};
+ let vector_fields = if crate::ingest_memory::enabled() {
+ #[cfg(test)]
+ count_disabled_semantic_access(|counters| &counters.schema_reads);
+ idx.semantic_vector_fields().await
+ } else {
+ std::collections::HashSet::new()
+ };
let mut valid_meta = Vec::with_capacity(batch.len());
let mut valid_sources = Vec::with_capacity(batch.len());
+ let mut valid_memory = crate::ingest_memory::enabled().then(|| {
+ #[cfg(test)]
+ count_disabled_semantic_access(|counters| &counters.guard_vectors);
+ Vec::with_capacity(batch.len())
+ });
for (item_idx, id, bytes) in batch {
- match serde_json::from_slice::<Value>(&bytes) {
+ let parsed = serde_json::from_slice::<Value>(&bytes);
+ match parsed {
Ok(source) => {
+ if let Some(valid_memory) = valid_memory.as_mut() {
+ #[cfg(test)]
+ count_disabled_semantic_access(|counters| &counters.sizing_walks);
+ let parsed_document = crate::ingest_memory::Retained::new(
+ crate::ingest_memory::Category::ParsedSource,
+ crate::ingest_memory::estimated_json_heap_excluding(
+ &source,
+ &vector_fields,
+ ),
+ );
+ let parsed_vector = crate::ingest_memory::Retained::new(
+ crate::ingest_memory::Category::ParsedVector,
+ crate::ingest_memory::vector_value_capacity(&source, &vector_fields),
+ );
+ valid_memory.push((parsed_document, parsed_vector));
+ }
+ if let Some(raw_memory) = semantic_raw_memory.as_mut() {
+ release_raw_owner(raw_memory, bytes);
+ } else {
+ drop(bytes);
+ }
valid_meta.push((item_idx, id));
valid_sources.push(source);
}
Err(e) => {
+ if let Some(raw_memory) = semantic_raw_memory.as_mut() {
+ release_raw_owner(raw_memory, bytes);
+ } else {
+ drop(bytes);
+ }
items[item_idx] = Some(BulkItemResult {
action: "index".into(),
index: index_name.clone(),
@@ -1056,12 +1245,49 @@ pub async fn process_bulk_with_opts(
}
}
}
+ #[cfg(test)]
+ let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::Parsed);
+ // Backend-owned chunk strings, tokenizer tensors, and inference
+ // scratch are not yet logically attributed. Their effect remains
+ // visible only as allocator/RSS physical delta; it is not zero.
let prepared = idx.apply_semantic_embeddings_batch(valid_sources).await;
+ #[cfg(test)]
+ let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::Prepared);
+ let mut valid_memory = valid_memory.map(Vec::into_iter);
for ((item_idx, id), prepared_source) in valid_meta.into_iter().zip(prepared) {
- let source = match prepared_source {
- Ok(source) => source,
+ let memory = valid_memory.as_mut().and_then(Iterator::next);
+ let (source, prepared_memory) = match prepared_source {
+ Ok(source) => {
+ let prepared_memory = if let Some((parsed_document, parsed_vector)) = memory {
+ #[cfg(test)]
+ count_disabled_semantic_access(|counters| &counters.sizing_walks);
+ let prepared_document_bytes =
+ crate::ingest_memory::estimated_json_heap_excluding(
+ &source,
+ &vector_fields,
+ );
+ let prepared_vector_bytes =
+ crate::ingest_memory::vector_value_capacity(&source, &vector_fields);
+ Some(crate::ingest_memory::Retained::replace_pair(
+ parsed_document,
+ parsed_vector,
+ (
+ crate::ingest_memory::Category::PreparedDocument,
+ prepared_document_bytes,
+ ),
+ (
+ crate::ingest_memory::Category::PreparedVector,
+ prepared_vector_bytes,
+ ),
+ ))
+ } else {
+ None
+ };
+ (source, prepared_memory)
+ }
Err(e) => {
+ drop(memory);
let status = engine_error_http_status(&e);
items[item_idx] = Some(BulkItemResult {
action: "index".into(),
@@ -1077,7 +1303,13 @@ pub async fn process_bulk_with_opts(
}
};
let error_id = id.clone().unwrap_or_default();
- match idx.index_document_prepared(id, source).await {
+ #[cfg(test)]
+ let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::BeforePublication);
+ let publication = idx.index_document_prepared(id, source).await;
+ drop(prepared_memory);
+ #[cfg(test)]
+ let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::AfterPublication);
+ match publication {
Ok(resp) => {
let status = if resp.result == "updated" { 200 } else { 201 };
items[item_idx] = Some(BulkItemResult {
@@ -2269,3 +2501,700 @@ async fn execute_bulk_action(
)),
}
}
+
+#[cfg(test)]
+mod semantic_lifecycle_tests {
+ use super::*;
+ use crate::ingest_memory::{Category, Ledger, Measurement};
+ use serde_json::json;
+ use std::sync::{Arc, Mutex};
+ use tempfile::TempDir;
+ use xerj_common::config::Config;
+ use xerj_common::types::{EmbeddingConfig, FieldConfig, FieldType, Schema};
+
+ fn semantic_engine(name: &str) -> (TempDir, Engine) {
+ semantic_engine_with_targets(name, &[("content", "content_vector")])
+ }
+
+ fn semantic_engine_with_targets(name: &str, targets: &[(&str, &str)]) -> (TempDir, Engine) {
+ let dir = TempDir::new().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = Engine::new(config).unwrap();
+ let mut schema = Schema::empty();
+ for (field, target) in targets {
+ let mut config = FieldConfig::new(*field, FieldType::Text);
+ config.options.dimensions = Some(16);
+ config.embedding = Some(EmbeddingConfig {
+ endpoint: None,
+ model: None,
+ target_field: Some((*target).into()),
+ });
+ schema.fields.push(config);
+ }
+ engine.create_index(name, schema).unwrap();
+ let properties = targets
+ .iter()
+ .map(|(field, _)| ((*field).to_string(), json!({"type": "semantic_text"})))
+ .collect::<serde_json::Map<_, _>>();
+ engine.put_index_mapping(name, json!({"properties": properties}));
+ (dir, engine)
+ }
+
+ fn current(ledger: &Ledger, category: Category) -> u64 {
+ ledger.gauge(category, Measurement::Estimated).current
+ }
+
+ fn all_semantic_current(ledger: &Ledger) -> u64 {
+ [
+ Category::RawSource,
+ Category::ParsedSource,
+ Category::ParsedVector,
+ Category::PreparedDocument,
+ Category::PreparedVector,
+ ]
+ .into_iter()
+ .map(|category| current(ledger, category))
+ .sum()
+ }
+
+ #[test]
+ fn raw_owner_token_outlives_every_arc_occurrence() {
+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));
+ let source: Arc<[u8]> = Arc::from(&b"{\"content\":\"retained\"}"[..]);
+ let owner = Arc::as_ptr(&source) as *const () as usize;
+ let mut owners = std::collections::HashMap::from([(
+ owner,
+ (
+ 2,
+ crate::ingest_memory::Retained::for_ledger(
+ ledger,
+ Category::RawSource,
+ source.len(),
+ ),
+ ),
+ )]);
+ let second = Arc::clone(&source);
+
+ release_raw_owner(&mut owners, second);
+ assert_eq!(Arc::strong_count(&source), 1);
+ assert_eq!(owners.get(&owner).map(|entry| entry.0), Some(1));
+ assert_eq!(current(ledger, Category::RawSource), source.len() as u64);
+
+ release_raw_owner(&mut owners, source);
+ assert!(owners.is_empty());
+ assert_eq!(current(ledger, Category::RawSource), 0);
+ }
+
+ async fn run_case(
+ name: &str,
+ fault: SemanticLifecycleFault,
+ ) -> (
+ BulkResult,
+ &'static Ledger,
+ Vec<(SemanticLifecyclePoint, [u64; 5])>,
+ ) {
+ let (_dir, engine) = semantic_engine_with_targets(
+ name,
+ &[("content", "content_vector"), ("summary", "summary_vector")],
+ );
+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));
+ // Zero-capacity rendezvous channels make every hook a genuine phase
+ // barrier: the ingest task cannot advance until the observer has read
+ // the live gauges and explicitly released it.
+ let expected = if fault == SemanticLifecycleFault::Embedding {
+ 4
+ } else {
+ 8
+ };
+ let (observation_tx, observation_rx) = std::sync::mpsc::sync_channel(0);
+ let (resume_tx, resume_rx) = std::sync::mpsc::sync_channel(0);
+ let resume_rx = Arc::new(Mutex::new(resume_rx));
+ let observer = std::thread::spawn(move || {
+ let mut points = Vec::with_capacity(expected);
+ for _ in 0..expected {
+ points.push(observation_rx.recv().unwrap());
+ resume_tx.send(()).unwrap();
+ }
+ points
+ });
+ let hook: SemanticLifecycleHook = Arc::new(move |_, point| {
+ observation_tx
+ .send((
+ point,
+ [
+ current(ledger, Category::RawSource),
+ current(ledger, Category::ParsedSource),
+ current(ledger, Category::ParsedVector),
+ current(ledger, Category::PreparedDocument),
+ current(ledger, Category::PreparedVector),
+ ],
+ ))
+ .unwrap();
+ resume_rx.lock().unwrap().recv().unwrap();
+ match (fault, point) {
+ (SemanticLifecycleFault::Embedding, SemanticLifecyclePoint::EmbeddingInside) => {
+ fault
+ }
+ (
+ SemanticLifecycleFault::Publication,
+ SemanticLifecyclePoint::PublicationInside,
+ ) => fault,
+ _ => SemanticLifecycleFault::None,
+ }
+ });
+ let body = format!(
+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"one\"}}}}\n\
+ {{\"content\":\"caller supplied vector\",\
+ \"content_vector\":[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,\
+ 0.9,1.0,1.1,1.2,1.3,1.4,1.5,1.6],\
+ \"summary\":\"a genuine semantic lifecycle embedding job\"}}\n"
+ );
+ let result = crate::ingest_memory::with_test_ledger(
+ ledger,
+ with_semantic_lifecycle_hook(name, hook, process_bulk(&engine, None, &body)),
+ )
+ .await;
+ let points = observer.join().unwrap();
+ (result, ledger, points)
+ }
+
+ #[tokio::test]
+ async fn semantic_lifecycle_success_balances_isolated_ledger() {
+ let (result, ledger, points) =
+ run_case("semantic-lifecycle-success", SemanticLifecycleFault::None).await;
+ assert!(!result.errors, "{:?}", result.items);
+ let phase_points: Vec<_> = points.iter().map(|(point, _)| *point).collect();
+ assert_eq!(
+ phase_points,
+ [
+ SemanticLifecyclePoint::Grouped,
+ SemanticLifecyclePoint::Parsed,
+ SemanticLifecyclePoint::EmbeddingInside,
+ SemanticLifecyclePoint::EmbeddingItemInside,
+ SemanticLifecyclePoint::Prepared,
+ SemanticLifecyclePoint::BeforePublication,
+ SemanticLifecyclePoint::PublicationInside,
+ SemanticLifecyclePoint::AfterPublication,
+ ]
+ );
+ let at = |point| points.iter().find(|(p, _)| *p == point).unwrap().1;
+ assert!(at(SemanticLifecyclePoint::Grouped)[0] > 0);
+ assert_eq!(at(SemanticLifecyclePoint::Parsed)[0], 0);
+ assert!(at(SemanticLifecyclePoint::Parsed)[1] > 0);
+ assert!(at(SemanticLifecyclePoint::EmbeddingInside)[1] > 0);
+ assert!(
+ at(SemanticLifecyclePoint::EmbeddingInside)[2] > 0,
+ "caller-supplied vector ownership must remain live while embedding is blocked"
+ );
+ assert_eq!(at(SemanticLifecyclePoint::BeforePublication)[1], 0);
+ assert!(at(SemanticLifecyclePoint::BeforePublication)[3] > 0);
+ assert!(at(SemanticLifecyclePoint::BeforePublication)[4] > 0);
+ assert_eq!(at(SemanticLifecyclePoint::AfterPublication), [0; 5]);
+ assert!(current(ledger, Category::RawSource) == 0);
+ assert!(
+ ledger
+ .gauge(Category::RawSource, Measurement::SerializedSize)
+ .peak
+ > 0
+ );
+ assert!(
+ ledger
+ .gauge(Category::ParsedSource, Measurement::Estimated)
+ .peak
+ > 0
+ );
+ assert!(
+ ledger
+ .gauge(Category::PreparedDocument, Measurement::Estimated)
+ .peak
+ > 0
+ );
+ assert!(
+ ledger
+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)
+ .peak
+ > 0
+ );
+ assert_eq!(all_semantic_current(ledger), 0);
+ assert_eq!(ledger.accounting_errors(), 0);
+ assert_eq!(all_semantic_current(&crate::ingest_memory::LEDGER), 0);
+ }
+
+ #[tokio::test]
+ async fn semantic_lifecycle_embedding_error_releases_parsed_ownership() {
+ let (result, ledger, points) = run_case(
+ "semantic-lifecycle-embed-error",
+ SemanticLifecycleFault::Embedding,
+ )
+ .await;
+ assert!(result.errors);
+ assert_eq!(result.items[0].status, 400);
+ let phase_points: Vec<_> = points.iter().map(|(point, _)| *point).collect();
+ assert_eq!(
+ phase_points,
+ [
+ SemanticLifecyclePoint::Grouped,
+ SemanticLifecyclePoint::Parsed,
+ SemanticLifecyclePoint::EmbeddingInside,
+ SemanticLifecyclePoint::Prepared,
+ ]
+ );
+ let inside = points
+ .iter()
+ .find(|(point, _)| *point == SemanticLifecyclePoint::EmbeddingInside)
+ .unwrap()
+ .1;
+ assert!(inside[1] > 0);
+ assert!(
+ inside[2] > 0,
+ "caller-supplied vector ownership must remain live while embedding is blocked"
+ );
+ assert!(
+ ledger
+ .gauge(Category::ParsedSource, Measurement::Estimated)
+ .peak
+ > 0
+ );
+ assert_eq!(all_semantic_current(ledger), 0);
+ assert_eq!(ledger.accounting_errors(), 0);
+ }
+
+ #[tokio::test]
+ async fn semantic_lifecycle_publication_error_releases_prepared_ownership() {
+ let (result, ledger, points) = run_case(
+ "semantic-lifecycle-publication-error",
+ SemanticLifecycleFault::Publication,
+ )
+ .await;
+ assert!(result.errors);
+ assert_eq!(result.items[0].status, 400);
+ let phase_points: Vec<_> = points.iter().map(|(point, _)| *point).collect();
+ assert_eq!(
+ phase_points,
+ [
+ SemanticLifecyclePoint::Grouped,
+ SemanticLifecyclePoint::Parsed,
+ SemanticLifecyclePoint::EmbeddingInside,
+ SemanticLifecyclePoint::EmbeddingItemInside,
+ SemanticLifecyclePoint::Prepared,
+ SemanticLifecyclePoint::BeforePublication,
+ SemanticLifecyclePoint::PublicationInside,
+ SemanticLifecyclePoint::AfterPublication,
+ ]
+ );
+ let inside = points
+ .iter()
+ .find(|(point, _)| *point == SemanticLifecyclePoint::PublicationInside)
+ .unwrap()
+ .1;
+ assert!(inside[3] > 0);
+ assert!(inside[4] > 0);
+ assert!(
+ ledger
+ .gauge(Category::PreparedDocument, Measurement::Estimated)
+ .peak
+ > 0
+ );
+ assert!(
+ ledger
+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)
+ .peak
+ > 0
+ );
+ assert_eq!(all_semantic_current(ledger), 0);
+ assert_eq!(ledger.accounting_errors(), 0);
+ }
+
+ #[tokio::test]
+ async fn semantic_lifecycle_hook_scope_is_panic_safe_and_target_specific() {
+ let (_dir, engine) = semantic_engine("semantic-lifecycle-panic");
+ let engine = Arc::new(engine);
+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));
+ let hook: SemanticLifecycleHook = Arc::new(|_, point| {
+ if point == SemanticLifecyclePoint::Grouped {
+ panic!("deliberate lifecycle hook panic");
+ }
+ SemanticLifecycleFault::None
+ });
+ let body = concat!(
+ "{\"index\":{\"_index\":\"semantic-lifecycle-panic\",\"_id\":\"one\"}}\n",
+ "{\"content\":\"panic cleanup\"}\n",
+ );
+ let panic_engine = Arc::clone(&engine);
+ let task = tokio::spawn(async move {
+ crate::ingest_memory::with_test_ledger(
+ ledger,
+ with_semantic_lifecycle_hook(
+ "semantic-lifecycle-panic",
+ hook,
+ process_bulk(&panic_engine, None, body),
+ ),
+ )
+ .await
+ });
+ assert!(task.await.unwrap_err().is_panic());
+ assert_eq!(all_semantic_current(ledger), 0);
+
+ let other_ledger = Box::leak(Box::new(Ledger::new()));
+ let rejecting_hook: SemanticLifecycleHook =
+ Arc::new(|_, _| SemanticLifecycleFault::Publication);
+ let result = crate::ingest_memory::with_test_ledger(
+ other_ledger,
+ with_semantic_lifecycle_hook(
+ "different-target",
+ rejecting_hook,
+ process_bulk(&engine, None, body),
+ ),
+ )
+ .await;
+ assert!(
+ !result.errors,
+ "target-specific hook leaked onto another target"
+ );
+ assert_eq!(all_semantic_current(other_ledger), 0);
+ assert_eq!(all_semantic_current(&crate::ingest_memory::LEDGER), 0);
+ }
+
+ #[tokio::test]
+ async fn semantic_lifecycle_declines_after_each_sequential_publication() {
+ let name = "semantic-lifecycle-sequential";
+ let (_dir, engine) = semantic_engine(name);
+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));
+ let observations = Arc::new(Mutex::new(Vec::<(SemanticLifecyclePoint, u64, u64)>::new()));
+ let captured = Arc::clone(&observations);
+ let embedded = Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let embedded_hook = Arc::clone(&embedded);
+ let hook: SemanticLifecycleHook = Arc::new(move |_, point| {
+ if point == SemanticLifecyclePoint::EmbeddingInside {
+ embedded_hook.store(true, std::sync::atomic::Ordering::Relaxed);
+ }
+ if matches!(
+ point,
+ SemanticLifecyclePoint::BeforePublication
+ | SemanticLifecyclePoint::AfterPublication
+ ) {
+ captured.lock().unwrap().push((
+ point,
+ current(ledger, Category::PreparedDocument),
+ current(ledger, Category::PreparedVector),
+ ));
+ }
+ SemanticLifecycleFault::None
+ });
+ let body = (0..3)
+ .map(|i| {
+ format!(
+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"{i}\"}}}}\n\
+ {{\"content\":\"semantic document number {i}\"}}\n"
+ )
+ })
+ .collect::<String>();
+ let result = crate::ingest_memory::with_test_ledger(
+ ledger,
+ with_semantic_lifecycle_hook(name, hook, process_bulk(&engine, None, &body)),
+ )
+ .await;
+ assert!(!result.errors, "{:?}", result.items);
+ assert!(embedded.load(std::sync::atomic::Ordering::Relaxed));
+ let observations = observations.lock().unwrap();
+ assert_eq!(observations.len(), 6);
+ for pair in observations.chunks_exact(2) {
+ assert_eq!(pair[0].0, SemanticLifecyclePoint::BeforePublication);
+ assert_eq!(pair[1].0, SemanticLifecyclePoint::AfterPublication);
+ assert!(pair[0].1 > pair[1].1);
+ assert!(pair[0].2 > pair[1].2);
+ }
+ for pair in observations.chunks_exact(2) {
+ assert!(pair[0].1 > 0);
+ assert!(pair[0].2 > 0);
+ assert_eq!(pair[1].1, 0, "published document ownership stayed charged");
+ assert_eq!(pair[1].2, 0, "published vector ownership stayed charged");
+ }
+ assert_eq!(observations.last().unwrap().1, 0);
+ assert_eq!(observations.last().unwrap().2, 0);
+ assert_eq!(all_semantic_current(ledger), 0);
+ assert_eq!(ledger.accounting_errors(), 0);
+ }
+
+ #[tokio::test]
+ async fn semantic_lifecycle_mixed_publication_outputs_balance_per_item() {
+ let name = "semantic-lifecycle-mixed";
+ let (_dir, engine) = semantic_engine(name);
+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));
+ let publication = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let publication_hook = Arc::clone(&publication);
+ let points = Arc::new(Mutex::new(Vec::new()));
+ let captured_points = Arc::clone(&points);
+ let hook: SemanticLifecycleHook = Arc::new(move |_, point| {
+ captured_points.lock().unwrap().push(point);
+ if point == SemanticLifecyclePoint::PublicationInside
+ && publication_hook.fetch_add(1, std::sync::atomic::Ordering::Relaxed) == 1
+ {
+ SemanticLifecycleFault::Publication
+ } else {
+ SemanticLifecycleFault::None
+ }
+ });
+ let body = (0..3)
+ .map(|i| {
+ format!(
+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"{i}\"}}}}\n\
+ {{\"content\":\"mixed output {i}\"}}\n"
+ )
+ })
+ .collect::<String>();
+ let result = crate::ingest_memory::with_test_ledger(
+ ledger,
+ with_semantic_lifecycle_hook(name, hook, process_bulk(&engine, None, &body)),
+ )
+ .await;
+ assert!(result.errors);
+ assert_eq!(
+ result
+ .items
+ .iter()
+ .map(|item| item.status)
+ .collect::<Vec<_>>(),
+ [201, 400, 201]
+ );
+ let points = points.lock().unwrap();
+ assert!(points.contains(&SemanticLifecyclePoint::EmbeddingInside));
+ assert_eq!(
+ points
+ .iter()
+ .filter(|point| **point == SemanticLifecyclePoint::PublicationInside)
+ .count(),
+ 3
+ );
+ assert_eq!(all_semantic_current(ledger), 0);
+ assert_eq!(ledger.accounting_errors(), 0);
+ }
+
+ #[tokio::test]
+ async fn semantic_lifecycle_mixed_embedding_outputs_balance_per_item() {
+ let name = "semantic-lifecycle-mixed-embedding";
+ let (_dir, engine) = semantic_engine(name);
+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));
+ let item = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let item_hook = Arc::clone(&item);
+ let points = Arc::new(Mutex::new(Vec::new()));
+ let captured_points = Arc::clone(&points);
+ let hook: SemanticLifecycleHook = Arc::new(move |_, point| {
+ captured_points.lock().unwrap().push((
+ point,
+ current(ledger, Category::ParsedSource),
+ current(ledger, Category::ParsedVector),
+ ));
+ if point == SemanticLifecyclePoint::EmbeddingItemInside
+ && item_hook.fetch_add(1, std::sync::atomic::Ordering::Relaxed) == 1
+ {
+ SemanticLifecycleFault::Embedding
+ } else {
+ SemanticLifecycleFault::None
+ }
+ });
+ let body = (0..3)
+ .map(|i| {
+ format!(
+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"{i}\"}}}}\n\
+ {{\"content\":\"mixed embedding output {i}\"}}\n"
+ )
+ })
+ .collect::<String>();
+ let result = crate::ingest_memory::with_test_ledger(
+ ledger,
+ with_semantic_lifecycle_hook(name, hook, process_bulk(&engine, None, &body)),
+ )
+ .await;
+ assert!(result.errors);
+ assert_eq!(
+ result
+ .items
+ .iter()
+ .map(|item| item.status)
+ .collect::<Vec<_>>(),
+ [201, 500, 201]
+ );
+ let points = points.lock().unwrap();
+ assert!(points.iter().any(|(point, parsed, _)| *point
+ == SemanticLifecyclePoint::EmbeddingInside
+ && *parsed > 0));
+ assert_eq!(
+ points
+ .iter()
+ .filter(|(point, _, _)| *point == SemanticLifecyclePoint::EmbeddingItemInside)
+ .count(),
+ 3
+ );
+ assert_eq!(
+ points
+ .iter()
+ .filter(|(point, _, _)| *point == SemanticLifecyclePoint::PublicationInside)
+ .count(),
+ 2
+ );
+ assert_eq!(all_semantic_current(ledger), 0);
+ assert_eq!(ledger.accounting_errors(), 0);
+ }
+
+ #[tokio::test]
+ async fn semantic_lifecycle_covers_malformed_caller_vectors_and_literal_dotted_target() {
+ let name = "semantic-lifecycle-input-shapes";
+ let (_dir, engine) =
+ semantic_engine_with_targets(name, &[("content", "custom.vector"), ("summary", "v2")]);
+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));
+ let points = Arc::new(Mutex::new(Vec::new()));
+ let captured_points = Arc::clone(&points);
+ let hook: SemanticLifecycleHook = Arc::new(move |_, point| {
+ captured_points.lock().unwrap().push(point);
+ SemanticLifecycleFault::None
+ });
+ let body = format!(
+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"bad\"}}}}\n\
+ {{\"content\":\n\
+ {{\"index\":{{\"_index\":\"{name}\",\"_id\":\"supplied\"}}}}\n\
+ {{\"content\":\"caller vector\",\"custom.vector\":[0.1,0.2,0.3],\
+ \"summary\":\"second target\",\"v2\":[0.4,0.5]}}\n\
+ {{\"index\":{{\"_index\":\"{name}\",\"_id\":\"derived\"}}}}\n\
+ {{\"content\":\"derive literal dotted target\",\"summary\":\"derive second\"}}\n"
+ );
+ let result = crate::ingest_memory::with_test_ledger(
+ ledger,
+ with_semantic_lifecycle_hook(name, hook, process_bulk(&engine, None, &body)),
+ )
+ .await;
+ assert!(result.errors);
+ assert_eq!(
+ result
+ .items
+ .iter()
+ .map(|item| item.status)
+ .collect::<Vec<_>>(),
+ [400, 201, 201]
+ );
+ assert!(
+ ledger
+ .gauge(Category::ParsedVector, Measurement::ExactCapacity)
+ .peak
+ > 0
+ );
+ let points = points.lock().unwrap();
+ assert!(points.contains(&SemanticLifecyclePoint::EmbeddingInside));
+ assert_eq!(
+ points
+ .iter()
+ .filter(|point| **point == SemanticLifecyclePoint::PublicationInside)
+ .count(),
+ 2
+ );
+ assert!(
+ ledger
+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)
+ .peak
+ > 0
+ );
+ assert_eq!(all_semantic_current(ledger), 0);
+ assert_eq!(ledger.accounting_errors(), 0);
+ }
+
+ #[tokio::test]
+ async fn semantic_lifecycle_multiple_targets_and_index_creation_failure_balance() {
+ let ok = "semantic-lifecycle-target-a";
+ let fails = "semantic-lifecycle-target-b";
+ let (dir, engine) = semantic_engine(ok);
+ let mut schema = Schema::empty();
+ let mut content = FieldConfig::new("content", FieldType::Text);
+ content.options.dimensions = Some(16);
+ content.embedding = Some(EmbeddingConfig {
+ endpoint: None,
+ model: None,
+ target_field: Some("content_vector".into()),
+ });
+ schema.fields.push(content);
+ engine.create_index(fails, schema).unwrap();
+ engine.put_index_mapping(
+ fails,
+ json!({"properties": {"content": {"type": "semantic_text"}}}),
+ );
+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));
+ let engine = Arc::new(engine);
+ let hook_engine = Arc::clone(&engine);
+ let collision = dir.path().join(fails);
+ let points = Arc::new(Mutex::new(Vec::new()));
+ let captured_points = Arc::clone(&points);
+ let hook: SemanticLifecycleHook = Arc::new(move |target, point| {
+ captured_points
+ .lock()
+ .unwrap()
+ .push((target.to_string(), point));
+ if target == fails && point == SemanticLifecyclePoint::Grouped {
+ let engine = Arc::clone(&hook_engine);
+ std::thread::spawn(move || {
+ tokio::runtime::Runtime::new()
+ .unwrap()
+ .block_on(engine.delete_index(fails))
+ .unwrap();
+ })
+ .join()
+ .unwrap();
+ std::fs::write(&collision, b"blocks index directory creation").unwrap();
+ }
+ SemanticLifecycleFault::None
+ });
+ let body = format!(
+ "{{\"index\":{{\"_index\":\"{ok}\",\"_id\":\"ok\"}}}}\n\
+ {{\"content\":\"first target\"}}\n\
+ {{\"index\":{{\"_index\":\"{fails}\",\"_id\":\"fail\"}}}}\n\
+ {{\"content\":\"second target\"}}\n"
+ );
+ let result = crate::ingest_memory::with_test_ledger(
+ ledger,
+ with_semantic_lifecycle_hook("*", hook, process_bulk(&engine, None, &body)),
+ )
+ .await;
+ assert!(result.errors);
+ assert_eq!(
+ result
+ .items
+ .iter()
+ .map(|item| item.status)
+ .collect::<Vec<_>>(),
+ [201, 500]
+ );
+ let points = points.lock().unwrap();
+ assert!(points.contains(&(ok.to_string(), SemanticLifecyclePoint::EmbeddingInside)));
+ assert!(points.contains(&(ok.to_string(), SemanticLifecyclePoint::PublicationInside)));
+ assert!(points.contains(&(fails.to_string(), SemanticLifecyclePoint::Grouped)));
+ assert!(!points.contains(&(fails.to_string(), SemanticLifecyclePoint::EmbeddingInside)));
+ assert_eq!(all_semantic_current(ledger), 0);
+ assert_eq!(ledger.accounting_errors(), 0);
+ }
+
+ #[tokio::test]
+ async fn tracing_off_semantic_path_skips_all_instrumentation_structures() {
+ assert_eq!(
+ crate::ingest_memory::config().mode,
+ crate::ingest_memory::TraceMode::Off,
+ "this structural proof requires tracing to be off; it must not silently skip"
+ );
+ let name = "semantic-lifecycle-disabled";
+ let (_dir, engine) = semantic_engine(name);
+ let counters = Arc::new(DisabledSemanticCounters::default());
+ let body = format!(
+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"one\"}}}}\n\
+ {{\"content\":\"disabled structural proof\"}}\n"
+ );
+ let result = DISABLED_SEMANTIC_COUNTERS
+ .scope(Arc::clone(&counters), process_bulk(&engine, None, &body))
+ .await;
+ assert!(!result.errors, "{:?}", result.items);
+ assert_eq!(
+ counters.snapshot(),
+ DisabledSemanticInstrumentation::default(),
+ "tracing-off path read schema, allocated its guard vector, or walked JSON for sizing"
+ );
+ assert_eq!(all_semantic_current(&crate::ingest_memory::LEDGER), 0);
+ assert_eq!(crate::ingest_memory::LEDGER.accounting_errors(), 0);
+ }
+}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 2674c13e4..e256fc68e 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -2062,6 +2062,18 @@ impl Index {
id: Option<String>,
source: Value,
) -> Result<IndexResponse> {
+ #[cfg(test)]
+ if crate::bulk::semantic_lifecycle_hook(
+ self.name.as_str(),
+ crate::bulk::SemanticLifecyclePoint::PublicationInside,
+ ) == crate::bulk::SemanticLifecycleFault::Publication
+ {
+ return Err(crate::EngineError::Common(
+ xerj_common::XerjError::invalid_query(
+ "injected semantic publication failure inside index_document_prepared",
+ ),
+ ));
+ }
self.index_document_with_version_inner(id, source, None, None, true)
.await
}
@@ -3276,6 +3288,8 @@ impl Index {
excluded_fts_fields,
on_drained,
warm_caches,
+ #[cfg(test)]
+ None,
)
.await;
// Fallback: if do_flush_shard aborted before on_drained fired
@@ -3502,6 +3516,8 @@ impl Index {
excluded_fts_fields.clone(),
|| {}, // serial refresh path — no permit to drop early
self.publish_warm_caches(),
+ #[cfg(test)]
+ None,
)
.await?;
}
@@ -3634,6 +3650,8 @@ impl Index {
excluded_fts_fields,
on_drained,
warm_caches,
+ #[cfg(test)]
+ None,
)
.await;
if let Ok(mut guard) = permit_cell.lock() {
@@ -6272,6 +6290,24 @@ impl Index {
})
.collect()
};
+ #[cfg(test)]
+ if crate::bulk::semantic_lifecycle_hook(
+ self.name.as_str(),
+ crate::bulk::SemanticLifecyclePoint::EmbeddingInside,
+ ) == crate::bulk::SemanticLifecycleFault::Embedding
+ {
+ return sources
+ .into_iter()
+ .map(|_| {
+ Err(crate::EngineError::Common(
+ xerj_common::XerjError::invalid_query(
+ "injected semantic embedding failure inside \
+ apply_semantic_embeddings_batch",
+ ),
+ ))
+ })
+ .collect();
+ }
if specs.is_empty() {
return sources.into_iter().map(Ok).collect();
}
@@ -6292,6 +6328,16 @@ impl Index {
.mode
.eq_ignore_ascii_case("onnx-experimental");
for (source_idx, source) in sources.iter().enumerate() {
+ #[cfg(test)]
+ if crate::bulk::semantic_lifecycle_hook(
+ self.name.as_str(),
+ crate::bulk::SemanticLifecyclePoint::EmbeddingItemInside,
+ ) == crate::bulk::SemanticLifecycleFault::Embedding
+ {
+ pre_failures[source_idx] =
+ Some("injected per-item semantic embedding failure".to_string());
+ continue;
+ }
let Some(obj) = source.as_object() else {
continue;
};
@@ -6496,6 +6542,11 @@ impl Index {
.collect()
}
+ pub(crate) async fn semantic_vector_fields(&self) -> std::collections::HashSet<String> {
+ let schema = self.schema.read().await;
+ crate::memtable::semantic_derived_vector_fields(&schema.schema)
+ }
+
/// Hybrid (multi-query + fusion) executor. Recursively runs each
/// sub-query as its own search and combines the result lists via
/// the chosen fusion strategy (RRF / Linear / Learned). Pagination
@@ -11486,6 +11537,10 @@ impl Index {
// bounds global flush concurrency against ingest-time
// background flushes on other indices.
let n_shards = self.memtable.shard_count();
+ #[cfg(test)]
+ let flush_test_hook = FLUSH_PUBLISHER_TEST_HOOK
+ .try_with(FlushPublisherTestHook::clone)
+ .ok();
let mut shard_futures = Vec::with_capacity(n_shards);
for shard_idx in 0..n_shards {
let sema = Arc::clone(&self.flush_sema);
@@ -11496,6 +11551,8 @@ impl Index {
let field_configs = field_configs.clone();
let excluded_fts_fields = excluded_fts_fields.clone();
let warm_caches = self.publish_warm_caches();
+ #[cfg(test)]
+ let test_hook = flush_test_hook.clone();
shard_futures.push(tokio::spawn(async move {
let permit = sema.acquire_owned().await.ok();
let permit_cell = Arc::new(std::sync::Mutex::new(permit));
@@ -11515,6 +11572,8 @@ impl Index {
excluded_fts_fields,
on_drained,
warm_caches,
+ #[cfg(test)]
+ test_hook,
)
.await;
// Defensive: in case on_drained didn't fire.
@@ -16822,6 +16881,18 @@ impl Drop for FlushDurationTimer {
}
#[allow(clippy::too_many_arguments)] // free fn threading the full flush-pipeline dependency set
+#[cfg(test)]
+#[derive(Clone)]
+struct FlushPublisherTestHook {
+ callback: Arc<dyn Fn() -> bool + Send + Sync>,
+ ledger: &'static crate::ingest_memory::Ledger,
+ target_memtable: usize,
+}
+#[cfg(test)]
+tokio::task_local! {
+ static FLUSH_PUBLISHER_TEST_HOOK: FlushPublisherTestHook;
+}
+
async fn do_flush_shard(
shard_idx: usize,
store: Arc<IndexStore>,
@@ -16835,6 +16906,7 @@ async fn do_flush_shard(
// read-path caches, threaded through because this is a free fn without
// access to `Index`.
warm_caches: PublishWarmCaches,
+ #[cfg(test)] test_hook: Option<FlushPublisherTestHook>,
) -> Result<()> {
// V4 M4.5: no outer flush_lock — concurrent flushes are allowed. The
// memtable write lock below is the only atomicity point we need for
@@ -16857,6 +16929,8 @@ async fn do_flush_shard(
// the row positions in the stored section.
// Peek BEFORE drain to determine if docs came from raw-bytes or FTS insert.
let is_raw_bytes_path = memtable.peek_shard_has_raw_bytes(shard_idx);
+ #[cfg(test)]
+ let test_target = Arc::as_ptr(&memtable) as usize;
// THROWAWAY prof: per-flush finalize breakdown, gated on XERJ_PROF.
let prof = std::env::var_os("XERJ_PROF").is_some();
@@ -16884,13 +16958,22 @@ async fn do_flush_shard(
std::sync::Arc<serde_json::Value>,
)>,
xerj_storage::index_store::DrainedMemtable,
+ crate::ingest_memory::Retained<'static>,
)> = {
let t_drain = std::time::Instant::now();
- let raw = if is_raw_bytes_path {
- memtable.drain_shard_raw(shard_idx)
+ #[cfg(test)]
+ let test_ledger = test_hook
+ .as_ref()
+ .filter(|hook| hook.target_memtable == test_target)
+ .map(|hook| hook.ledger);
+ #[cfg(test)]
+ let (raw, _drained_memory) = if let Some(ledger) = test_ledger {
+ memtable.drain_shard_accounted_for_test(shard_idx, is_raw_bytes_path, ledger)
} else {
- memtable.drain_shard(shard_idx)
+ memtable.drain_shard_accounted(shard_idx, is_raw_bytes_path)
};
+ #[cfg(not(test))]
+ let (raw, _drained_memory) = memtable.drain_shard_accounted(shard_idx, is_raw_bytes_path);
prof_drain_us = t_drain.elapsed().as_micros();
let t_prep = std::time::Instant::now();
let _ = &t_prep;
@@ -16969,7 +17052,7 @@ async fn do_flush_shard(
entries: storage_entries,
};
prof_prep_us = t_prep.elapsed().as_micros();
- Some((drained_fts, storage_drained))
+ Some((drained_fts, storage_drained, _drained_memory))
}
};
@@ -16986,7 +17069,7 @@ async fn do_flush_shard(
// tmpfs.
on_drained();
- let (drained_fts, storage_drained) = match drained_opt {
+ let (drained_fts, storage_drained, _drained_memory) = match drained_opt {
Some(pair) => pair,
None => return Ok(()),
};
@@ -17019,6 +17102,17 @@ async fn do_flush_shard(
// 50-100 ms segment write that follows, and 100K-doc/s ingest
// bench numbers are unaffected.
let build_fts = move |meta: &xerj_storage::segment::SegmentMeta| -> xerj_storage::Result<()> {
+ #[cfg(test)]
+ let test_callback = test_hook
+ .as_ref()
+ .filter(|hook| hook.target_memtable == test_target)
+ .map(|hook| Arc::clone(&hook.callback));
+ #[cfg(test)]
+ if let Some(callback) = test_callback {
+ if callback() {
+ return Err(std::io::Error::other("injected flush publisher failure").into());
+ }
+ }
// Whole side-car build runs on the dedicated ingest pool so a
// flush burst can't queue search/agg par_iters behind it on the
// global rayon pool (read-under-write; see `crate::ingest_pool`).
@@ -29053,3 +29147,85 @@ mod write_publication_integration_tests {
assert_eq!(aggs["tags"]["buckets"][0]["key"], "current");
}
}
+
+#[cfg(test)]
+mod flush_memory_integration_tests {
+ use super::*;
+ use crate::ingest_memory::{Category, Ledger, Measurement};
+ use serde_json::json;
+ struct PublisherRelease(Option<std::sync::mpsc::SyncSender<()>>);
+ impl PublisherRelease {
+ fn release(&mut self) {
+ if let Some(tx) = self.0.take() {
+ let _ = tx.send(());
+ }
+ }
+ }
+ impl Drop for PublisherRelease {
+ fn drop(&mut self) {
+ self.release();
+ }
+ }
+
+ async fn run_flush_memory_case(name: &'static str, inject_error: bool) {
+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ engine.create_index(name, Schema::empty()).unwrap();
+ let idx = engine.get_index(name).unwrap();
+ idx.index_document(Some("doc".into()), json!({"body": "retained"}))
+ .await
+ .unwrap();
+ assert!(idx.memtable_bytes() > 0);
+
+ let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
+ let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1);
+ let mut publisher_release = PublisherRelease(Some(release_tx));
+ let entered_tx = parking_lot::Mutex::new(Some(entered_tx));
+ let release_rx = parking_lot::Mutex::new(Some(release_rx));
+ let hook = FlushPublisherTestHook {
+ callback: Arc::new(move || {
+ entered_tx.lock().take().unwrap().send(()).unwrap();
+ let _ = release_rx.lock().take().unwrap().recv();
+ inject_error
+ }),
+ ledger,
+ target_memtable: Arc::as_ptr(&idx.memtable) as usize,
+ };
+
+ let flush = {
+ let idx = Arc::clone(&idx);
+ tokio::spawn(FLUSH_PUBLISHER_TEST_HOOK.scope(hook, async move { idx.flush().await }))
+ };
+ tokio::task::spawn_blocking(move || entered_rx.recv().unwrap())
+ .await
+ .unwrap();
+ assert_eq!(idx.memtable_bytes(), 0);
+ assert!(
+ ledger
+ .gauge(Category::FlushDrained, Measurement::Estimated)
+ .current
+ > 0
+ );
+ publisher_release.release();
+ let result = flush.await.unwrap();
+ assert_eq!(result.is_err(), inject_error);
+ assert_eq!(
+ ledger
+ .gauge(Category::FlushDrained, Measurement::Estimated)
+ .current,
+ 0
+ );
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+ async fn parallel_real_flushes_route_isolated_ledgers_through_success_and_error() {
+ let (success, failure) = tokio::join!(
+ run_flush_memory_case("flush-memory-success", false),
+ run_flush_memory_case("flush-memory-failure", true),
+ );
+ assert_eq!((success, failure), ((), ()));
+ }
+}
diff --git a/engine/crates/xerj-engine/src/ingest_memory.rs b/engine/crates/xerj-engine/src/ingest_memory.rs
new file mode 100644
index 000000000..f635d7b12
--- /dev/null
+++ b/engine/crates/xerj-engine/src/ingest_memory.rs
@@ -0,0 +1,982 @@
+//! Opt-in ingest memory attribution.
+//!
+//! This module is deliberately independent of admission and scheduling. When
+//! `XERJ_INGEST_MEMORY_TRACE` is absent or `off`, transition helpers return
+//! before touching atomics, allocating, walking documents, probing `/proc`, or
+//! constructing events.
+//!
+//! `summary` is the only enabled mode in v1: it emits bounded periodic
+//! start/snapshot/stop NDJSON. Lifecycle `EventKind` variants and merge/cache
+//! categories are reserved schema vocabulary; until their owners are wired,
+//! their measurements are `unavailable` and their gauges must not be treated
+//! as measured zero.
+//!
+//! `MemtableActive` is sampled from the engine while `FlushDrained` is owned at
+//! the authoritative drain boundary. Those authorities cannot be sampled
+//! atomically, so a transient snapshot may count the same estimate in both.
+//! Logical totals are attribution diagnostics, never allocator identities.
+
+use serde::Serialize;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, LazyLock, Mutex};
+
+pub const SCHEMA: &str = "xerj.ingest_memory.v1";
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum TraceMode {
+ Off,
+ Summary,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct TraceConfig {
+ pub mode: TraceMode,
+ pub sample_ms: u64,
+}
+
+impl TraceConfig {
+ fn from_env() -> Self {
+ let mode = std::env::var("XERJ_INGEST_MEMORY_TRACE").unwrap_or_default();
+ let sample_ms = std::env::var("XERJ_INGEST_MEMORY_SAMPLE_MS").unwrap_or_default();
+ Self::parse(&mode, &sample_ms)
+ }
+
+ fn parse(mode: &str, sample_ms: &str) -> Self {
+ let mode = match mode.to_ascii_lowercase().as_str() {
+ "summary" | "1" | "true" | "on" => TraceMode::Summary,
+ _ => TraceMode::Off,
+ };
+ let sample_ms = sample_ms
+ .parse::<u64>()
+ .ok()
+ .unwrap_or(100)
+ .clamp(25, 60_000);
+ Self { mode, sample_ms }
+ }
+}
+
+static CONFIG: LazyLock<TraceConfig> = LazyLock::new(TraceConfig::from_env);
+
+#[cfg(test)]
+tokio::task_local! {
+ static TEST_LEDGER: &'static Ledger;
+}
+
+#[cfg(test)]
+fn test_ledger() -> Option<&'static Ledger> {
+ TEST_LEDGER.try_with(|ledger| *ledger).ok()
+}
+
+#[cfg(not(test))]
+fn test_ledger() -> Option<&'static Ledger> {
+ None
+}
+
+#[inline]
+pub fn config() -> TraceConfig {
+ *CONFIG
+}
+
+#[inline]
+pub fn enabled() -> bool {
+ active_ledger().is_some()
+}
+
+#[inline]
+pub(crate) fn active_ledger() -> Option<&'static Ledger> {
+ test_ledger().or_else(|| (CONFIG.mode != TraceMode::Off).then_some(&LEDGER))
+}
+
+#[cfg(test)]
+pub(crate) async fn with_test_ledger<F: std::future::Future>(
+ ledger: &'static Ledger,
+ future: F,
+) -> F::Output {
+ TEST_LEDGER.scope(ledger, future).await
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+#[repr(u8)]
+pub enum Category {
+ HttpBody = 0,
+ HttpRewriteBuffer,
+ RawSource,
+ ParsedSource,
+ ParsedVector,
+ PreparedDocument,
+ PreparedVector,
+ MemtableActive,
+ FlushDrained,
+ MergeDecoded,
+ MergeSurvivor,
+ MergeJsonBuffer,
+ MergeParsed,
+ MergeEncoded,
+ CacheStoredSlices,
+ CacheDocValues,
+ CacheStoredValues,
+ CacheSortShadow,
+ CacheIdPositions,
+ CacheRowSequences,
+ CacheDecodedStored,
+}
+
+impl Category {
+ pub const ALL: [Self; 21] = [
+ Self::HttpBody,
+ Self::HttpRewriteBuffer,
+ Self::RawSource,
+ Self::ParsedSource,
+ Self::ParsedVector,
+ Self::PreparedDocument,
+ Self::PreparedVector,
+ Self::MemtableActive,
+ Self::FlushDrained,
+ Self::MergeDecoded,
+ Self::MergeSurvivor,
+ Self::MergeJsonBuffer,
+ Self::MergeParsed,
+ Self::MergeEncoded,
+ Self::CacheStoredSlices,
+ Self::CacheDocValues,
+ Self::CacheStoredValues,
+ Self::CacheSortShadow,
+ Self::CacheIdPositions,
+ Self::CacheRowSequences,
+ Self::CacheDecodedStored,
+ ];
+
+ const COUNT: usize = Self::ALL.len();
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Measurement {
+ ExactCapacity,
+ SerializedSize,
+ Estimated,
+ Unavailable,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum EventKind {
+ TraceStart,
+ TraceStop,
+ TraceSnapshot,
+ HttpBulkAccept,
+ HttpBulkRelease,
+ BulkGroupRetain,
+ BulkGroupRelease,
+ SemanticParsedRetain,
+ SemanticParsedRelease,
+ SemanticPreparedRetain,
+ SemanticPreparedRelease,
+ MemtableInsert,
+ MemtableDrain,
+ FlushDrainedRetain,
+ FlushFinalizeWaitEnter,
+ FlushFinalizeStart,
+ FlushFinalizeEnd,
+ FlushDrainedRelease,
+ MergeRetain,
+ MergePhase,
+ MergeRelease,
+ CacheInsert,
+ CacheReplace,
+ CacheEvict,
+ CacheClear,
+ TraceError,
+}
+
+#[derive(Clone, Copy, Debug, Default, Serialize)]
+pub struct Scope {
+ pub index_hash: Option<u32>,
+ pub request: Option<u64>,
+ pub flush: Option<u64>,
+ pub merge: Option<u64>,
+ pub shard: Option<u32>,
+}
+
+#[derive(Clone, Copy, Debug, Serialize)]
+pub struct Gauge {
+ pub category: Category,
+ pub current: u64,
+ pub peak: u64,
+ pub measurement: Measurement,
+}
+
+#[derive(Clone, Copy, Debug, Default, Serialize)]
+pub struct AllocatorSnapshot {
+ pub allocated: Option<u64>,
+ pub active: Option<u64>,
+ pub resident: Option<u64>,
+ pub epoch_ok: bool,
+}
+
+#[derive(Clone, Copy, Debug, Default, Serialize)]
+pub struct ProcessSnapshot {
+ pub rss: Option<u64>,
+ pub rss_ok: bool,
+ pub cpu_time_ns: Option<u64>,
+ pub cpu_time_ok: bool,
+}
+
+#[derive(Debug, Serialize)]
+pub struct Event {
+ pub schema: &'static str,
+ pub seq: u64,
+ pub ts_unix_ns: u128,
+ pub event: EventKind,
+ pub scope: Scope,
+ pub logical: Vec<Gauge>,
+ /// Sum of all logical category currents in this coherent snapshot.
+ /// This is diagnostic attribution, not an allocator/RSS identity.
+ pub logical_total_current: u64,
+ /// Retain/release underflows observed since process start.
+ pub accounting_errors: u64,
+ /// Trace events rejected by the bounded output sink since trace start.
+ pub dropped_events: u64,
+ pub allocator: AllocatorSnapshot,
+ pub process: ProcessSnapshot,
+ pub flags: EventFlags,
+}
+
+#[derive(Clone, Copy, Debug, Serialize)]
+pub struct EventFlags {
+ pub sampled: bool,
+ pub logical_exact: bool,
+ pub allocator_supported: bool,
+}
+
+struct Cell {
+ current: AtomicU64,
+ peak: AtomicU64,
+}
+
+impl Cell {
+ const fn new() -> Self {
+ Self {
+ current: AtomicU64::new(0),
+ peak: AtomicU64::new(0),
+ }
+ }
+}
+
+pub struct Ledger {
+ transition: Mutex<()>,
+ cells: [Cell; Category::COUNT],
+ seq: AtomicU64,
+ accounting_errors: AtomicU64,
+ physical_allocated: AtomicU64,
+ physical_active: AtomicU64,
+ physical_resident: AtomicU64,
+ physical_rss: AtomicU64,
+ physical_cpu_time_ns: AtomicU64,
+ physical_flags: AtomicU64,
+ dropped_events: AtomicU64,
+}
+
+impl Ledger {
+ pub const fn new() -> Self {
+ Self {
+ transition: Mutex::new(()),
+ cells: [const { Cell::new() }; Category::COUNT],
+ seq: AtomicU64::new(0),
+ accounting_errors: AtomicU64::new(0),
+ physical_allocated: AtomicU64::new(0),
+ physical_active: AtomicU64::new(0),
+ physical_resident: AtomicU64::new(0),
+ physical_rss: AtomicU64::new(0),
+ physical_cpu_time_ns: AtomicU64::new(0),
+ physical_flags: AtomicU64::new(0),
+ dropped_events: AtomicU64::new(0),
+ }
+ }
+
+ fn retain(&self, category: Category, bytes: u64) {
+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());
+ self.retain_locked(category, bytes);
+ }
+
+ fn retain_locked(&self, category: Category, bytes: u64) {
+ if bytes == 0 {
+ return;
+ }
+ let cell = &self.cells[category as usize];
+ let next = cell.current.fetch_add(bytes, Ordering::Relaxed) + bytes;
+ let mut peak = cell.peak.load(Ordering::Relaxed);
+ while next > peak {
+ match cell
+ .peak
+ .compare_exchange_weak(peak, next, Ordering::Relaxed, Ordering::Relaxed)
+ {
+ Ok(_) => break,
+ Err(observed) => peak = observed,
+ }
+ }
+ }
+
+ fn release(&self, category: Category, bytes: u64) {
+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());
+ self.release_locked(category, bytes);
+ }
+
+ fn release_locked(&self, category: Category, bytes: u64) -> bool {
+ if bytes == 0 {
+ return true;
+ }
+ let cell = &self.cells[category as usize];
+ let mut current = cell.current.load(Ordering::Relaxed);
+ loop {
+ if bytes > current {
+ self.accounting_errors.fetch_add(1, Ordering::Relaxed);
+ return false;
+ }
+ match cell.current.compare_exchange_weak(
+ current,
+ current - bytes,
+ Ordering::Relaxed,
+ Ordering::Relaxed,
+ ) {
+ Ok(_) => return true,
+ Err(observed) => current = observed,
+ }
+ }
+ }
+
+ fn transfer(&self, from: Category, to: Category, bytes: u64) {
+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());
+ if self.release_locked(from, bytes) {
+ self.retain_locked(to, bytes);
+ }
+ }
+
+ pub fn gauge(&self, category: Category, measurement: Measurement) -> Gauge {
+ let cell = &self.cells[category as usize];
+ Gauge {
+ category,
+ current: cell.current.load(Ordering::Relaxed),
+ peak: cell.peak.load(Ordering::Relaxed),
+ measurement,
+ }
+ }
+
+ pub fn accounting_errors(&self) -> u64 {
+ self.accounting_errors.load(Ordering::Relaxed)
+ }
+
+ pub fn record_dropped_event(&self) {
+ self.dropped_events.fetch_add(1, Ordering::Relaxed);
+ }
+
+ pub fn observe(&self, category: Category, bytes: usize) {
+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());
+ let cell = &self.cells[category as usize];
+ let bytes = bytes as u64;
+ cell.current.store(bytes, Ordering::Relaxed);
+ cell.peak.fetch_max(bytes, Ordering::Relaxed);
+ }
+
+ pub fn update_physical(&self, allocator: AllocatorSnapshot, process: ProcessSnapshot) {
+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());
+ const ALLOCATOR_OK: u64 = 1;
+ const RSS_OK: u64 = 2;
+ const CPU_TIME_OK: u64 = 4;
+ let mut flags = 0;
+ if allocator.epoch_ok {
+ flags |= ALLOCATOR_OK;
+ }
+ if process.rss_ok {
+ flags |= RSS_OK;
+ }
+ if process.cpu_time_ok {
+ flags |= CPU_TIME_OK;
+ }
+ self.physical_allocated
+ .store(allocator.allocated.unwrap_or(0), Ordering::Relaxed);
+ self.physical_active
+ .store(allocator.active.unwrap_or(0), Ordering::Relaxed);
+ self.physical_resident
+ .store(allocator.resident.unwrap_or(0), Ordering::Relaxed);
+ self.physical_rss
+ .store(process.rss.unwrap_or(0), Ordering::Relaxed);
+ self.physical_cpu_time_ns
+ .store(process.cpu_time_ns.unwrap_or(0), Ordering::Relaxed);
+ self.physical_flags.store(flags, Ordering::Release);
+ }
+
+ fn event(&self, kind: EventKind, scope: Scope, sampled: bool) -> Event {
+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());
+ let flags = self.physical_flags.load(Ordering::Acquire);
+ let allocator_ok = flags & 1 != 0;
+ let rss_ok = flags & 2 != 0;
+ let cpu_time_ok = flags & 4 != 0;
+ let logical: Vec<_> = Category::ALL
+ .into_iter()
+ .map(|category| self.gauge(category, measurement(category)))
+ .collect();
+ let logical_total_current = logical.iter().map(|gauge| gauge.current).sum();
+ Event {
+ schema: SCHEMA,
+ seq: self.seq.fetch_add(1, Ordering::Relaxed) + 1,
+ ts_unix_ns: std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_nanos(),
+ event: kind,
+ scope,
+ logical,
+ logical_total_current,
+ accounting_errors: self.accounting_errors.load(Ordering::Relaxed),
+ dropped_events: self.dropped_events.load(Ordering::Relaxed),
+ allocator: AllocatorSnapshot {
+ allocated: allocator_ok.then(|| self.physical_allocated.load(Ordering::Relaxed)),
+ active: allocator_ok.then(|| self.physical_active.load(Ordering::Relaxed)),
+ resident: allocator_ok.then(|| self.physical_resident.load(Ordering::Relaxed)),
+ epoch_ok: allocator_ok,
+ },
+ process: ProcessSnapshot {
+ rss: rss_ok.then(|| self.physical_rss.load(Ordering::Relaxed)),
+ rss_ok,
+ cpu_time_ns: cpu_time_ok.then(|| self.physical_cpu_time_ns.load(Ordering::Relaxed)),
+ cpu_time_ok,
+ },
+ flags: EventFlags {
+ sampled,
+ logical_exact: false,
+ allocator_supported: allocator_ok,
+ },
+ }
+ }
+}
+
+pub static LEDGER: Ledger = Ledger::new();
+
+/// Build one sampler-owned event. This is the only public event constructor:
+/// ingest hot paths cannot accidentally allocate events in `off` or `summary`.
+pub fn snapshot_event(kind: EventKind, sampled: bool) -> Option<Event> {
+ if !enabled()
+ || !matches!(
+ kind,
+ EventKind::TraceStart | EventKind::TraceStop | EventKind::TraceSnapshot
+ )
+ {
+ return None;
+ }
+ Some(LEDGER.event(kind, Scope::default(), sampled))
+}
+
+fn measurement(category: Category) -> Measurement {
+ match category {
+ Category::ParsedVector | Category::PreparedVector | Category::HttpRewriteBuffer => {
+ Measurement::ExactCapacity
+ }
+ Category::HttpBody | Category::RawSource => Measurement::SerializedSize,
+ Category::MemtableActive
+ | Category::ParsedSource
+ | Category::PreparedDocument
+ | Category::FlushDrained => Measurement::Estimated,
+ _ => Measurement::Unavailable,
+ }
+}
+
+pub struct Retained<'a> {
+ ledger: &'a Ledger,
+ category: Category,
+ bytes: u64,
+ active: bool,
+}
+
+impl Retained<'static> {
+ #[inline]
+ pub fn new(category: Category, bytes: usize) -> Self {
+ let ledger = active_ledger();
+ let Some(ledger) = ledger else {
+ return Self::disabled();
+ };
+ let bytes = bytes as u64;
+ ledger.retain(category, bytes);
+ Self {
+ ledger,
+ category,
+ bytes,
+ active: true,
+ }
+ }
+
+ #[inline]
+ pub const fn disabled() -> Self {
+ Self {
+ ledger: &LEDGER,
+ category: Category::HttpBody,
+ bytes: 0,
+ active: false,
+ }
+ }
+}
+
+impl<'a> Retained<'a> {
+ pub(crate) fn for_ledger(ledger: &'a Ledger, category: Category, bytes: usize) -> Self {
+ let bytes = bytes as u64;
+ ledger.retain(category, bytes);
+ Self {
+ ledger,
+ category,
+ bytes,
+ active: true,
+ }
+ }
+
+ pub fn transfer(mut self, category: Category) -> Self {
+ if self.active {
+ self.ledger.transfer(self.category, category, self.bytes);
+ self.category = category;
+ }
+ self
+ }
+
+ pub fn replace(mut self, category: Category, bytes: usize) -> Self {
+ if self.active {
+ let _transition = self
+ .ledger
+ .transition
+ .lock()
+ .unwrap_or_else(|e| e.into_inner());
+ if self.ledger.release_locked(self.category, self.bytes) {
+ self.ledger.retain_locked(category, bytes as u64);
+ self.category = category;
+ self.bytes = bytes as u64;
+ }
+ }
+ self
+ }
+
+ pub fn replace_split(
+ mut self,
+ first: (Category, usize),
+ second: (Category, usize),
+ ) -> (Self, Self) {
+ let mut other = Self {
+ ledger: self.ledger,
+ category: second.0,
+ bytes: second.1 as u64,
+ active: false,
+ };
+ if self.active {
+ let _transition = self
+ .ledger
+ .transition
+ .lock()
+ .unwrap_or_else(|e| e.into_inner());
+ if self.ledger.release_locked(self.category, self.bytes) {
+ self.ledger.retain_locked(first.0, first.1 as u64);
+ self.ledger.retain_locked(second.0, second.1 as u64);
+ self.category = first.0;
+ self.bytes = first.1 as u64;
+ other.active = true;
+ }
+ }
+ (self, other)
+ }
+
+ pub fn replace_pair(
+ mut first: Self,
+ mut second: Self,
+ first_next: (Category, usize),
+ second_next: (Category, usize),
+ ) -> (Self, Self) {
+ if first.active && second.active && std::ptr::eq(first.ledger, second.ledger) {
+ let ledger = first.ledger;
+ let _transition = ledger.transition.lock().unwrap_or_else(|e| e.into_inner());
+ let first_current = ledger.cells[first.category as usize]
+ .current
+ .load(Ordering::Relaxed);
+ let second_current = ledger.cells[second.category as usize]
+ .current
+ .load(Ordering::Relaxed);
+ let enough = if first.category == second.category {
+ first_current >= first.bytes.saturating_add(second.bytes)
+ } else {
+ first_current >= first.bytes && second_current >= second.bytes
+ };
+ if enough {
+ let _ = ledger.release_locked(first.category, first.bytes);
+ let _ = ledger.release_locked(second.category, second.bytes);
+ ledger.retain_locked(first_next.0, first_next.1 as u64);
+ ledger.retain_locked(second_next.0, second_next.1 as u64);
+ first.category = first_next.0;
+ first.bytes = first_next.1 as u64;
+ second.category = second_next.0;
+ second.bytes = second_next.1 as u64;
+ } else {
+ ledger.accounting_errors.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+ (first, second)
+ }
+}
+
+/// Non-allocating estimate of heap retained by a JSON tree.
+pub fn estimated_json_heap(value: &serde_json::Value) -> usize {
+ match value {
+ serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => 0,
+ serde_json::Value::String(s) => s.capacity(),
+ serde_json::Value::Array(values) => {
+ values.capacity() * std::mem::size_of::<serde_json::Value>()
+ + values.iter().map(estimated_json_heap).sum::<usize>()
+ }
+ serde_json::Value::Object(values) => {
+ values.len()
+ * (std::mem::size_of::<String>() + std::mem::size_of::<serde_json::Value>())
+ + values
+ .iter()
+ .map(|(key, value)| key.capacity() + estimated_json_heap(value))
+ .sum::<usize>()
+ }
+ }
+}
+
+pub fn estimated_json_heap_excluding(
+ value: &serde_json::Value,
+ excluded: &std::collections::HashSet<String>,
+) -> usize {
+ fn visit(value: &serde_json::Value, excluded: &std::collections::HashSet<String>) -> usize {
+ match value {
+ serde_json::Value::Object(values) => {
+ values.len()
+ * (std::mem::size_of::<String>() + std::mem::size_of::<serde_json::Value>())
+ + values
+ .iter()
+ .map(|(key, value)| {
+ key.capacity()
+ + if excluded.contains(key) {
+ 0
+ } else {
+ visit(value, excluded)
+ }
+ })
+ .sum::<usize>()
+ }
+ serde_json::Value::Array(values) => {
+ values.capacity() * std::mem::size_of::<serde_json::Value>()
+ + values
+ .iter()
+ .map(|value| visit(value, excluded))
+ .sum::<usize>()
+ }
+ serde_json::Value::String(value) => value.capacity(),
+ _ => 0,
+ }
+ }
+ visit(value, excluded)
+}
+
+pub fn vector_value_capacity(
+ value: &serde_json::Value,
+ fields: &std::collections::HashSet<String>,
+) -> usize {
+ fn arrays(value: &serde_json::Value) -> usize {
+ match value {
+ serde_json::Value::Array(values) => {
+ values.capacity() * std::mem::size_of::<serde_json::Value>()
+ + values.iter().map(arrays).sum::<usize>()
+ }
+ serde_json::Value::Object(values) => values.values().map(arrays).sum(),
+ _ => 0,
+ }
+ }
+ match value {
+ serde_json::Value::Object(values) => values
+ .iter()
+ .map(|(key, value)| {
+ if fields.contains(key) {
+ arrays(value)
+ } else {
+ vector_value_capacity(value, fields)
+ }
+ })
+ .sum(),
+ serde_json::Value::Array(values) => values
+ .iter()
+ .map(|value| vector_value_capacity(value, fields))
+ .sum(),
+ _ => 0,
+ }
+}
+
+impl Drop for Retained<'_> {
+ fn drop(&mut self) {
+ if self.active {
+ self.ledger.release(self.category, self.bytes);
+ }
+ }
+}
+
+pub struct TrackedArc<T> {
+ value: T,
+ _retained: Retained<'static>,
+}
+
+impl<T> TrackedArc<T> {
+ pub fn new(value: T, category: Category, bytes: usize) -> Arc<Self> {
+ Arc::new(Self {
+ value,
+ _retained: Retained::new(category, bytes),
+ })
+ }
+
+ pub fn get(&self) -> &T {
+ &self.value
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn trace_modes_are_distinct_and_unknown_events_mode_fails_closed() {
+ assert_eq!(TraceConfig::parse("off", "100").mode, TraceMode::Off);
+ assert_eq!(
+ TraceConfig::parse("summary", "100").mode,
+ TraceMode::Summary
+ );
+ assert_eq!(
+ TraceConfig::parse("events", "100").mode,
+ TraceMode::Off,
+ "v1 must not claim unwired lifecycle events"
+ );
+ assert_eq!(TraceConfig::parse("summary", "1").sample_ms, 25);
+ assert_eq!(TraceConfig::parse("summary", "999999").sample_ms, 60_000);
+ }
+
+ #[test]
+ fn retain_release_peak_and_underflow_are_deterministic() {
+ let ledger = Ledger::new();
+ ledger.retain(Category::RawSource, 100);
+ ledger.retain(Category::RawSource, 60);
+ ledger.release(Category::RawSource, 100);
+ let gauge = ledger.gauge(Category::RawSource, Measurement::SerializedSize);
+ assert_eq!((gauge.current, gauge.peak), (60, 160));
+ ledger.release(Category::RawSource, 61);
+ assert_eq!(ledger.accounting_errors(), 1);
+ assert_eq!(
+ ledger
+ .gauge(Category::RawSource, Measurement::SerializedSize)
+ .current,
+ 60
+ );
+ }
+
+ #[test]
+ fn snapshot_self_reports_logical_total_accounting_and_sink_loss() {
+ let ledger = Ledger::new();
+ ledger.retain(Category::RawSource, 100);
+ ledger.retain(Category::PreparedVector, 28);
+ ledger.release(Category::RawSource, 101);
+ ledger.record_dropped_event();
+ let event = ledger.event(EventKind::TraceSnapshot, Scope::default(), true);
+ assert_eq!(event.logical_total_current, 128);
+ assert_eq!(event.accounting_errors, 1);
+ assert_eq!(event.dropped_events, 1);
+ }
+
+ #[test]
+ fn tracked_arc_charges_one_backing_owner() {
+ let ledger = Ledger::new();
+ struct LocalTracked<'a, T> {
+ value: T,
+ _retained: Retained<'a>,
+ }
+ let owner = Arc::new(LocalTracked {
+ value: vec![0u8; 100],
+ _retained: Retained::for_ledger(&ledger, Category::RawSource, 100),
+ });
+ let clones: Vec<_> = (0..100).map(|_| Arc::clone(&owner)).collect();
+ assert_eq!(owner.value.len(), 100);
+ assert_eq!(
+ ledger
+ .gauge(Category::RawSource, Measurement::SerializedSize)
+ .current,
+ 100
+ );
+ drop(clones);
+ drop(owner);
+ assert_eq!(
+ ledger
+ .gauge(Category::RawSource, Measurement::SerializedSize)
+ .current,
+ 0
+ );
+ }
+
+ #[test]
+ fn transfer_snapshot_has_no_zero_or_double_count_gap() {
+ let ledger = Ledger::new();
+ let retained = Retained::for_ledger(&ledger, Category::MemtableActive, 77);
+ let retained = retained.transfer(Category::FlushDrained);
+ let active = ledger.gauge(Category::MemtableActive, Measurement::Estimated);
+ let drained = ledger.gauge(Category::FlushDrained, Measurement::Estimated);
+ assert_eq!((active.current, drained.current), (0, 77));
+ drop(retained);
+ assert_eq!(
+ ledger
+ .gauge(Category::FlushDrained, Measurement::Estimated)
+ .current,
+ 0
+ );
+ }
+
+ #[test]
+ fn failed_transfer_does_not_charge_destination() {
+ let ledger = Ledger::new();
+ ledger.retain(Category::MemtableActive, 10);
+ ledger.transfer(Category::MemtableActive, Category::FlushDrained, 11);
+ assert_eq!(
+ ledger
+ .gauge(Category::MemtableActive, Measurement::Estimated)
+ .current,
+ 10
+ );
+ assert_eq!(
+ ledger
+ .gauge(Category::FlushDrained, Measurement::Estimated)
+ .current,
+ 0
+ );
+ assert_eq!(ledger.accounting_errors(), 1);
+ }
+
+ #[test]
+ fn semantic_phase_split_is_non_overlapping_and_balances_on_drop() {
+ let ledger = Ledger::new();
+ let parsed = Retained::for_ledger(&ledger, Category::ParsedSource, 120);
+ let (document, vector) = parsed.replace_split(
+ (Category::PreparedDocument, 80),
+ (Category::PreparedVector, 40),
+ );
+ assert_eq!(
+ ledger
+ .gauge(Category::ParsedSource, Measurement::Estimated)
+ .current,
+ 0
+ );
+ assert_eq!(
+ ledger
+ .gauge(Category::PreparedDocument, Measurement::Estimated)
+ .current,
+ 80
+ );
+ assert_eq!(
+ ledger
+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)
+ .current,
+ 40
+ );
+ drop((document, vector));
+ assert_eq!(
+ ledger
+ .gauge(Category::PreparedDocument, Measurement::Estimated)
+ .current
+ + ledger
+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)
+ .current,
+ 0
+ );
+ }
+
+ #[test]
+ fn caller_vector_and_document_transition_as_one_snapshot() {
+ let ledger = Ledger::new();
+ let document = Retained::for_ledger(&ledger, Category::ParsedSource, 70);
+ let vector = Retained::for_ledger(&ledger, Category::ParsedVector, 30);
+ let (document, vector) = Retained::replace_pair(
+ document,
+ vector,
+ (Category::PreparedDocument, 60),
+ (Category::PreparedVector, 40),
+ );
+ assert_eq!(
+ ledger
+ .gauge(Category::ParsedSource, Measurement::Estimated)
+ .current
+ + ledger
+ .gauge(Category::ParsedVector, Measurement::ExactCapacity)
+ .current,
+ 0
+ );
+ assert_eq!(
+ ledger
+ .gauge(Category::PreparedDocument, Measurement::Estimated)
+ .current
+ + ledger
+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)
+ .current,
+ 100
+ );
+ drop((document, vector));
+ assert_eq!(
+ ledger
+ .gauge(Category::PreparedDocument, Measurement::Estimated)
+ .current
+ + ledger
+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)
+ .current,
+ 0
+ );
+ }
+
+ #[test]
+ fn custom_vector_fields_are_excluded_and_capacity_counted() {
+ let value = serde_json::json!({
+ "body": "hello",
+ "custom_embedding": [0.1, 0.2, 0.3],
+ });
+ let fields = std::collections::HashSet::from(["custom_embedding".to_string()]);
+ let without = estimated_json_heap_excluding(&value, &fields);
+ let with = estimated_json_heap(&value);
+ assert!(without < with);
+ assert_eq!(
+ vector_value_capacity(&value, &fields),
+ value["custom_embedding"].as_array().unwrap().capacity()
+ * std::mem::size_of::<serde_json::Value>()
+ );
+ }
+
+ #[test]
+ fn guard_balances_error_and_panic_unwind() {
+ let ledger = Ledger::new();
+ let result = std::panic::catch_unwind(|| {
+ ledger.retain(Category::FlushDrained, 42);
+ struct Local<'a>(&'a Ledger);
+ impl Drop for Local<'_> {
+ fn drop(&mut self) {
+ self.0.release(Category::FlushDrained, 42);
+ }
+ }
+ let _guard = Local(&ledger);
+ panic!("intentional");
+ });
+ assert!(result.is_err());
+ assert_eq!(
+ ledger
+ .gauge(Category::FlushDrained, Measurement::Estimated)
+ .current,
+ 0
+ );
+ }
+
+ #[test]
+ fn physical_failure_is_null_not_zero() {
+ let ledger = Ledger::new();
+ ledger.update_physical(AllocatorSnapshot::default(), ProcessSnapshot::default());
+ let event = ledger.event(EventKind::TraceSnapshot, Scope::default(), true);
+ assert_eq!(event.allocator.allocated, None);
+ assert_eq!(event.process.rss, None);
+ let json = serde_json::to_value(event).unwrap();
+ assert_eq!(json["schema"], SCHEMA);
+ assert!(json["allocator"]["allocated"].is_null());
+ }
+}
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 1bb665b5e..d53fd7fb0 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -22,6 +22,7 @@ pub mod bulk;
pub mod engine;
pub mod governor;
pub mod index;
+pub mod ingest_memory;
pub mod memtable;
pub mod painless;
pub mod rbac;
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index e828b173d..6a31b176c 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -25,6 +25,81 @@ pub struct MemtableHit {
pub score: f32,
}
+#[cfg(test)]
+mod ingest_memory_drain_tests {
+ use super::*;
+ use crate::ingest_memory::{Category, Ledger, Measurement};
+ use std::sync::{Arc, Barrier};
+
+ #[test]
+ fn authoritative_active_moves_to_drained_through_blocked_finalize_success_and_error() {
+ for inject_error in [false, true] {
+ let mem = ShardedFtsMemtable::new();
+ mem.insert(
+ "doc-1".to_string(),
+ &serde_json::json!({"body": "retained"}),
+ &Schema::default(),
+ 1,
+ );
+ let shard = mem
+ .shard_loads()
+ .into_iter()
+ .find(|(_, docs, _)| *docs != 0)
+ .unwrap()
+ .0;
+ let ledger = Ledger::new();
+ ledger.observe(Category::MemtableActive, mem.size_bytes());
+ assert!(
+ ledger
+ .gauge(Category::MemtableActive, Measurement::Estimated)
+ .current
+ > 0
+ );
+
+ let ledger: &'static Ledger = Box::leak(Box::new(ledger));
+ let (_entries, guard) = mem.drain_shard_accounted_for_test(shard, false, ledger);
+ ledger.observe(Category::MemtableActive, mem.size_bytes());
+ let entered = Arc::new(Barrier::new(2));
+ let release = Arc::new(Barrier::new(2));
+ std::thread::scope(|scope| {
+ let entered_worker = Arc::clone(&entered);
+ let release_worker = Arc::clone(&release);
+ let handle = scope.spawn(move || -> Result<(), &'static str> {
+ let _guard = guard;
+ entered_worker.wait();
+ release_worker.wait();
+ if inject_error {
+ Err("injected finalizer error")
+ } else {
+ Ok(())
+ }
+ });
+ entered.wait();
+ assert_eq!(
+ ledger
+ .gauge(Category::MemtableActive, Measurement::Estimated)
+ .current,
+ 0
+ );
+ assert!(
+ ledger
+ .gauge(Category::FlushDrained, Measurement::Estimated)
+ .current
+ > 0
+ );
+ release.wait();
+ assert_eq!(handle.join().unwrap().is_err(), inject_error);
+ });
+ assert_eq!(
+ ledger
+ .gauge(Category::FlushDrained, Measurement::Estimated)
+ .current,
+ 0
+ );
+ }
+ }
+}
+
/// Entry stored in the memtable.
///
/// Post-M4.9 the `fields: HashMap<String, String>` was removed — it was
@@ -1486,21 +1561,55 @@ impl ShardedFtsMemtable {
/// tuples in WAL-sequence order. Raw bytes are passed through to the
/// segment writer so it can skip re-serializing the Value.
pub fn drain_shard(&self, shard_idx: usize) -> Vec<(u64, String, Arc<Value>, Arc<[u8]>)> {
- self.drain_shard_inner(shard_idx, false)
+ self.drain_shard_inner(shard_idx, false, None).0
}
/// Drain without parsing raw-bytes entries. Returns Value::Null for
/// entries that came from insert_raw_bytes_with_seq. Use when neither
/// FTS nor DV sidecars will be built (turbo/CLI ingest path).
pub fn drain_shard_raw(&self, shard_idx: usize) -> Vec<(u64, String, Arc<Value>, Arc<[u8]>)> {
- self.drain_shard_inner(shard_idx, true)
+ self.drain_shard_inner(shard_idx, true, None).0
}
- fn drain_shard_inner(
+ pub fn drain_shard_accounted(
&self,
shard_idx: usize,
skip_parse: bool,
- ) -> Vec<(u64, String, Arc<Value>, Arc<[u8]>)> {
+ ) -> (
+ Vec<(u64, String, Arc<Value>, Arc<[u8]>)>,
+ crate::ingest_memory::Retained<'static>,
+ ) {
+ let ledger = crate::ingest_memory::active_ledger();
+ let (entries, guard) = self.drain_shard_inner(shard_idx, skip_parse, ledger);
+ (
+ entries,
+ guard.unwrap_or_else(crate::ingest_memory::Retained::disabled),
+ )
+ }
+
+ #[cfg(test)]
+ pub(crate) fn drain_shard_accounted_for_test(
+ &self,
+ shard_idx: usize,
+ skip_parse: bool,
+ ledger: &'static crate::ingest_memory::Ledger,
+ ) -> (
+ Vec<(u64, String, Arc<Value>, Arc<[u8]>)>,
+ crate::ingest_memory::Retained<'static>,
+ ) {
+ let (entries, guard) = self.drain_shard_inner(shard_idx, skip_parse, Some(ledger));
+ (entries, guard.expect("test ledger always creates guard"))
+ }
+
+ fn drain_shard_inner<'a>(
+ &self,
+ shard_idx: usize,
+ skip_parse: bool,
+ ledger: Option<&'a crate::ingest_memory::Ledger>,
+ ) -> (
+ Vec<(u64, String, Arc<Value>, Arc<[u8]>)>,
+ Option<crate::ingest_memory::Retained<'a>>,
+ ) {
// Swap the shard's maps out under the write lock (pointer moves,
// O(1)) and deallocate them AFTER the lock is released, on a
// detached thread. Pre-fix the reset assignments freed the
@@ -1511,8 +1620,9 @@ impl ShardedFtsMemtable {
// drain stalled every in-flight bulk request. 32 flushes per
// 1 M docs × ~95 ms = a fixed ~3 s Amdahl serial term that
// capped 8-client ingest at ~3.1× single-client throughput.
- let (drained, dead) = {
+ let (drained, dead, drain_guard) = {
let mut g = self.shards[shard_idx].write();
+ let removed_bytes = g.total_bytes;
let d: Vec<MemEntry> = std::mem::take(&mut g.docs);
let dead_index = std::mem::take(&mut g.index);
let dead_dv = std::mem::take(&mut g.doc_values);
@@ -1520,6 +1630,17 @@ impl ShardedFtsMemtable {
let dead_afl = std::mem::take(&mut g.avg_field_lengths);
let dead_dii = std::mem::take(&mut g.doc_id_index);
g.total_bytes = 0;
+ // Created while the authoritative shard lock is still held so
+ // the drained lifetime includes detached-map handoff and parsing.
+ // Periodic active-vs-drained snapshots remain best-effort because
+ // the two authorities are sampled separately.
+ let drain_guard = ledger.map(|ledger| {
+ crate::ingest_memory::Retained::for_ledger(
+ ledger,
+ crate::ingest_memory::Category::FlushDrained,
+ removed_bytes,
+ )
+ });
// Flush == merge: purge delete-aware ghost collection stats.
g.ghost_docs = 0;
let dead_gfl = std::mem::take(&mut g.ghost_field_len);
@@ -1529,6 +1650,7 @@ impl ShardedFtsMemtable {
(
dead_index, dead_dv, dead_fl, dead_afl, dead_dii, dead_gfl, dead_gdf,
),
+ drain_guard,
)
};
// Free the dead maps off the flush critical path too — the
@@ -1558,7 +1680,7 @@ impl ShardedFtsMemtable {
})
.collect();
out.sort_by_key(|(seq, _, _, _)| *seq);
- out
+ (out, drain_guard)
}
/// Check if a shard's first entry was inserted via the raw-bytes
diff --git a/engine/crates/xerj-server/Cargo.toml b/engine/crates/xerj-server/Cargo.toml
index 98e8fcb0c..d0a756590 100644
--- a/engine/crates/xerj-server/Cargo.toml
+++ b/engine/crates/xerj-server/Cargo.toml
@@ -76,6 +76,8 @@ debug-profiling = [
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = "0.6"
+tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }
+libc = "0.2"
[target.'cfg(all(target_os = "linux", not(target_env = "msvc")))'.dependencies]
jemalloc_pprof = { version = "0.8", features = ["symbolize"], optional = true }
diff --git a/engine/crates/xerj-server/src/ingest_memory_trace.rs b/engine/crates/xerj-server/src/ingest_memory_trace.rs
new file mode 100644
index 000000000..a7db3d8e6
--- /dev/null
+++ b/engine/crates/xerj-server/src/ingest_memory_trace.rs
@@ -0,0 +1,411 @@
+use std::io::Write;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{mpsc, Arc, Condvar, Mutex, Weak};
+use std::thread::JoinHandle;
+use std::time::Duration;
+
+use xerj_engine::ingest_memory::{
+ self, AllocatorSnapshot, EventKind, ProcessSnapshot, TraceMode, LEDGER,
+};
+use xerj_engine::Engine;
+
+const CHANNEL_CAPACITY: usize = 256;
+
+pub struct TraceRuntime {
+ stop: Arc<AtomicBool>,
+ wake: Arc<(Mutex<()>, Condvar)>,
+ sampler: Option<JoinHandle<()>>,
+ writer: Option<JoinHandle<()>>,
+ dropped: Arc<AtomicU64>,
+}
+
+impl Drop for TraceRuntime {
+ fn drop(&mut self) {
+ self.stop.store(true, Ordering::Release);
+ self.wake.1.notify_all();
+ if let Some(handle) = self.sampler.take() {
+ let _ = handle.join();
+ }
+ if let Some(handle) = self.writer.take() {
+ let _ = handle.join();
+ }
+ let dropped = self.dropped.load(Ordering::Relaxed);
+ if dropped != 0 {
+ tracing::warn!(
+ target: "xerj::ingest_memory",
+ dropped,
+ "ingest-memory trace events dropped by bounded sink"
+ );
+ }
+ }
+}
+
+pub fn spawn(engine: &Arc<Engine>) -> Option<TraceRuntime> {
+ let config = ingest_memory::config();
+ if config.mode == TraceMode::Off {
+ return None;
+ }
+
+ let (tx, rx) = mpsc::sync_channel::<String>(CHANNEL_CAPACITY);
+ let stop = Arc::new(AtomicBool::new(false));
+ let dropped = Arc::new(AtomicU64::new(0));
+ let writer_dropped = Arc::clone(&dropped);
+ let writer = std::thread::Builder::new()
+ .name("xerj-ingest-memory-writer".into())
+ .spawn(move || writer_loop(rx, writer_dropped))
+ .ok()?;
+
+ let weak = Arc::downgrade(engine);
+ let sampler_stop = Arc::clone(&stop);
+ let wake = Arc::new((Mutex::new(()), Condvar::new()));
+ let sampler_wake = Arc::clone(&wake);
+ let sampler_dropped = Arc::clone(&dropped);
+ let sampler = std::thread::Builder::new()
+ .name("xerj-ingest-memory-sampler".into())
+ .spawn(move || {
+ send_snapshot(&tx, &sampler_dropped, EventKind::TraceStart, false);
+ sample_loop(
+ weak,
+ tx,
+ sampler_stop,
+ sampler_wake,
+ sampler_dropped,
+ Duration::from_millis(config.sample_ms),
+ );
+ })
+ .ok()?;
+
+ Some(TraceRuntime {
+ stop,
+ wake,
+ sampler: Some(sampler),
+ writer: Some(writer),
+ dropped,
+ })
+}
+
+fn sample_loop(
+ engine: Weak<Engine>,
+ tx: mpsc::SyncSender<String>,
+ stop: Arc<AtomicBool>,
+ wake: Arc<(Mutex<()>, Condvar)>,
+ dropped: Arc<AtomicU64>,
+ period: Duration,
+) {
+ while !stop.load(Ordering::Acquire) {
+ let Some(engine) = engine.upgrade() else {
+ break;
+ };
+ refresh_snapshot_state(&engine);
+ send_snapshot(&tx, &dropped, EventKind::TraceSnapshot, true);
+ wait_for_stop_or_period(&stop, &wake, period);
+ }
+
+ // The shutdown path force-flushes every index before dropping
+ // `TraceRuntime`. That can change the sampled memtable gauge after the
+ // final periodic snapshot. Refresh synchronously here so `trace_stop`
+ // describes the post-flush state even when shutdown happens immediately
+ // after ingest, without waiting for another sampler period.
+ if let Some(engine) = engine.upgrade() {
+ refresh_snapshot_state(&engine);
+ }
+ send_snapshot(&tx, &dropped, EventKind::TraceStop, false);
+}
+
+fn refresh_snapshot_state(engine: &Engine) {
+ refresh_snapshot_state_with(
+ engine.total_memtable_bytes(),
+ allocator_snapshot(),
+ xerj_engine::governor::current_rss_bytes(),
+ process_cpu_time_ns(),
+ );
+}
+
+fn refresh_snapshot_state_with(
+ memtable_bytes: u64,
+ allocator: AllocatorSnapshot,
+ rss: u64,
+ cpu_time_ns: Option<u64>,
+) {
+ LEDGER.observe(
+ ingest_memory::Category::MemtableActive,
+ memtable_bytes as usize,
+ );
+ let process = ProcessSnapshot {
+ rss: (rss != 0).then_some(rss),
+ rss_ok: rss != 0,
+ cpu_time_ns,
+ cpu_time_ok: cpu_time_ns.is_some(),
+ };
+ LEDGER.update_physical(allocator, process);
+}
+
+fn wait_for_stop_or_period(stop: &AtomicBool, wake: &(Mutex<()>, Condvar), period: Duration) {
+ let guard = wake.0.lock().unwrap_or_else(|e| e.into_inner());
+ let _ = wake
+ .1
+ .wait_timeout_while(guard, period, |_| !stop.load(Ordering::Acquire));
+}
+
+fn send_snapshot(
+ tx: &mpsc::SyncSender<String>,
+ dropped: &AtomicU64,
+ kind: EventKind,
+ sampled: bool,
+) {
+ let Some(event) = ingest_memory::snapshot_event(kind, sampled) else {
+ return;
+ };
+ match serde_json::to_string(&event) {
+ Ok(line) => {
+ if tx.try_send(line).is_err() {
+ dropped.fetch_add(1, Ordering::Relaxed);
+ LEDGER.record_dropped_event();
+ }
+ }
+ Err(_) => {
+ dropped.fetch_add(1, Ordering::Relaxed);
+ LEDGER.record_dropped_event();
+ }
+ }
+}
+
+#[cfg(unix)]
+fn process_cpu_time_ns() -> Option<u64> {
+ let mut time = libc::timespec {
+ tv_sec: 0,
+ tv_nsec: 0,
+ };
+ // SAFETY: `time` is a valid writable timespec and the process CPU clock
+ // requires no caller-managed lifetime.
+ if unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, &mut time) } != 0 {
+ return None;
+ }
+ let seconds = u64::try_from(time.tv_sec).ok()?;
+ let nanos = u64::try_from(time.tv_nsec).ok()?;
+ seconds.checked_mul(1_000_000_000)?.checked_add(nanos)
+}
+
+#[cfg(not(unix))]
+fn process_cpu_time_ns() -> Option<u64> {
+ None
+}
+
+fn writer_loop(rx: mpsc::Receiver<String>, dropped: Arc<AtomicU64>) {
+ let output =
+ std::env::var("XERJ_INGEST_MEMORY_OUTPUT").unwrap_or_else(|_| "tracing".to_string());
+ if output == "tracing" {
+ loop {
+ match rx.recv() {
+ Ok(line) => tracing::info!(target: "xerj::ingest_memory", "{line}"),
+ Err(_) => break,
+ }
+ }
+ while let Ok(line) = rx.try_recv() {
+ tracing::info!(target: "xerj::ingest_memory", "{line}");
+ }
+ return;
+ }
+
+ let mut file = match open_output(&output) {
+ Ok(file) => file,
+ Err(error) => {
+ tracing::warn!(
+ target: "xerj::ingest_memory",
+ %error,
+ path = %output,
+ "could not open ingest-memory NDJSON output"
+ );
+ return;
+ }
+ };
+ loop {
+ match rx.recv() {
+ Ok(line) => {
+ if let Err(error) = writeln!(file, "{line}") {
+ record_sink_error(&dropped, &output, &error);
+ break;
+ }
+ }
+ Err(_) => break,
+ }
+ }
+ while let Ok(line) = rx.try_recv() {
+ if let Err(error) = writeln!(file, "{line}") {
+ record_sink_error(&dropped, &output, &error);
+ break;
+ }
+ }
+}
+
+fn open_output(path: &str) -> std::io::Result<std::fs::File> {
+ let mut options = std::fs::OpenOptions::new();
+ options.create(true).append(true);
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::OpenOptionsExt;
+ // `mode` applies only when this call creates the file. An existing
+ // operator-managed file keeps its permissions and append semantics.
+ options.mode(0o600);
+ }
+ options.open(path)
+}
+
+fn record_sink_error(dropped: &AtomicU64, path: &str, error: &std::io::Error) {
+ dropped.fetch_add(1, Ordering::Relaxed);
+ LEDGER.record_dropped_event();
+ tracing::warn!(
+ target: "xerj::ingest_memory",
+ %error,
+ path,
+ "could not write ingest-memory NDJSON output; stopping file sink"
+ );
+}
+
+#[cfg(not(target_env = "msvc"))]
+fn allocator_snapshot() -> AllocatorSnapshot {
+ use tikv_jemalloc_ctl::{epoch, stats};
+ if epoch::advance().is_err() {
+ return AllocatorSnapshot::default();
+ }
+ match (
+ stats::allocated::read(),
+ stats::active::read(),
+ stats::resident::read(),
+ ) {
+ (Ok(allocated), Ok(active), Ok(resident)) => AllocatorSnapshot {
+ allocated: Some(allocated as u64),
+ active: Some(active as u64),
+ resident: Some(resident as u64),
+ epoch_ok: true,
+ },
+ _ => AllocatorSnapshot::default(),
+ }
+}
+
+#[cfg(target_env = "msvc")]
+fn allocator_snapshot() -> AllocatorSnapshot {
+ AllocatorSnapshot::default()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn bounded_sink_reports_overflow_without_blocking() {
+ let (tx, _rx) = mpsc::sync_channel(1);
+ let dropped = AtomicU64::new(0);
+ let line = r#"{"schema":"xerj.ingest_memory.v1"}"#.to_string();
+ assert!(tx.try_send(line.clone()).is_ok());
+ if tx.try_send(line).is_err() {
+ dropped.fetch_add(1, Ordering::Relaxed);
+ }
+ assert_eq!(dropped.load(Ordering::Relaxed), 1);
+ }
+
+ #[test]
+ fn allocator_probe_is_explicitly_available_or_null() {
+ let snapshot = allocator_snapshot();
+ if snapshot.epoch_ok {
+ assert!(snapshot.allocated.is_some());
+ assert!(snapshot.active.is_some());
+ assert!(snapshot.resident.is_some());
+ } else {
+ assert_eq!(snapshot.allocated, None);
+ assert_eq!(snapshot.active, None);
+ assert_eq!(snapshot.resident, None);
+ }
+ }
+
+ #[test]
+ fn process_cpu_probe_is_explicitly_available_or_null() {
+ let first = process_cpu_time_ns();
+ let second = process_cpu_time_ns();
+ match (first, second) {
+ (Some(first), Some(second)) => assert!(second >= first),
+ (None, None) => {}
+ _ => panic!("CPU-time probe support changed within one process"),
+ }
+ }
+
+ #[test]
+ fn sampler_wait_is_interruptible_even_at_max_period() {
+ let stop = Arc::new(AtomicBool::new(false));
+ let wake = Arc::new((Mutex::new(()), Condvar::new()));
+ let thread_stop = Arc::clone(&stop);
+ let thread_wake = Arc::clone(&wake);
+ let started = std::time::Instant::now();
+ let handle = std::thread::spawn(move || {
+ wait_for_stop_or_period(&thread_stop, &thread_wake, Duration::from_secs(60));
+ });
+ stop.store(true, Ordering::Release);
+ wake.1.notify_all();
+ handle.join().unwrap();
+ assert!(started.elapsed() < Duration::from_secs(1));
+ }
+
+ #[test]
+ fn stop_refresh_observes_changed_gauge_without_waiting_for_period() {
+ LEDGER.observe(ingest_memory::Category::MemtableActive, 8192);
+
+ let started = std::time::Instant::now();
+ refresh_snapshot_state_with(
+ 0,
+ AllocatorSnapshot {
+ allocated: Some(11),
+ active: Some(12),
+ resident: Some(13),
+ epoch_ok: true,
+ },
+ 14,
+ Some(15),
+ );
+
+ assert!(started.elapsed() < Duration::from_secs(1));
+ let memtable = LEDGER.gauge(
+ ingest_memory::Category::MemtableActive,
+ ingest_memory::Measurement::Estimated,
+ );
+ assert_eq!(memtable.current, 0);
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn output_file_is_private_when_created_and_preserves_existing_mode() {
+ use std::os::unix::fs::PermissionsExt;
+
+ let dir = tempfile::tempdir().unwrap();
+ let created = dir.path().join("created.ndjson");
+ drop(open_output(created.to_str().unwrap()).unwrap());
+ assert_eq!(
+ std::fs::metadata(&created).unwrap().permissions().mode() & 0o777,
+ 0o600
+ );
+
+ let existing = dir.path().join("existing.ndjson");
+ std::fs::write(&existing, b"existing\n").unwrap();
+ std::fs::set_permissions(&existing, std::fs::Permissions::from_mode(0o640)).unwrap();
+ let mut file = open_output(existing.to_str().unwrap()).unwrap();
+ writeln!(file, "appended").unwrap();
+ drop(file);
+ assert_eq!(
+ std::fs::metadata(&existing).unwrap().permissions().mode() & 0o777,
+ 0o640
+ );
+ assert_eq!(
+ std::fs::read_to_string(existing).unwrap(),
+ "existing\nappended\n"
+ );
+ }
+
+ #[cfg(target_os = "linux")]
+ #[test]
+ fn output_write_failure_is_accounted() {
+ let mut file = open_output("/dev/full").unwrap();
+ let dropped = AtomicU64::new(0);
+ let error = writeln!(file, "cannot fit").unwrap_err();
+ record_sink_error(&dropped, "/dev/full", &error);
+ assert_eq!(dropped.load(Ordering::Relaxed), 1);
+ }
+}
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index 5f8814b81..e577e6132 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -77,6 +77,7 @@ use xerj_engine::Engine;
#[cfg(all(target_os = "linux", feature = "debug-profiling"))]
mod debug_profiling;
mod grpc;
+mod ingest_memory_trace;
// ─────────────────────────────────────────────────────────────────────────────
// CLI
@@ -209,6 +210,12 @@ fn print_help() {
XERJ_ONNX_TOKENIZER matching tokenizer.json path\n\
XERJ_COMPAT_DISTRIBUTION elasticsearch|opensearch — same as --compat-distribution\n\
XERJ_COMPAT_VERSION same as --compat-version\n\
+ XERJ_INGEST_MEMORY_TRACE off|summary (default: off); summary emits bounded\n\
+ periodic NDJSON diagnostics, never per-item events\n\
+ XERJ_INGEST_MEMORY_SAMPLE_MS sampler period, clamped to 25..60000 ms\n\
+ XERJ_INGEST_MEMORY_OUTPUT tracing|<PATH> (default: tracing); append NDJSON to path\n\
+ Merge/cache fields are reserved/unavailable in v1.\n\
+ Active+drained may transiently overlap across sampling.\n\
\n\
WIRE-COMPAT IDENTITY — three ways to run this, pick what fits:\n\
\n\
@@ -1482,6 +1489,8 @@ async fn async_main() -> Result<()> {
// is the structural guard against the 112 GiB OOM class — writes get
// a 429 circuit_breaking_exception before the kernel OOM-kills us.
state.engine.spawn_resource_sampler();
+
+ let _ingest_memory_trace = ingest_memory_trace::spawn(&state.engine);
// 13. Start servers concurrently
let rest_tls = tls_config.clone();
let rest = tokio::spawn(async move {