AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(engine): add opt-in ingest memory attribution
buger43397f03344d
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 6f0e1b58c..59b1e5064 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -6452,6 +6452,7 @@ dependencies = ["chrono","flate2","jemalloc_pprof",+ "libc","memchr","memmap2","pprof",@@ -6465,6 +6466,7 @@ dependencies = ["serde","serde_json","tempfile",+ "tikv-jemalloc-ctl","tikv-jemallocator","tokio","tonic",diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex aa9ee9f93..384502028 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -12130,6 +12130,13 @@ async fn process_bulk_body(_started: Instant,opts: xerj_engine::bulk::BulkOpts,) -> axum::response::Response {+ // The request body remains owned by this handler through parsing,+ // grouping, and publication. The opt-in guard is a single branch and+ // performs no atomic work when tracing is disabled.+ let _http_body_memory = xerj_engine::ingest_memory::Retained::new(+ xerj_engine::ingest_memory::Category::HttpBody,+ body.len(),+ );let text = match std::str::from_utf8(body) {Ok(t) => t,Err(_) => {@@ -12151,9 +12158,16 @@ async fn process_bulk_body(// dimensions and the instant are identical, matching ES. Returns `None`// when no action targets a TSDB index, so normal bulk bodies are unchanged.let ts_rewritten = rewrite_bulk_time_series_ids(state, default_index, text);+ let _ts_rewritten_memory = ts_rewritten.as_ref().map(|replacement| {+ xerj_engine::ingest_memory::Retained::new(+ xerj_engine::ingest_memory::Category::HttpRewriteBuffer,+ replacement.capacity(),+ )+ });let base_text: &str = ts_rewritten.as_deref().unwrap_or(text);let rewritten_body;+ let mut rewritten_capacity = None;// P2.2 — the ignore_malformed rewrite parses + re-serialises EVERY// source line serially (client-blocking). `apply_ignore_malformed`// can only modify a doc when some field in the target mapping carries@@ -12170,8 +12184,15 @@ async fn process_bulk_body(base_text} else {rewritten_body = rewrite_bulk_ignore_malformed(state, default_index, base_text);+ rewritten_capacity = Some(rewritten_body.capacity());&rewritten_body};+ let _ignore_rewritten_memory = rewritten_capacity.map(|capacity| {+ xerj_engine::ingest_memory::Retained::new(+ xerj_engine::ingest_memory::Category::HttpRewriteBuffer,+ capacity,+ )+ });let result =xerj_engine::bulk::process_bulk_with_opts(&state.engine, default_index, text_ref, opts)diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex 822b9f209..d3d1ca908 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -15,6 +15,103 @@ use crate::engine::Engine;use crate::index::IndexResponse;use crate::{EngineError, Result};+#[cfg(test)]+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub(crate) enum SemanticLifecyclePoint {+ Grouped,+ Parsed,+ Prepared,+ EmbeddingInside,+ EmbeddingItemInside,+ BeforePublication,+ PublicationInside,+ AfterPublication,+}++#[cfg(test)]+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub(crate) enum SemanticLifecycleFault {+ None,+ Embedding,+ Publication,+}++#[cfg(test)]+type SemanticLifecycleHook = std::sync::Arc<+ dyn Fn(&str, SemanticLifecyclePoint) -> SemanticLifecycleFault + Send + Sync + 'static,+>;++#[cfg(test)]+tokio::task_local! {+ static SEMANTIC_LIFECYCLE_HOOK: (String, SemanticLifecycleHook);+ static DISABLED_SEMANTIC_COUNTERS: std::sync::Arc<DisabledSemanticCounters>;+}++#[cfg(test)]+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]+struct DisabledSemanticInstrumentation {+ schema_reads: usize,+ guard_vectors: usize,+ sizing_walks: usize,+}++#[cfg(test)]+#[derive(Default)]+struct DisabledSemanticCounters {+ schema_reads: std::sync::atomic::AtomicUsize,+ guard_vectors: std::sync::atomic::AtomicUsize,+ sizing_walks: std::sync::atomic::AtomicUsize,+}++#[cfg(test)]+impl DisabledSemanticCounters {+ fn snapshot(&self) -> DisabledSemanticInstrumentation {+ use std::sync::atomic::Ordering;+ DisabledSemanticInstrumentation {+ schema_reads: self.schema_reads.load(Ordering::Relaxed),+ guard_vectors: self.guard_vectors.load(Ordering::Relaxed),+ sizing_walks: self.sizing_walks.load(Ordering::Relaxed),+ }+ }+}++#[cfg(test)]+fn count_disabled_semantic_access(+ select: fn(&DisabledSemanticCounters) -> &std::sync::atomic::AtomicUsize,+) {+ use std::sync::atomic::Ordering;+ let _ = DISABLED_SEMANTIC_COUNTERS.try_with(|counters| {+ select(counters).fetch_add(1, Ordering::Relaxed);+ });+}++#[cfg(test)]+pub(crate) fn semantic_lifecycle_hook(+ target: &str,+ point: SemanticLifecyclePoint,+) -> SemanticLifecycleFault {+ SEMANTIC_LIFECYCLE_HOOK+ .try_with(|(hook_target, hook)| {+ if hook_target == "*" || hook_target == target {+ hook(target, point)+ } else {+ SemanticLifecycleFault::None+ }+ })+ .unwrap_or(SemanticLifecycleFault::None)+}++#[cfg(test)]+async fn with_semantic_lifecycle_hook<F: std::future::Future>(+ target: impl Into<String>,+ hook: SemanticLifecycleHook,+ future: F,+) -> F::Output {+ SEMANTIC_LIFECYCLE_HOOK+ .scope((target.into(), hook), future)+ .await+}+// ── Types ─────────────────────────────────────────────────────────────────────#[derive(Debug, Clone, Serialize, Deserialize)]@@ -124,6 +221,30 @@ pub async fn process_bulk(engine: &Engine, default_index: Option<&str>, body: &sprocess_bulk_with_opts(engine, default_index, body, BulkOpts::default()).await}+fn release_raw_owner(+ owners: &mut std::collections::HashMap<usize, (usize, crate::ingest_memory::Retained<'static>)>,+ source: std::sync::Arc<[u8]>,+) {+ let owner = std::sync::Arc::as_ptr(&source) as *const () as usize;+ // The logical owner must outlive every Arc clone. Dropping the ledger+ // token first would publish a false zero while this clone still retains+ // the backing allocation.+ drop(source);+ release_raw_owner_by_id(owners, owner);+}++fn release_raw_owner_by_id(+ owners: &mut std::collections::HashMap<usize, (usize, crate::ingest_memory::Retained<'static>)>,+ owner: usize,+) {+ if let Some((remaining, _)) = owners.get_mut(&owner) {+ *remaining -= 1;+ if *remaining == 0 {+ owners.remove(&owner);+ }+ }+}+#[derive(Clone, Default)]pub struct BulkOpts {pub require_alias: Option<bool>,@@ -1014,11 +1135,40 @@ pub async fn process_bulk_with_opts(// still used after preparation; it observes the supplied companion vector// and therefore preserves WAL, overwrite status, response ordering, and// item-level errors.+ let mut semantic_raw_memory = crate::ingest_memory::enabled().then(+ std::collections::HashMap::<usize, (usize, crate::ingest_memory::Retained<'static>)>::new,+ );+ if let Some(raw_memory) = semantic_raw_memory.as_mut() {+ for batch in semantic_index_batches.values() {+ for (_, _, source) in batch {+ let owner = std::sync::Arc::as_ptr(source) as *const () as usize;+ raw_memory+ .entry(owner)+ .and_modify(|(remaining, _)| *remaining += 1)+ .or_insert_with(|| {+ (+ 1usize,+ crate::ingest_memory::Retained::new(+ crate::ingest_memory::Category::RawSource,+ source.len(),+ ),+ )+ });+ }+ }+ }for (index_name, batch) in semantic_index_batches {+ #[cfg(test)]+ let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::Grouped);let idx = match engine.get_or_create_index(&index_name) {Ok(i) => i,Err(e) => {- for (item_idx, id, _) in batch {+ for (item_idx, id, source) in batch {+ if let Some(raw_memory) = semantic_raw_memory.as_mut() {+ release_raw_owner(raw_memory, source);+ } else {+ drop(source);+ }items[item_idx] = Some(BulkItemResult {action: "index".into(),index: index_name.clone(),@@ -1033,16 +1183,55 @@ pub async fn process_bulk_with_opts(continue;}};+ let vector_fields = if crate::ingest_memory::enabled() {+ #[cfg(test)]+ count_disabled_semantic_access(|counters| &counters.schema_reads);+ idx.semantic_vector_fields().await+ } else {+ std::collections::HashSet::new()+ };let mut valid_meta = Vec::with_capacity(batch.len());let mut valid_sources = Vec::with_capacity(batch.len());+ let mut valid_memory = crate::ingest_memory::enabled().then(|| {+ #[cfg(test)]+ count_disabled_semantic_access(|counters| &counters.guard_vectors);+ Vec::with_capacity(batch.len())+ });for (item_idx, id, bytes) in batch {- match serde_json::from_slice::<Value>(&bytes) {+ let parsed = serde_json::from_slice::<Value>(&bytes);+ match parsed {Ok(source) => {+ if let Some(valid_memory) = valid_memory.as_mut() {+ #[cfg(test)]+ count_disabled_semantic_access(|counters| &counters.sizing_walks);+ let parsed_document = crate::ingest_memory::Retained::new(+ crate::ingest_memory::Category::ParsedSource,+ crate::ingest_memory::estimated_json_heap_excluding(+ &source,+ &vector_fields,+ ),+ );+ let parsed_vector = crate::ingest_memory::Retained::new(+ crate::ingest_memory::Category::ParsedVector,+ crate::ingest_memory::vector_value_capacity(&source, &vector_fields),+ );+ valid_memory.push((parsed_document, parsed_vector));+ }+ if let Some(raw_memory) = semantic_raw_memory.as_mut() {+ release_raw_owner(raw_memory, bytes);+ } else {+ drop(bytes);+ }valid_meta.push((item_idx, id));valid_sources.push(source);}Err(e) => {+ if let Some(raw_memory) = semantic_raw_memory.as_mut() {+ release_raw_owner(raw_memory, bytes);+ } else {+ drop(bytes);+ }items[item_idx] = Some(BulkItemResult {action: "index".into(),index: index_name.clone(),@@ -1056,12 +1245,49 @@ pub async fn process_bulk_with_opts(}}}+ #[cfg(test)]+ let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::Parsed);+ // Backend-owned chunk strings, tokenizer tensors, and inference+ // scratch are not yet logically attributed. Their effect remains+ // visible only as allocator/RSS physical delta; it is not zero.let prepared = idx.apply_semantic_embeddings_batch(valid_sources).await;+ #[cfg(test)]+ let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::Prepared);+ let mut valid_memory = valid_memory.map(Vec::into_iter);for ((item_idx, id), prepared_source) in valid_meta.into_iter().zip(prepared) {- let source = match prepared_source {- Ok(source) => source,+ let memory = valid_memory.as_mut().and_then(Iterator::next);+ let (source, prepared_memory) = match prepared_source {+ Ok(source) => {+ let prepared_memory = if let Some((parsed_document, parsed_vector)) = memory {+ #[cfg(test)]+ count_disabled_semantic_access(|counters| &counters.sizing_walks);+ let prepared_document_bytes =+ crate::ingest_memory::estimated_json_heap_excluding(+ &source,+ &vector_fields,+ );+ let prepared_vector_bytes =+ crate::ingest_memory::vector_value_capacity(&source, &vector_fields);+ Some(crate::ingest_memory::Retained::replace_pair(+ parsed_document,+ parsed_vector,+ (+ crate::ingest_memory::Category::PreparedDocument,+ prepared_document_bytes,+ ),+ (+ crate::ingest_memory::Category::PreparedVector,+ prepared_vector_bytes,+ ),+ ))+ } else {+ None+ };+ (source, prepared_memory)+ }Err(e) => {+ drop(memory);let status = engine_error_http_status(&e);items[item_idx] = Some(BulkItemResult {action: "index".into(),@@ -1077,7 +1303,13 @@ pub async fn process_bulk_with_opts(}};let error_id = id.clone().unwrap_or_default();- match idx.index_document_prepared(id, source).await {+ #[cfg(test)]+ let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::BeforePublication);+ let publication = idx.index_document_prepared(id, source).await;+ drop(prepared_memory);+ #[cfg(test)]+ let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::AfterPublication);+ match publication {Ok(resp) => {let status = if resp.result == "updated" { 200 } else { 201 };items[item_idx] = Some(BulkItemResult {@@ -2269,3 +2501,700 @@ async fn execute_bulk_action()),}}++#[cfg(test)]+mod semantic_lifecycle_tests {+ use super::*;+ use crate::ingest_memory::{Category, Ledger, Measurement};+ use serde_json::json;+ use std::sync::{Arc, Mutex};+ use tempfile::TempDir;+ use xerj_common::config::Config;+ use xerj_common::types::{EmbeddingConfig, FieldConfig, FieldType, Schema};++ fn semantic_engine(name: &str) -> (TempDir, Engine) {+ semantic_engine_with_targets(name, &[("content", "content_vector")])+ }++ fn semantic_engine_with_targets(name: &str, targets: &[(&str, &str)]) -> (TempDir, Engine) {+ let dir = TempDir::new().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = Engine::new(config).unwrap();+ let mut schema = Schema::empty();+ for (field, target) in targets {+ let mut config = FieldConfig::new(*field, FieldType::Text);+ config.options.dimensions = Some(16);+ config.embedding = Some(EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: Some((*target).into()),+ });+ schema.fields.push(config);+ }+ engine.create_index(name, schema).unwrap();+ let properties = targets+ .iter()+ .map(|(field, _)| ((*field).to_string(), json!({"type": "semantic_text"})))+ .collect::<serde_json::Map<_, _>>();+ engine.put_index_mapping(name, json!({"properties": properties}));+ (dir, engine)+ }++ fn current(ledger: &Ledger, category: Category) -> u64 {+ ledger.gauge(category, Measurement::Estimated).current+ }++ fn all_semantic_current(ledger: &Ledger) -> u64 {+ [+ Category::RawSource,+ Category::ParsedSource,+ Category::ParsedVector,+ Category::PreparedDocument,+ Category::PreparedVector,+ ]+ .into_iter()+ .map(|category| current(ledger, category))+ .sum()+ }++ #[test]+ fn raw_owner_token_outlives_every_arc_occurrence() {+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));+ let source: Arc<[u8]> = Arc::from(&b"{\"content\":\"retained\"}"[..]);+ let owner = Arc::as_ptr(&source) as *const () as usize;+ let mut owners = std::collections::HashMap::from([(+ owner,+ (+ 2,+ crate::ingest_memory::Retained::for_ledger(+ ledger,+ Category::RawSource,+ source.len(),+ ),+ ),+ )]);+ let second = Arc::clone(&source);++ release_raw_owner(&mut owners, second);+ assert_eq!(Arc::strong_count(&source), 1);+ assert_eq!(owners.get(&owner).map(|entry| entry.0), Some(1));+ assert_eq!(current(ledger, Category::RawSource), source.len() as u64);++ release_raw_owner(&mut owners, source);+ assert!(owners.is_empty());+ assert_eq!(current(ledger, Category::RawSource), 0);+ }++ async fn run_case(+ name: &str,+ fault: SemanticLifecycleFault,+ ) -> (+ BulkResult,+ &'static Ledger,+ Vec<(SemanticLifecyclePoint, [u64; 5])>,+ ) {+ let (_dir, engine) = semantic_engine_with_targets(+ name,+ &[("content", "content_vector"), ("summary", "summary_vector")],+ );+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));+ // Zero-capacity rendezvous channels make every hook a genuine phase+ // barrier: the ingest task cannot advance until the observer has read+ // the live gauges and explicitly released it.+ let expected = if fault == SemanticLifecycleFault::Embedding {+ 4+ } else {+ 8+ };+ let (observation_tx, observation_rx) = std::sync::mpsc::sync_channel(0);+ let (resume_tx, resume_rx) = std::sync::mpsc::sync_channel(0);+ let resume_rx = Arc::new(Mutex::new(resume_rx));+ let observer = std::thread::spawn(move || {+ let mut points = Vec::with_capacity(expected);+ for _ in 0..expected {+ points.push(observation_rx.recv().unwrap());+ resume_tx.send(()).unwrap();+ }+ points+ });+ let hook: SemanticLifecycleHook = Arc::new(move |_, point| {+ observation_tx+ .send((+ point,+ [+ current(ledger, Category::RawSource),+ current(ledger, Category::ParsedSource),+ current(ledger, Category::ParsedVector),+ current(ledger, Category::PreparedDocument),+ current(ledger, Category::PreparedVector),+ ],+ ))+ .unwrap();+ resume_rx.lock().unwrap().recv().unwrap();+ match (fault, point) {+ (SemanticLifecycleFault::Embedding, SemanticLifecyclePoint::EmbeddingInside) => {+ fault+ }+ (+ SemanticLifecycleFault::Publication,+ SemanticLifecyclePoint::PublicationInside,+ ) => fault,+ _ => SemanticLifecycleFault::None,+ }+ });+ let body = format!(+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"one\"}}}}\n\+ {{\"content\":\"caller supplied vector\",\+ \"content_vector\":[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,\+ 0.9,1.0,1.1,1.2,1.3,1.4,1.5,1.6],\+ \"summary\":\"a genuine semantic lifecycle embedding job\"}}\n"+ );+ let result = crate::ingest_memory::with_test_ledger(+ ledger,+ with_semantic_lifecycle_hook(name, hook, process_bulk(&engine, None, &body)),+ )+ .await;+ let points = observer.join().unwrap();+ (result, ledger, points)+ }++ #[tokio::test]+ async fn semantic_lifecycle_success_balances_isolated_ledger() {+ let (result, ledger, points) =+ run_case("semantic-lifecycle-success", SemanticLifecycleFault::None).await;+ assert!(!result.errors, "{:?}", result.items);+ let phase_points: Vec<_> = points.iter().map(|(point, _)| *point).collect();+ assert_eq!(+ phase_points,+ [+ SemanticLifecyclePoint::Grouped,+ SemanticLifecyclePoint::Parsed,+ SemanticLifecyclePoint::EmbeddingInside,+ SemanticLifecyclePoint::EmbeddingItemInside,+ SemanticLifecyclePoint::Prepared,+ SemanticLifecyclePoint::BeforePublication,+ SemanticLifecyclePoint::PublicationInside,+ SemanticLifecyclePoint::AfterPublication,+ ]+ );+ let at = |point| points.iter().find(|(p, _)| *p == point).unwrap().1;+ assert!(at(SemanticLifecyclePoint::Grouped)[0] > 0);+ assert_eq!(at(SemanticLifecyclePoint::Parsed)[0], 0);+ assert!(at(SemanticLifecyclePoint::Parsed)[1] > 0);+ assert!(at(SemanticLifecyclePoint::EmbeddingInside)[1] > 0);+ assert!(+ at(SemanticLifecyclePoint::EmbeddingInside)[2] > 0,+ "caller-supplied vector ownership must remain live while embedding is blocked"+ );+ assert_eq!(at(SemanticLifecyclePoint::BeforePublication)[1], 0);+ assert!(at(SemanticLifecyclePoint::BeforePublication)[3] > 0);+ assert!(at(SemanticLifecyclePoint::BeforePublication)[4] > 0);+ assert_eq!(at(SemanticLifecyclePoint::AfterPublication), [0; 5]);+ assert!(current(ledger, Category::RawSource) == 0);+ assert!(+ ledger+ .gauge(Category::RawSource, Measurement::SerializedSize)+ .peak+ > 0+ );+ assert!(+ ledger+ .gauge(Category::ParsedSource, Measurement::Estimated)+ .peak+ > 0+ );+ assert!(+ ledger+ .gauge(Category::PreparedDocument, Measurement::Estimated)+ .peak+ > 0+ );+ assert!(+ ledger+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)+ .peak+ > 0+ );+ assert_eq!(all_semantic_current(ledger), 0);+ assert_eq!(ledger.accounting_errors(), 0);+ assert_eq!(all_semantic_current(&crate::ingest_memory::LEDGER), 0);+ }++ #[tokio::test]+ async fn semantic_lifecycle_embedding_error_releases_parsed_ownership() {+ let (result, ledger, points) = run_case(+ "semantic-lifecycle-embed-error",+ SemanticLifecycleFault::Embedding,+ )+ .await;+ assert!(result.errors);+ assert_eq!(result.items[0].status, 400);+ let phase_points: Vec<_> = points.iter().map(|(point, _)| *point).collect();+ assert_eq!(+ phase_points,+ [+ SemanticLifecyclePoint::Grouped,+ SemanticLifecyclePoint::Parsed,+ SemanticLifecyclePoint::EmbeddingInside,+ SemanticLifecyclePoint::Prepared,+ ]+ );+ let inside = points+ .iter()+ .find(|(point, _)| *point == SemanticLifecyclePoint::EmbeddingInside)+ .unwrap()+ .1;+ assert!(inside[1] > 0);+ assert!(+ inside[2] > 0,+ "caller-supplied vector ownership must remain live while embedding is blocked"+ );+ assert!(+ ledger+ .gauge(Category::ParsedSource, Measurement::Estimated)+ .peak+ > 0+ );+ assert_eq!(all_semantic_current(ledger), 0);+ assert_eq!(ledger.accounting_errors(), 0);+ }++ #[tokio::test]+ async fn semantic_lifecycle_publication_error_releases_prepared_ownership() {+ let (result, ledger, points) = run_case(+ "semantic-lifecycle-publication-error",+ SemanticLifecycleFault::Publication,+ )+ .await;+ assert!(result.errors);+ assert_eq!(result.items[0].status, 400);+ let phase_points: Vec<_> = points.iter().map(|(point, _)| *point).collect();+ assert_eq!(+ phase_points,+ [+ SemanticLifecyclePoint::Grouped,+ SemanticLifecyclePoint::Parsed,+ SemanticLifecyclePoint::EmbeddingInside,+ SemanticLifecyclePoint::EmbeddingItemInside,+ SemanticLifecyclePoint::Prepared,+ SemanticLifecyclePoint::BeforePublication,+ SemanticLifecyclePoint::PublicationInside,+ SemanticLifecyclePoint::AfterPublication,+ ]+ );+ let inside = points+ .iter()+ .find(|(point, _)| *point == SemanticLifecyclePoint::PublicationInside)+ .unwrap()+ .1;+ assert!(inside[3] > 0);+ assert!(inside[4] > 0);+ assert!(+ ledger+ .gauge(Category::PreparedDocument, Measurement::Estimated)+ .peak+ > 0+ );+ assert!(+ ledger+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)+ .peak+ > 0+ );+ assert_eq!(all_semantic_current(ledger), 0);+ assert_eq!(ledger.accounting_errors(), 0);+ }++ #[tokio::test]+ async fn semantic_lifecycle_hook_scope_is_panic_safe_and_target_specific() {+ let (_dir, engine) = semantic_engine("semantic-lifecycle-panic");+ let engine = Arc::new(engine);+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));+ let hook: SemanticLifecycleHook = Arc::new(|_, point| {+ if point == SemanticLifecyclePoint::Grouped {+ panic!("deliberate lifecycle hook panic");+ }+ SemanticLifecycleFault::None+ });+ let body = concat!(+ "{\"index\":{\"_index\":\"semantic-lifecycle-panic\",\"_id\":\"one\"}}\n",+ "{\"content\":\"panic cleanup\"}\n",+ );+ let panic_engine = Arc::clone(&engine);+ let task = tokio::spawn(async move {+ crate::ingest_memory::with_test_ledger(+ ledger,+ with_semantic_lifecycle_hook(+ "semantic-lifecycle-panic",+ hook,+ process_bulk(&panic_engine, None, body),+ ),+ )+ .await+ });+ assert!(task.await.unwrap_err().is_panic());+ assert_eq!(all_semantic_current(ledger), 0);++ let other_ledger = Box::leak(Box::new(Ledger::new()));+ let rejecting_hook: SemanticLifecycleHook =+ Arc::new(|_, _| SemanticLifecycleFault::Publication);+ let result = crate::ingest_memory::with_test_ledger(+ other_ledger,+ with_semantic_lifecycle_hook(+ "different-target",+ rejecting_hook,+ process_bulk(&engine, None, body),+ ),+ )+ .await;+ assert!(+ !result.errors,+ "target-specific hook leaked onto another target"+ );+ assert_eq!(all_semantic_current(other_ledger), 0);+ assert_eq!(all_semantic_current(&crate::ingest_memory::LEDGER), 0);+ }++ #[tokio::test]+ async fn semantic_lifecycle_declines_after_each_sequential_publication() {+ let name = "semantic-lifecycle-sequential";+ let (_dir, engine) = semantic_engine(name);+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));+ let observations = Arc::new(Mutex::new(Vec::<(SemanticLifecyclePoint, u64, u64)>::new()));+ let captured = Arc::clone(&observations);+ let embedded = Arc::new(std::sync::atomic::AtomicBool::new(false));+ let embedded_hook = Arc::clone(&embedded);+ let hook: SemanticLifecycleHook = Arc::new(move |_, point| {+ if point == SemanticLifecyclePoint::EmbeddingInside {+ embedded_hook.store(true, std::sync::atomic::Ordering::Relaxed);+ }+ if matches!(+ point,+ SemanticLifecyclePoint::BeforePublication+ | SemanticLifecyclePoint::AfterPublication+ ) {+ captured.lock().unwrap().push((+ point,+ current(ledger, Category::PreparedDocument),+ current(ledger, Category::PreparedVector),+ ));+ }+ SemanticLifecycleFault::None+ });+ let body = (0..3)+ .map(|i| {+ format!(+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"{i}\"}}}}\n\+ {{\"content\":\"semantic document number {i}\"}}\n"+ )+ })+ .collect::<String>();+ let result = crate::ingest_memory::with_test_ledger(+ ledger,+ with_semantic_lifecycle_hook(name, hook, process_bulk(&engine, None, &body)),+ )+ .await;+ assert!(!result.errors, "{:?}", result.items);+ assert!(embedded.load(std::sync::atomic::Ordering::Relaxed));+ let observations = observations.lock().unwrap();+ assert_eq!(observations.len(), 6);+ for pair in observations.chunks_exact(2) {+ assert_eq!(pair[0].0, SemanticLifecyclePoint::BeforePublication);+ assert_eq!(pair[1].0, SemanticLifecyclePoint::AfterPublication);+ assert!(pair[0].1 > pair[1].1);+ assert!(pair[0].2 > pair[1].2);+ }+ for pair in observations.chunks_exact(2) {+ assert!(pair[0].1 > 0);+ assert!(pair[0].2 > 0);+ assert_eq!(pair[1].1, 0, "published document ownership stayed charged");+ assert_eq!(pair[1].2, 0, "published vector ownership stayed charged");+ }+ assert_eq!(observations.last().unwrap().1, 0);+ assert_eq!(observations.last().unwrap().2, 0);+ assert_eq!(all_semantic_current(ledger), 0);+ assert_eq!(ledger.accounting_errors(), 0);+ }++ #[tokio::test]+ async fn semantic_lifecycle_mixed_publication_outputs_balance_per_item() {+ let name = "semantic-lifecycle-mixed";+ let (_dir, engine) = semantic_engine(name);+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));+ let publication = Arc::new(std::sync::atomic::AtomicUsize::new(0));+ let publication_hook = Arc::clone(&publication);+ let points = Arc::new(Mutex::new(Vec::new()));+ let captured_points = Arc::clone(&points);+ let hook: SemanticLifecycleHook = Arc::new(move |_, point| {+ captured_points.lock().unwrap().push(point);+ if point == SemanticLifecyclePoint::PublicationInside+ && publication_hook.fetch_add(1, std::sync::atomic::Ordering::Relaxed) == 1+ {+ SemanticLifecycleFault::Publication+ } else {+ SemanticLifecycleFault::None+ }+ });+ let body = (0..3)+ .map(|i| {+ format!(+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"{i}\"}}}}\n\+ {{\"content\":\"mixed output {i}\"}}\n"+ )+ })+ .collect::<String>();+ let result = crate::ingest_memory::with_test_ledger(+ ledger,+ with_semantic_lifecycle_hook(name, hook, process_bulk(&engine, None, &body)),+ )+ .await;+ assert!(result.errors);+ assert_eq!(+ result+ .items+ .iter()+ .map(|item| item.status)+ .collect::<Vec<_>>(),+ [201, 400, 201]+ );+ let points = points.lock().unwrap();+ assert!(points.contains(&SemanticLifecyclePoint::EmbeddingInside));+ assert_eq!(+ points+ .iter()+ .filter(|point| **point == SemanticLifecyclePoint::PublicationInside)+ .count(),+ 3+ );+ assert_eq!(all_semantic_current(ledger), 0);+ assert_eq!(ledger.accounting_errors(), 0);+ }++ #[tokio::test]+ async fn semantic_lifecycle_mixed_embedding_outputs_balance_per_item() {+ let name = "semantic-lifecycle-mixed-embedding";+ let (_dir, engine) = semantic_engine(name);+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));+ let item = Arc::new(std::sync::atomic::AtomicUsize::new(0));+ let item_hook = Arc::clone(&item);+ let points = Arc::new(Mutex::new(Vec::new()));+ let captured_points = Arc::clone(&points);+ let hook: SemanticLifecycleHook = Arc::new(move |_, point| {+ captured_points.lock().unwrap().push((+ point,+ current(ledger, Category::ParsedSource),+ current(ledger, Category::ParsedVector),+ ));+ if point == SemanticLifecyclePoint::EmbeddingItemInside+ && item_hook.fetch_add(1, std::sync::atomic::Ordering::Relaxed) == 1+ {+ SemanticLifecycleFault::Embedding+ } else {+ SemanticLifecycleFault::None+ }+ });+ let body = (0..3)+ .map(|i| {+ format!(+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"{i}\"}}}}\n\+ {{\"content\":\"mixed embedding output {i}\"}}\n"+ )+ })+ .collect::<String>();+ let result = crate::ingest_memory::with_test_ledger(+ ledger,+ with_semantic_lifecycle_hook(name, hook, process_bulk(&engine, None, &body)),+ )+ .await;+ assert!(result.errors);+ assert_eq!(+ result+ .items+ .iter()+ .map(|item| item.status)+ .collect::<Vec<_>>(),+ [201, 500, 201]+ );+ let points = points.lock().unwrap();+ assert!(points.iter().any(|(point, parsed, _)| *point+ == SemanticLifecyclePoint::EmbeddingInside+ && *parsed > 0));+ assert_eq!(+ points+ .iter()+ .filter(|(point, _, _)| *point == SemanticLifecyclePoint::EmbeddingItemInside)+ .count(),+ 3+ );+ assert_eq!(+ points+ .iter()+ .filter(|(point, _, _)| *point == SemanticLifecyclePoint::PublicationInside)+ .count(),+ 2+ );+ assert_eq!(all_semantic_current(ledger), 0);+ assert_eq!(ledger.accounting_errors(), 0);+ }++ #[tokio::test]+ async fn semantic_lifecycle_covers_malformed_caller_vectors_and_literal_dotted_target() {+ let name = "semantic-lifecycle-input-shapes";+ let (_dir, engine) =+ semantic_engine_with_targets(name, &[("content", "custom.vector"), ("summary", "v2")]);+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));+ let points = Arc::new(Mutex::new(Vec::new()));+ let captured_points = Arc::clone(&points);+ let hook: SemanticLifecycleHook = Arc::new(move |_, point| {+ captured_points.lock().unwrap().push(point);+ SemanticLifecycleFault::None+ });+ let body = format!(+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"bad\"}}}}\n\+ {{\"content\":\n\+ {{\"index\":{{\"_index\":\"{name}\",\"_id\":\"supplied\"}}}}\n\+ {{\"content\":\"caller vector\",\"custom.vector\":[0.1,0.2,0.3],\+ \"summary\":\"second target\",\"v2\":[0.4,0.5]}}\n\+ {{\"index\":{{\"_index\":\"{name}\",\"_id\":\"derived\"}}}}\n\+ {{\"content\":\"derive literal dotted target\",\"summary\":\"derive second\"}}\n"+ );+ let result = crate::ingest_memory::with_test_ledger(+ ledger,+ with_semantic_lifecycle_hook(name, hook, process_bulk(&engine, None, &body)),+ )+ .await;+ assert!(result.errors);+ assert_eq!(+ result+ .items+ .iter()+ .map(|item| item.status)+ .collect::<Vec<_>>(),+ [400, 201, 201]+ );+ assert!(+ ledger+ .gauge(Category::ParsedVector, Measurement::ExactCapacity)+ .peak+ > 0+ );+ let points = points.lock().unwrap();+ assert!(points.contains(&SemanticLifecyclePoint::EmbeddingInside));+ assert_eq!(+ points+ .iter()+ .filter(|point| **point == SemanticLifecyclePoint::PublicationInside)+ .count(),+ 2+ );+ assert!(+ ledger+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)+ .peak+ > 0+ );+ assert_eq!(all_semantic_current(ledger), 0);+ assert_eq!(ledger.accounting_errors(), 0);+ }++ #[tokio::test]+ async fn semantic_lifecycle_multiple_targets_and_index_creation_failure_balance() {+ let ok = "semantic-lifecycle-target-a";+ let fails = "semantic-lifecycle-target-b";+ let (dir, engine) = semantic_engine(ok);+ let mut schema = Schema::empty();+ let mut content = FieldConfig::new("content", FieldType::Text);+ content.options.dimensions = Some(16);+ content.embedding = Some(EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: Some("content_vector".into()),+ });+ schema.fields.push(content);+ engine.create_index(fails, schema).unwrap();+ engine.put_index_mapping(+ fails,+ json!({"properties": {"content": {"type": "semantic_text"}}}),+ );+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));+ let engine = Arc::new(engine);+ let hook_engine = Arc::clone(&engine);+ let collision = dir.path().join(fails);+ let points = Arc::new(Mutex::new(Vec::new()));+ let captured_points = Arc::clone(&points);+ let hook: SemanticLifecycleHook = Arc::new(move |target, point| {+ captured_points+ .lock()+ .unwrap()+ .push((target.to_string(), point));+ if target == fails && point == SemanticLifecyclePoint::Grouped {+ let engine = Arc::clone(&hook_engine);+ std::thread::spawn(move || {+ tokio::runtime::Runtime::new()+ .unwrap()+ .block_on(engine.delete_index(fails))+ .unwrap();+ })+ .join()+ .unwrap();+ std::fs::write(&collision, b"blocks index directory creation").unwrap();+ }+ SemanticLifecycleFault::None+ });+ let body = format!(+ "{{\"index\":{{\"_index\":\"{ok}\",\"_id\":\"ok\"}}}}\n\+ {{\"content\":\"first target\"}}\n\+ {{\"index\":{{\"_index\":\"{fails}\",\"_id\":\"fail\"}}}}\n\+ {{\"content\":\"second target\"}}\n"+ );+ let result = crate::ingest_memory::with_test_ledger(+ ledger,+ with_semantic_lifecycle_hook("*", hook, process_bulk(&engine, None, &body)),+ )+ .await;+ assert!(result.errors);+ assert_eq!(+ result+ .items+ .iter()+ .map(|item| item.status)+ .collect::<Vec<_>>(),+ [201, 500]+ );+ let points = points.lock().unwrap();+ assert!(points.contains(&(ok.to_string(), SemanticLifecyclePoint::EmbeddingInside)));+ assert!(points.contains(&(ok.to_string(), SemanticLifecyclePoint::PublicationInside)));+ assert!(points.contains(&(fails.to_string(), SemanticLifecyclePoint::Grouped)));+ assert!(!points.contains(&(fails.to_string(), SemanticLifecyclePoint::EmbeddingInside)));+ assert_eq!(all_semantic_current(ledger), 0);+ assert_eq!(ledger.accounting_errors(), 0);+ }++ #[tokio::test]+ async fn tracing_off_semantic_path_skips_all_instrumentation_structures() {+ assert_eq!(+ crate::ingest_memory::config().mode,+ crate::ingest_memory::TraceMode::Off,+ "this structural proof requires tracing to be off; it must not silently skip"+ );+ let name = "semantic-lifecycle-disabled";+ let (_dir, engine) = semantic_engine(name);+ let counters = Arc::new(DisabledSemanticCounters::default());+ let body = format!(+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"one\"}}}}\n\+ {{\"content\":\"disabled structural proof\"}}\n"+ );+ let result = DISABLED_SEMANTIC_COUNTERS+ .scope(Arc::clone(&counters), process_bulk(&engine, None, &body))+ .await;+ assert!(!result.errors, "{:?}", result.items);+ assert_eq!(+ counters.snapshot(),+ DisabledSemanticInstrumentation::default(),+ "tracing-off path read schema, allocated its guard vector, or walked JSON for sizing"+ );+ assert_eq!(all_semantic_current(&crate::ingest_memory::LEDGER), 0);+ assert_eq!(crate::ingest_memory::LEDGER.accounting_errors(), 0);+ }+}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 2674c13e4..e256fc68e 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -2062,6 +2062,18 @@ impl Index {id: Option<String>,source: Value,) -> Result<IndexResponse> {+ #[cfg(test)]+ if crate::bulk::semantic_lifecycle_hook(+ self.name.as_str(),+ crate::bulk::SemanticLifecyclePoint::PublicationInside,+ ) == crate::bulk::SemanticLifecycleFault::Publication+ {+ return Err(crate::EngineError::Common(+ xerj_common::XerjError::invalid_query(+ "injected semantic publication failure inside index_document_prepared",+ ),+ ));+ }self.index_document_with_version_inner(id, source, None, None, true).await}@@ -3276,6 +3288,8 @@ impl Index {excluded_fts_fields,on_drained,warm_caches,+ #[cfg(test)]+ None,).await;// Fallback: if do_flush_shard aborted before on_drained fired@@ -3502,6 +3516,8 @@ impl Index {excluded_fts_fields.clone(),|| {}, // serial refresh path — no permit to drop earlyself.publish_warm_caches(),+ #[cfg(test)]+ None,).await?;}@@ -3634,6 +3650,8 @@ impl Index {excluded_fts_fields,on_drained,warm_caches,+ #[cfg(test)]+ None,).await;if let Ok(mut guard) = permit_cell.lock() {@@ -6272,6 +6290,24 @@ impl Index {}).collect()};+ #[cfg(test)]+ if crate::bulk::semantic_lifecycle_hook(+ self.name.as_str(),+ crate::bulk::SemanticLifecyclePoint::EmbeddingInside,+ ) == crate::bulk::SemanticLifecycleFault::Embedding+ {+ return sources+ .into_iter()+ .map(|_| {+ Err(crate::EngineError::Common(+ xerj_common::XerjError::invalid_query(+ "injected semantic embedding failure inside \+ apply_semantic_embeddings_batch",+ ),+ ))+ })+ .collect();+ }if specs.is_empty() {return sources.into_iter().map(Ok).collect();}@@ -6292,6 +6328,16 @@ impl Index {.mode.eq_ignore_ascii_case("onnx-experimental");for (source_idx, source) in sources.iter().enumerate() {+ #[cfg(test)]+ if crate::bulk::semantic_lifecycle_hook(+ self.name.as_str(),+ crate::bulk::SemanticLifecyclePoint::EmbeddingItemInside,+ ) == crate::bulk::SemanticLifecycleFault::Embedding+ {+ pre_failures[source_idx] =+ Some("injected per-item semantic embedding failure".to_string());+ continue;+ }let Some(obj) = source.as_object() else {continue;};@@ -6496,6 +6542,11 @@ impl Index {.collect()}+ pub(crate) async fn semantic_vector_fields(&self) -> std::collections::HashSet<String> {+ let schema = self.schema.read().await;+ crate::memtable::semantic_derived_vector_fields(&schema.schema)+ }+/// Hybrid (multi-query + fusion) executor. Recursively runs each/// sub-query as its own search and combines the result lists via/// the chosen fusion strategy (RRF / Linear / Learned). Pagination@@ -11486,6 +11537,10 @@ impl Index {// bounds global flush concurrency against ingest-time// background flushes on other indices.let n_shards = self.memtable.shard_count();+ #[cfg(test)]+ let flush_test_hook = FLUSH_PUBLISHER_TEST_HOOK+ .try_with(FlushPublisherTestHook::clone)+ .ok();let mut shard_futures = Vec::with_capacity(n_shards);for shard_idx in 0..n_shards {let sema = Arc::clone(&self.flush_sema);@@ -11496,6 +11551,8 @@ impl Index {let field_configs = field_configs.clone();let excluded_fts_fields = excluded_fts_fields.clone();let warm_caches = self.publish_warm_caches();+ #[cfg(test)]+ let test_hook = flush_test_hook.clone();shard_futures.push(tokio::spawn(async move {let permit = sema.acquire_owned().await.ok();let permit_cell = Arc::new(std::sync::Mutex::new(permit));@@ -11515,6 +11572,8 @@ impl Index {excluded_fts_fields,on_drained,warm_caches,+ #[cfg(test)]+ test_hook,).await;// Defensive: in case on_drained didn't fire.@@ -16822,6 +16881,18 @@ impl Drop for FlushDurationTimer {}#[allow(clippy::too_many_arguments)] // free fn threading the full flush-pipeline dependency set+#[cfg(test)]+#[derive(Clone)]+struct FlushPublisherTestHook {+ callback: Arc<dyn Fn() -> bool + Send + Sync>,+ ledger: &'static crate::ingest_memory::Ledger,+ target_memtable: usize,+}+#[cfg(test)]+tokio::task_local! {+ static FLUSH_PUBLISHER_TEST_HOOK: FlushPublisherTestHook;+}+async fn do_flush_shard(shard_idx: usize,store: Arc<IndexStore>,@@ -16835,6 +16906,7 @@ async fn do_flush_shard(// read-path caches, threaded through because this is a free fn without// access to `Index`.warm_caches: PublishWarmCaches,+ #[cfg(test)] test_hook: Option<FlushPublisherTestHook>,) -> Result<()> {// V4 M4.5: no outer flush_lock — concurrent flushes are allowed. The// memtable write lock below is the only atomicity point we need for@@ -16857,6 +16929,8 @@ async fn do_flush_shard(// the row positions in the stored section.// Peek BEFORE drain to determine if docs came from raw-bytes or FTS insert.let is_raw_bytes_path = memtable.peek_shard_has_raw_bytes(shard_idx);+ #[cfg(test)]+ let test_target = Arc::as_ptr(&memtable) as usize;// THROWAWAY prof: per-flush finalize breakdown, gated on XERJ_PROF.let prof = std::env::var_os("XERJ_PROF").is_some();@@ -16884,13 +16958,22 @@ async fn do_flush_shard(std::sync::Arc<serde_json::Value>,)>,xerj_storage::index_store::DrainedMemtable,+ crate::ingest_memory::Retained<'static>,)> = {let t_drain = std::time::Instant::now();- let raw = if is_raw_bytes_path {- memtable.drain_shard_raw(shard_idx)+ #[cfg(test)]+ let test_ledger = test_hook+ .as_ref()+ .filter(|hook| hook.target_memtable == test_target)+ .map(|hook| hook.ledger);+ #[cfg(test)]+ let (raw, _drained_memory) = if let Some(ledger) = test_ledger {+ memtable.drain_shard_accounted_for_test(shard_idx, is_raw_bytes_path, ledger)} else {- memtable.drain_shard(shard_idx)+ memtable.drain_shard_accounted(shard_idx, is_raw_bytes_path)};+ #[cfg(not(test))]+ let (raw, _drained_memory) = memtable.drain_shard_accounted(shard_idx, is_raw_bytes_path);prof_drain_us = t_drain.elapsed().as_micros();let t_prep = std::time::Instant::now();let _ = &t_prep;@@ -16969,7 +17052,7 @@ async fn do_flush_shard(entries: storage_entries,};prof_prep_us = t_prep.elapsed().as_micros();- Some((drained_fts, storage_drained))+ Some((drained_fts, storage_drained, _drained_memory))}};@@ -16986,7 +17069,7 @@ async fn do_flush_shard(// tmpfs.on_drained();- let (drained_fts, storage_drained) = match drained_opt {+ let (drained_fts, storage_drained, _drained_memory) = match drained_opt {Some(pair) => pair,None => return Ok(()),};@@ -17019,6 +17102,17 @@ async fn do_flush_shard(// 50-100 ms segment write that follows, and 100K-doc/s ingest// bench numbers are unaffected.let build_fts = move |meta: &xerj_storage::segment::SegmentMeta| -> xerj_storage::Result<()> {+ #[cfg(test)]+ let test_callback = test_hook+ .as_ref()+ .filter(|hook| hook.target_memtable == test_target)+ .map(|hook| Arc::clone(&hook.callback));+ #[cfg(test)]+ if let Some(callback) = test_callback {+ if callback() {+ return Err(std::io::Error::other("injected flush publisher failure").into());+ }+ }// Whole side-car build runs on the dedicated ingest pool so a// flush burst can't queue search/agg par_iters behind it on the// global rayon pool (read-under-write; see `crate::ingest_pool`).@@ -29053,3 +29147,85 @@ mod write_publication_integration_tests {assert_eq!(aggs["tags"]["buckets"][0]["key"], "current");}}++#[cfg(test)]+mod flush_memory_integration_tests {+ use super::*;+ use crate::ingest_memory::{Category, Ledger, Measurement};+ use serde_json::json;+ struct PublisherRelease(Option<std::sync::mpsc::SyncSender<()>>);+ impl PublisherRelease {+ fn release(&mut self) {+ if let Some(tx) = self.0.take() {+ let _ = tx.send(());+ }+ }+ }+ impl Drop for PublisherRelease {+ fn drop(&mut self) {+ self.release();+ }+ }++ async fn run_flush_memory_case(name: &'static str, inject_error: bool) {+ let ledger: &'static Ledger = Box::leak(Box::new(Ledger::new()));+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ engine.create_index(name, Schema::empty()).unwrap();+ let idx = engine.get_index(name).unwrap();+ idx.index_document(Some("doc".into()), json!({"body": "retained"}))+ .await+ .unwrap();+ assert!(idx.memtable_bytes() > 0);++ let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);+ let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1);+ let mut publisher_release = PublisherRelease(Some(release_tx));+ let entered_tx = parking_lot::Mutex::new(Some(entered_tx));+ let release_rx = parking_lot::Mutex::new(Some(release_rx));+ let hook = FlushPublisherTestHook {+ callback: Arc::new(move || {+ entered_tx.lock().take().unwrap().send(()).unwrap();+ let _ = release_rx.lock().take().unwrap().recv();+ inject_error+ }),+ ledger,+ target_memtable: Arc::as_ptr(&idx.memtable) as usize,+ };++ let flush = {+ let idx = Arc::clone(&idx);+ tokio::spawn(FLUSH_PUBLISHER_TEST_HOOK.scope(hook, async move { idx.flush().await }))+ };+ tokio::task::spawn_blocking(move || entered_rx.recv().unwrap())+ .await+ .unwrap();+ assert_eq!(idx.memtable_bytes(), 0);+ assert!(+ ledger+ .gauge(Category::FlushDrained, Measurement::Estimated)+ .current+ > 0+ );+ publisher_release.release();+ let result = flush.await.unwrap();+ assert_eq!(result.is_err(), inject_error);+ assert_eq!(+ ledger+ .gauge(Category::FlushDrained, Measurement::Estimated)+ .current,+ 0+ );+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn parallel_real_flushes_route_isolated_ledgers_through_success_and_error() {+ let (success, failure) = tokio::join!(+ run_flush_memory_case("flush-memory-success", false),+ run_flush_memory_case("flush-memory-failure", true),+ );+ assert_eq!((success, failure), ((), ()));+ }+}diff --git a/engine/crates/xerj-engine/src/ingest_memory.rs b/engine/crates/xerj-engine/src/ingest_memory.rsnew file mode 100644index 000000000..f635d7b12--- /dev/null+++ b/engine/crates/xerj-engine/src/ingest_memory.rs@@ -0,0 +1,982 @@+//! Opt-in ingest memory attribution.+//!+//! This module is deliberately independent of admission and scheduling. When+//! `XERJ_INGEST_MEMORY_TRACE` is absent or `off`, transition helpers return+//! before touching atomics, allocating, walking documents, probing `/proc`, or+//! constructing events.+//!+//! `summary` is the only enabled mode in v1: it emits bounded periodic+//! start/snapshot/stop NDJSON. Lifecycle `EventKind` variants and merge/cache+//! categories are reserved schema vocabulary; until their owners are wired,+//! their measurements are `unavailable` and their gauges must not be treated+//! as measured zero.+//!+//! `MemtableActive` is sampled from the engine while `FlushDrained` is owned at+//! the authoritative drain boundary. Those authorities cannot be sampled+//! atomically, so a transient snapshot may count the same estimate in both.+//! Logical totals are attribution diagnostics, never allocator identities.++use serde::Serialize;+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::{Arc, LazyLock, Mutex};++pub const SCHEMA: &str = "xerj.ingest_memory.v1";++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum TraceMode {+ Off,+ Summary,+}++#[derive(Clone, Copy, Debug)]+pub struct TraceConfig {+ pub mode: TraceMode,+ pub sample_ms: u64,+}++impl TraceConfig {+ fn from_env() -> Self {+ let mode = std::env::var("XERJ_INGEST_MEMORY_TRACE").unwrap_or_default();+ let sample_ms = std::env::var("XERJ_INGEST_MEMORY_SAMPLE_MS").unwrap_or_default();+ Self::parse(&mode, &sample_ms)+ }++ fn parse(mode: &str, sample_ms: &str) -> Self {+ let mode = match mode.to_ascii_lowercase().as_str() {+ "summary" | "1" | "true" | "on" => TraceMode::Summary,+ _ => TraceMode::Off,+ };+ let sample_ms = sample_ms+ .parse::<u64>()+ .ok()+ .unwrap_or(100)+ .clamp(25, 60_000);+ Self { mode, sample_ms }+ }+}++static CONFIG: LazyLock<TraceConfig> = LazyLock::new(TraceConfig::from_env);++#[cfg(test)]+tokio::task_local! {+ static TEST_LEDGER: &'static Ledger;+}++#[cfg(test)]+fn test_ledger() -> Option<&'static Ledger> {+ TEST_LEDGER.try_with(|ledger| *ledger).ok()+}++#[cfg(not(test))]+fn test_ledger() -> Option<&'static Ledger> {+ None+}++#[inline]+pub fn config() -> TraceConfig {+ *CONFIG+}++#[inline]+pub fn enabled() -> bool {+ active_ledger().is_some()+}++#[inline]+pub(crate) fn active_ledger() -> Option<&'static Ledger> {+ test_ledger().or_else(|| (CONFIG.mode != TraceMode::Off).then_some(&LEDGER))+}++#[cfg(test)]+pub(crate) async fn with_test_ledger<F: std::future::Future>(+ ledger: &'static Ledger,+ future: F,+) -> F::Output {+ TEST_LEDGER.scope(ledger, future).await+}++#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]+#[serde(rename_all = "snake_case")]+#[repr(u8)]+pub enum Category {+ HttpBody = 0,+ HttpRewriteBuffer,+ RawSource,+ ParsedSource,+ ParsedVector,+ PreparedDocument,+ PreparedVector,+ MemtableActive,+ FlushDrained,+ MergeDecoded,+ MergeSurvivor,+ MergeJsonBuffer,+ MergeParsed,+ MergeEncoded,+ CacheStoredSlices,+ CacheDocValues,+ CacheStoredValues,+ CacheSortShadow,+ CacheIdPositions,+ CacheRowSequences,+ CacheDecodedStored,+}++impl Category {+ pub const ALL: [Self; 21] = [+ Self::HttpBody,+ Self::HttpRewriteBuffer,+ Self::RawSource,+ Self::ParsedSource,+ Self::ParsedVector,+ Self::PreparedDocument,+ Self::PreparedVector,+ Self::MemtableActive,+ Self::FlushDrained,+ Self::MergeDecoded,+ Self::MergeSurvivor,+ Self::MergeJsonBuffer,+ Self::MergeParsed,+ Self::MergeEncoded,+ Self::CacheStoredSlices,+ Self::CacheDocValues,+ Self::CacheStoredValues,+ Self::CacheSortShadow,+ Self::CacheIdPositions,+ Self::CacheRowSequences,+ Self::CacheDecodedStored,+ ];++ const COUNT: usize = Self::ALL.len();+}++#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]+#[serde(rename_all = "snake_case")]+pub enum Measurement {+ ExactCapacity,+ SerializedSize,+ Estimated,+ Unavailable,+}++#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]+#[serde(rename_all = "snake_case")]+pub enum EventKind {+ TraceStart,+ TraceStop,+ TraceSnapshot,+ HttpBulkAccept,+ HttpBulkRelease,+ BulkGroupRetain,+ BulkGroupRelease,+ SemanticParsedRetain,+ SemanticParsedRelease,+ SemanticPreparedRetain,+ SemanticPreparedRelease,+ MemtableInsert,+ MemtableDrain,+ FlushDrainedRetain,+ FlushFinalizeWaitEnter,+ FlushFinalizeStart,+ FlushFinalizeEnd,+ FlushDrainedRelease,+ MergeRetain,+ MergePhase,+ MergeRelease,+ CacheInsert,+ CacheReplace,+ CacheEvict,+ CacheClear,+ TraceError,+}++#[derive(Clone, Copy, Debug, Default, Serialize)]+pub struct Scope {+ pub index_hash: Option<u32>,+ pub request: Option<u64>,+ pub flush: Option<u64>,+ pub merge: Option<u64>,+ pub shard: Option<u32>,+}++#[derive(Clone, Copy, Debug, Serialize)]+pub struct Gauge {+ pub category: Category,+ pub current: u64,+ pub peak: u64,+ pub measurement: Measurement,+}++#[derive(Clone, Copy, Debug, Default, Serialize)]+pub struct AllocatorSnapshot {+ pub allocated: Option<u64>,+ pub active: Option<u64>,+ pub resident: Option<u64>,+ pub epoch_ok: bool,+}++#[derive(Clone, Copy, Debug, Default, Serialize)]+pub struct ProcessSnapshot {+ pub rss: Option<u64>,+ pub rss_ok: bool,+ pub cpu_time_ns: Option<u64>,+ pub cpu_time_ok: bool,+}++#[derive(Debug, Serialize)]+pub struct Event {+ pub schema: &'static str,+ pub seq: u64,+ pub ts_unix_ns: u128,+ pub event: EventKind,+ pub scope: Scope,+ pub logical: Vec<Gauge>,+ /// Sum of all logical category currents in this coherent snapshot.+ /// This is diagnostic attribution, not an allocator/RSS identity.+ pub logical_total_current: u64,+ /// Retain/release underflows observed since process start.+ pub accounting_errors: u64,+ /// Trace events rejected by the bounded output sink since trace start.+ pub dropped_events: u64,+ pub allocator: AllocatorSnapshot,+ pub process: ProcessSnapshot,+ pub flags: EventFlags,+}++#[derive(Clone, Copy, Debug, Serialize)]+pub struct EventFlags {+ pub sampled: bool,+ pub logical_exact: bool,+ pub allocator_supported: bool,+}++struct Cell {+ current: AtomicU64,+ peak: AtomicU64,+}++impl Cell {+ const fn new() -> Self {+ Self {+ current: AtomicU64::new(0),+ peak: AtomicU64::new(0),+ }+ }+}++pub struct Ledger {+ transition: Mutex<()>,+ cells: [Cell; Category::COUNT],+ seq: AtomicU64,+ accounting_errors: AtomicU64,+ physical_allocated: AtomicU64,+ physical_active: AtomicU64,+ physical_resident: AtomicU64,+ physical_rss: AtomicU64,+ physical_cpu_time_ns: AtomicU64,+ physical_flags: AtomicU64,+ dropped_events: AtomicU64,+}++impl Ledger {+ pub const fn new() -> Self {+ Self {+ transition: Mutex::new(()),+ cells: [const { Cell::new() }; Category::COUNT],+ seq: AtomicU64::new(0),+ accounting_errors: AtomicU64::new(0),+ physical_allocated: AtomicU64::new(0),+ physical_active: AtomicU64::new(0),+ physical_resident: AtomicU64::new(0),+ physical_rss: AtomicU64::new(0),+ physical_cpu_time_ns: AtomicU64::new(0),+ physical_flags: AtomicU64::new(0),+ dropped_events: AtomicU64::new(0),+ }+ }++ fn retain(&self, category: Category, bytes: u64) {+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());+ self.retain_locked(category, bytes);+ }++ fn retain_locked(&self, category: Category, bytes: u64) {+ if bytes == 0 {+ return;+ }+ let cell = &self.cells[category as usize];+ let next = cell.current.fetch_add(bytes, Ordering::Relaxed) + bytes;+ let mut peak = cell.peak.load(Ordering::Relaxed);+ while next > peak {+ match cell+ .peak+ .compare_exchange_weak(peak, next, Ordering::Relaxed, Ordering::Relaxed)+ {+ Ok(_) => break,+ Err(observed) => peak = observed,+ }+ }+ }++ fn release(&self, category: Category, bytes: u64) {+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());+ self.release_locked(category, bytes);+ }++ fn release_locked(&self, category: Category, bytes: u64) -> bool {+ if bytes == 0 {+ return true;+ }+ let cell = &self.cells[category as usize];+ let mut current = cell.current.load(Ordering::Relaxed);+ loop {+ if bytes > current {+ self.accounting_errors.fetch_add(1, Ordering::Relaxed);+ return false;+ }+ match cell.current.compare_exchange_weak(+ current,+ current - bytes,+ Ordering::Relaxed,+ Ordering::Relaxed,+ ) {+ Ok(_) => return true,+ Err(observed) => current = observed,+ }+ }+ }++ fn transfer(&self, from: Category, to: Category, bytes: u64) {+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());+ if self.release_locked(from, bytes) {+ self.retain_locked(to, bytes);+ }+ }++ pub fn gauge(&self, category: Category, measurement: Measurement) -> Gauge {+ let cell = &self.cells[category as usize];+ Gauge {+ category,+ current: cell.current.load(Ordering::Relaxed),+ peak: cell.peak.load(Ordering::Relaxed),+ measurement,+ }+ }++ pub fn accounting_errors(&self) -> u64 {+ self.accounting_errors.load(Ordering::Relaxed)+ }++ pub fn record_dropped_event(&self) {+ self.dropped_events.fetch_add(1, Ordering::Relaxed);+ }++ pub fn observe(&self, category: Category, bytes: usize) {+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());+ let cell = &self.cells[category as usize];+ let bytes = bytes as u64;+ cell.current.store(bytes, Ordering::Relaxed);+ cell.peak.fetch_max(bytes, Ordering::Relaxed);+ }++ pub fn update_physical(&self, allocator: AllocatorSnapshot, process: ProcessSnapshot) {+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());+ const ALLOCATOR_OK: u64 = 1;+ const RSS_OK: u64 = 2;+ const CPU_TIME_OK: u64 = 4;+ let mut flags = 0;+ if allocator.epoch_ok {+ flags |= ALLOCATOR_OK;+ }+ if process.rss_ok {+ flags |= RSS_OK;+ }+ if process.cpu_time_ok {+ flags |= CPU_TIME_OK;+ }+ self.physical_allocated+ .store(allocator.allocated.unwrap_or(0), Ordering::Relaxed);+ self.physical_active+ .store(allocator.active.unwrap_or(0), Ordering::Relaxed);+ self.physical_resident+ .store(allocator.resident.unwrap_or(0), Ordering::Relaxed);+ self.physical_rss+ .store(process.rss.unwrap_or(0), Ordering::Relaxed);+ self.physical_cpu_time_ns+ .store(process.cpu_time_ns.unwrap_or(0), Ordering::Relaxed);+ self.physical_flags.store(flags, Ordering::Release);+ }++ fn event(&self, kind: EventKind, scope: Scope, sampled: bool) -> Event {+ let _transition = self.transition.lock().unwrap_or_else(|e| e.into_inner());+ let flags = self.physical_flags.load(Ordering::Acquire);+ let allocator_ok = flags & 1 != 0;+ let rss_ok = flags & 2 != 0;+ let cpu_time_ok = flags & 4 != 0;+ let logical: Vec<_> = Category::ALL+ .into_iter()+ .map(|category| self.gauge(category, measurement(category)))+ .collect();+ let logical_total_current = logical.iter().map(|gauge| gauge.current).sum();+ Event {+ schema: SCHEMA,+ seq: self.seq.fetch_add(1, Ordering::Relaxed) + 1,+ ts_unix_ns: std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap_or_default()+ .as_nanos(),+ event: kind,+ scope,+ logical,+ logical_total_current,+ accounting_errors: self.accounting_errors.load(Ordering::Relaxed),+ dropped_events: self.dropped_events.load(Ordering::Relaxed),+ allocator: AllocatorSnapshot {+ allocated: allocator_ok.then(|| self.physical_allocated.load(Ordering::Relaxed)),+ active: allocator_ok.then(|| self.physical_active.load(Ordering::Relaxed)),+ resident: allocator_ok.then(|| self.physical_resident.load(Ordering::Relaxed)),+ epoch_ok: allocator_ok,+ },+ process: ProcessSnapshot {+ rss: rss_ok.then(|| self.physical_rss.load(Ordering::Relaxed)),+ rss_ok,+ cpu_time_ns: cpu_time_ok.then(|| self.physical_cpu_time_ns.load(Ordering::Relaxed)),+ cpu_time_ok,+ },+ flags: EventFlags {+ sampled,+ logical_exact: false,+ allocator_supported: allocator_ok,+ },+ }+ }+}++pub static LEDGER: Ledger = Ledger::new();++/// Build one sampler-owned event. This is the only public event constructor:+/// ingest hot paths cannot accidentally allocate events in `off` or `summary`.+pub fn snapshot_event(kind: EventKind, sampled: bool) -> Option<Event> {+ if !enabled()+ || !matches!(+ kind,+ EventKind::TraceStart | EventKind::TraceStop | EventKind::TraceSnapshot+ )+ {+ return None;+ }+ Some(LEDGER.event(kind, Scope::default(), sampled))+}++fn measurement(category: Category) -> Measurement {+ match category {+ Category::ParsedVector | Category::PreparedVector | Category::HttpRewriteBuffer => {+ Measurement::ExactCapacity+ }+ Category::HttpBody | Category::RawSource => Measurement::SerializedSize,+ Category::MemtableActive+ | Category::ParsedSource+ | Category::PreparedDocument+ | Category::FlushDrained => Measurement::Estimated,+ _ => Measurement::Unavailable,+ }+}++pub struct Retained<'a> {+ ledger: &'a Ledger,+ category: Category,+ bytes: u64,+ active: bool,+}++impl Retained<'static> {+ #[inline]+ pub fn new(category: Category, bytes: usize) -> Self {+ let ledger = active_ledger();+ let Some(ledger) = ledger else {+ return Self::disabled();+ };+ let bytes = bytes as u64;+ ledger.retain(category, bytes);+ Self {+ ledger,+ category,+ bytes,+ active: true,+ }+ }++ #[inline]+ pub const fn disabled() -> Self {+ Self {+ ledger: &LEDGER,+ category: Category::HttpBody,+ bytes: 0,+ active: false,+ }+ }+}++impl<'a> Retained<'a> {+ pub(crate) fn for_ledger(ledger: &'a Ledger, category: Category, bytes: usize) -> Self {+ let bytes = bytes as u64;+ ledger.retain(category, bytes);+ Self {+ ledger,+ category,+ bytes,+ active: true,+ }+ }++ pub fn transfer(mut self, category: Category) -> Self {+ if self.active {+ self.ledger.transfer(self.category, category, self.bytes);+ self.category = category;+ }+ self+ }++ pub fn replace(mut self, category: Category, bytes: usize) -> Self {+ if self.active {+ let _transition = self+ .ledger+ .transition+ .lock()+ .unwrap_or_else(|e| e.into_inner());+ if self.ledger.release_locked(self.category, self.bytes) {+ self.ledger.retain_locked(category, bytes as u64);+ self.category = category;+ self.bytes = bytes as u64;+ }+ }+ self+ }++ pub fn replace_split(+ mut self,+ first: (Category, usize),+ second: (Category, usize),+ ) -> (Self, Self) {+ let mut other = Self {+ ledger: self.ledger,+ category: second.0,+ bytes: second.1 as u64,+ active: false,+ };+ if self.active {+ let _transition = self+ .ledger+ .transition+ .lock()+ .unwrap_or_else(|e| e.into_inner());+ if self.ledger.release_locked(self.category, self.bytes) {+ self.ledger.retain_locked(first.0, first.1 as u64);+ self.ledger.retain_locked(second.0, second.1 as u64);+ self.category = first.0;+ self.bytes = first.1 as u64;+ other.active = true;+ }+ }+ (self, other)+ }++ pub fn replace_pair(+ mut first: Self,+ mut second: Self,+ first_next: (Category, usize),+ second_next: (Category, usize),+ ) -> (Self, Self) {+ if first.active && second.active && std::ptr::eq(first.ledger, second.ledger) {+ let ledger = first.ledger;+ let _transition = ledger.transition.lock().unwrap_or_else(|e| e.into_inner());+ let first_current = ledger.cells[first.category as usize]+ .current+ .load(Ordering::Relaxed);+ let second_current = ledger.cells[second.category as usize]+ .current+ .load(Ordering::Relaxed);+ let enough = if first.category == second.category {+ first_current >= first.bytes.saturating_add(second.bytes)+ } else {+ first_current >= first.bytes && second_current >= second.bytes+ };+ if enough {+ let _ = ledger.release_locked(first.category, first.bytes);+ let _ = ledger.release_locked(second.category, second.bytes);+ ledger.retain_locked(first_next.0, first_next.1 as u64);+ ledger.retain_locked(second_next.0, second_next.1 as u64);+ first.category = first_next.0;+ first.bytes = first_next.1 as u64;+ second.category = second_next.0;+ second.bytes = second_next.1 as u64;+ } else {+ ledger.accounting_errors.fetch_add(1, Ordering::Relaxed);+ }+ }+ (first, second)+ }+}++/// Non-allocating estimate of heap retained by a JSON tree.+pub fn estimated_json_heap(value: &serde_json::Value) -> usize {+ match value {+ serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => 0,+ serde_json::Value::String(s) => s.capacity(),+ serde_json::Value::Array(values) => {+ values.capacity() * std::mem::size_of::<serde_json::Value>()+ + values.iter().map(estimated_json_heap).sum::<usize>()+ }+ serde_json::Value::Object(values) => {+ values.len()+ * (std::mem::size_of::<String>() + std::mem::size_of::<serde_json::Value>())+ + values+ .iter()+ .map(|(key, value)| key.capacity() + estimated_json_heap(value))+ .sum::<usize>()+ }+ }+}++pub fn estimated_json_heap_excluding(+ value: &serde_json::Value,+ excluded: &std::collections::HashSet<String>,+) -> usize {+ fn visit(value: &serde_json::Value, excluded: &std::collections::HashSet<String>) -> usize {+ match value {+ serde_json::Value::Object(values) => {+ values.len()+ * (std::mem::size_of::<String>() + std::mem::size_of::<serde_json::Value>())+ + values+ .iter()+ .map(|(key, value)| {+ key.capacity()+ + if excluded.contains(key) {+ 0+ } else {+ visit(value, excluded)+ }+ })+ .sum::<usize>()+ }+ serde_json::Value::Array(values) => {+ values.capacity() * std::mem::size_of::<serde_json::Value>()+ + values+ .iter()+ .map(|value| visit(value, excluded))+ .sum::<usize>()+ }+ serde_json::Value::String(value) => value.capacity(),+ _ => 0,+ }+ }+ visit(value, excluded)+}++pub fn vector_value_capacity(+ value: &serde_json::Value,+ fields: &std::collections::HashSet<String>,+) -> usize {+ fn arrays(value: &serde_json::Value) -> usize {+ match value {+ serde_json::Value::Array(values) => {+ values.capacity() * std::mem::size_of::<serde_json::Value>()+ + values.iter().map(arrays).sum::<usize>()+ }+ serde_json::Value::Object(values) => values.values().map(arrays).sum(),+ _ => 0,+ }+ }+ match value {+ serde_json::Value::Object(values) => values+ .iter()+ .map(|(key, value)| {+ if fields.contains(key) {+ arrays(value)+ } else {+ vector_value_capacity(value, fields)+ }+ })+ .sum(),+ serde_json::Value::Array(values) => values+ .iter()+ .map(|value| vector_value_capacity(value, fields))+ .sum(),+ _ => 0,+ }+}++impl Drop for Retained<'_> {+ fn drop(&mut self) {+ if self.active {+ self.ledger.release(self.category, self.bytes);+ }+ }+}++pub struct TrackedArc<T> {+ value: T,+ _retained: Retained<'static>,+}++impl<T> TrackedArc<T> {+ pub fn new(value: T, category: Category, bytes: usize) -> Arc<Self> {+ Arc::new(Self {+ value,+ _retained: Retained::new(category, bytes),+ })+ }++ pub fn get(&self) -> &T {+ &self.value+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn trace_modes_are_distinct_and_unknown_events_mode_fails_closed() {+ assert_eq!(TraceConfig::parse("off", "100").mode, TraceMode::Off);+ assert_eq!(+ TraceConfig::parse("summary", "100").mode,+ TraceMode::Summary+ );+ assert_eq!(+ TraceConfig::parse("events", "100").mode,+ TraceMode::Off,+ "v1 must not claim unwired lifecycle events"+ );+ assert_eq!(TraceConfig::parse("summary", "1").sample_ms, 25);+ assert_eq!(TraceConfig::parse("summary", "999999").sample_ms, 60_000);+ }++ #[test]+ fn retain_release_peak_and_underflow_are_deterministic() {+ let ledger = Ledger::new();+ ledger.retain(Category::RawSource, 100);+ ledger.retain(Category::RawSource, 60);+ ledger.release(Category::RawSource, 100);+ let gauge = ledger.gauge(Category::RawSource, Measurement::SerializedSize);+ assert_eq!((gauge.current, gauge.peak), (60, 160));+ ledger.release(Category::RawSource, 61);+ assert_eq!(ledger.accounting_errors(), 1);+ assert_eq!(+ ledger+ .gauge(Category::RawSource, Measurement::SerializedSize)+ .current,+ 60+ );+ }++ #[test]+ fn snapshot_self_reports_logical_total_accounting_and_sink_loss() {+ let ledger = Ledger::new();+ ledger.retain(Category::RawSource, 100);+ ledger.retain(Category::PreparedVector, 28);+ ledger.release(Category::RawSource, 101);+ ledger.record_dropped_event();+ let event = ledger.event(EventKind::TraceSnapshot, Scope::default(), true);+ assert_eq!(event.logical_total_current, 128);+ assert_eq!(event.accounting_errors, 1);+ assert_eq!(event.dropped_events, 1);+ }++ #[test]+ fn tracked_arc_charges_one_backing_owner() {+ let ledger = Ledger::new();+ struct LocalTracked<'a, T> {+ value: T,+ _retained: Retained<'a>,+ }+ let owner = Arc::new(LocalTracked {+ value: vec![0u8; 100],+ _retained: Retained::for_ledger(&ledger, Category::RawSource, 100),+ });+ let clones: Vec<_> = (0..100).map(|_| Arc::clone(&owner)).collect();+ assert_eq!(owner.value.len(), 100);+ assert_eq!(+ ledger+ .gauge(Category::RawSource, Measurement::SerializedSize)+ .current,+ 100+ );+ drop(clones);+ drop(owner);+ assert_eq!(+ ledger+ .gauge(Category::RawSource, Measurement::SerializedSize)+ .current,+ 0+ );+ }++ #[test]+ fn transfer_snapshot_has_no_zero_or_double_count_gap() {+ let ledger = Ledger::new();+ let retained = Retained::for_ledger(&ledger, Category::MemtableActive, 77);+ let retained = retained.transfer(Category::FlushDrained);+ let active = ledger.gauge(Category::MemtableActive, Measurement::Estimated);+ let drained = ledger.gauge(Category::FlushDrained, Measurement::Estimated);+ assert_eq!((active.current, drained.current), (0, 77));+ drop(retained);+ assert_eq!(+ ledger+ .gauge(Category::FlushDrained, Measurement::Estimated)+ .current,+ 0+ );+ }++ #[test]+ fn failed_transfer_does_not_charge_destination() {+ let ledger = Ledger::new();+ ledger.retain(Category::MemtableActive, 10);+ ledger.transfer(Category::MemtableActive, Category::FlushDrained, 11);+ assert_eq!(+ ledger+ .gauge(Category::MemtableActive, Measurement::Estimated)+ .current,+ 10+ );+ assert_eq!(+ ledger+ .gauge(Category::FlushDrained, Measurement::Estimated)+ .current,+ 0+ );+ assert_eq!(ledger.accounting_errors(), 1);+ }++ #[test]+ fn semantic_phase_split_is_non_overlapping_and_balances_on_drop() {+ let ledger = Ledger::new();+ let parsed = Retained::for_ledger(&ledger, Category::ParsedSource, 120);+ let (document, vector) = parsed.replace_split(+ (Category::PreparedDocument, 80),+ (Category::PreparedVector, 40),+ );+ assert_eq!(+ ledger+ .gauge(Category::ParsedSource, Measurement::Estimated)+ .current,+ 0+ );+ assert_eq!(+ ledger+ .gauge(Category::PreparedDocument, Measurement::Estimated)+ .current,+ 80+ );+ assert_eq!(+ ledger+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)+ .current,+ 40+ );+ drop((document, vector));+ assert_eq!(+ ledger+ .gauge(Category::PreparedDocument, Measurement::Estimated)+ .current+ + ledger+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)+ .current,+ 0+ );+ }++ #[test]+ fn caller_vector_and_document_transition_as_one_snapshot() {+ let ledger = Ledger::new();+ let document = Retained::for_ledger(&ledger, Category::ParsedSource, 70);+ let vector = Retained::for_ledger(&ledger, Category::ParsedVector, 30);+ let (document, vector) = Retained::replace_pair(+ document,+ vector,+ (Category::PreparedDocument, 60),+ (Category::PreparedVector, 40),+ );+ assert_eq!(+ ledger+ .gauge(Category::ParsedSource, Measurement::Estimated)+ .current+ + ledger+ .gauge(Category::ParsedVector, Measurement::ExactCapacity)+ .current,+ 0+ );+ assert_eq!(+ ledger+ .gauge(Category::PreparedDocument, Measurement::Estimated)+ .current+ + ledger+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)+ .current,+ 100+ );+ drop((document, vector));+ assert_eq!(+ ledger+ .gauge(Category::PreparedDocument, Measurement::Estimated)+ .current+ + ledger+ .gauge(Category::PreparedVector, Measurement::ExactCapacity)+ .current,+ 0+ );+ }++ #[test]+ fn custom_vector_fields_are_excluded_and_capacity_counted() {+ let value = serde_json::json!({+ "body": "hello",+ "custom_embedding": [0.1, 0.2, 0.3],+ });+ let fields = std::collections::HashSet::from(["custom_embedding".to_string()]);+ let without = estimated_json_heap_excluding(&value, &fields);+ let with = estimated_json_heap(&value);+ assert!(without < with);+ assert_eq!(+ vector_value_capacity(&value, &fields),+ value["custom_embedding"].as_array().unwrap().capacity()+ * std::mem::size_of::<serde_json::Value>()+ );+ }++ #[test]+ fn guard_balances_error_and_panic_unwind() {+ let ledger = Ledger::new();+ let result = std::panic::catch_unwind(|| {+ ledger.retain(Category::FlushDrained, 42);+ struct Local<'a>(&'a Ledger);+ impl Drop for Local<'_> {+ fn drop(&mut self) {+ self.0.release(Category::FlushDrained, 42);+ }+ }+ let _guard = Local(&ledger);+ panic!("intentional");+ });+ assert!(result.is_err());+ assert_eq!(+ ledger+ .gauge(Category::FlushDrained, Measurement::Estimated)+ .current,+ 0+ );+ }++ #[test]+ fn physical_failure_is_null_not_zero() {+ let ledger = Ledger::new();+ ledger.update_physical(AllocatorSnapshot::default(), ProcessSnapshot::default());+ let event = ledger.event(EventKind::TraceSnapshot, Scope::default(), true);+ assert_eq!(event.allocator.allocated, None);+ assert_eq!(event.process.rss, None);+ let json = serde_json::to_value(event).unwrap();+ assert_eq!(json["schema"], SCHEMA);+ assert!(json["allocator"]["allocated"].is_null());+ }+}diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rsindex 1bb665b5e..d53fd7fb0 100644--- a/engine/crates/xerj-engine/src/lib.rs+++ b/engine/crates/xerj-engine/src/lib.rs@@ -22,6 +22,7 @@ pub mod bulk;pub mod engine;pub mod governor;pub mod index;+pub mod ingest_memory;pub mod memtable;pub mod painless;pub mod rbac;diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex e828b173d..6a31b176c 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -25,6 +25,81 @@ pub struct MemtableHit {pub score: f32,}+#[cfg(test)]+mod ingest_memory_drain_tests {+ use super::*;+ use crate::ingest_memory::{Category, Ledger, Measurement};+ use std::sync::{Arc, Barrier};++ #[test]+ fn authoritative_active_moves_to_drained_through_blocked_finalize_success_and_error() {+ for inject_error in [false, true] {+ let mem = ShardedFtsMemtable::new();+ mem.insert(+ "doc-1".to_string(),+ &serde_json::json!({"body": "retained"}),+ &Schema::default(),+ 1,+ );+ let shard = mem+ .shard_loads()+ .into_iter()+ .find(|(_, docs, _)| *docs != 0)+ .unwrap()+ .0;+ let ledger = Ledger::new();+ ledger.observe(Category::MemtableActive, mem.size_bytes());+ assert!(+ ledger+ .gauge(Category::MemtableActive, Measurement::Estimated)+ .current+ > 0+ );++ let ledger: &'static Ledger = Box::leak(Box::new(ledger));+ let (_entries, guard) = mem.drain_shard_accounted_for_test(shard, false, ledger);+ ledger.observe(Category::MemtableActive, mem.size_bytes());+ let entered = Arc::new(Barrier::new(2));+ let release = Arc::new(Barrier::new(2));+ std::thread::scope(|scope| {+ let entered_worker = Arc::clone(&entered);+ let release_worker = Arc::clone(&release);+ let handle = scope.spawn(move || -> Result<(), &'static str> {+ let _guard = guard;+ entered_worker.wait();+ release_worker.wait();+ if inject_error {+ Err("injected finalizer error")+ } else {+ Ok(())+ }+ });+ entered.wait();+ assert_eq!(+ ledger+ .gauge(Category::MemtableActive, Measurement::Estimated)+ .current,+ 0+ );+ assert!(+ ledger+ .gauge(Category::FlushDrained, Measurement::Estimated)+ .current+ > 0+ );+ release.wait();+ assert_eq!(handle.join().unwrap().is_err(), inject_error);+ });+ assert_eq!(+ ledger+ .gauge(Category::FlushDrained, Measurement::Estimated)+ .current,+ 0+ );+ }+ }+}+/// Entry stored in the memtable.////// Post-M4.9 the `fields: HashMap<String, String>` was removed — it was@@ -1486,21 +1561,55 @@ impl ShardedFtsMemtable {/// tuples in WAL-sequence order. Raw bytes are passed through to the/// segment writer so it can skip re-serializing the Value.pub fn drain_shard(&self, shard_idx: usize) -> Vec<(u64, String, Arc<Value>, Arc<[u8]>)> {- self.drain_shard_inner(shard_idx, false)+ self.drain_shard_inner(shard_idx, false, None).0}/// Drain without parsing raw-bytes entries. Returns Value::Null for/// entries that came from insert_raw_bytes_with_seq. Use when neither/// FTS nor DV sidecars will be built (turbo/CLI ingest path).pub fn drain_shard_raw(&self, shard_idx: usize) -> Vec<(u64, String, Arc<Value>, Arc<[u8]>)> {- self.drain_shard_inner(shard_idx, true)+ self.drain_shard_inner(shard_idx, true, None).0}- fn drain_shard_inner(+ pub fn drain_shard_accounted(&self,shard_idx: usize,skip_parse: bool,- ) -> Vec<(u64, String, Arc<Value>, Arc<[u8]>)> {+ ) -> (+ Vec<(u64, String, Arc<Value>, Arc<[u8]>)>,+ crate::ingest_memory::Retained<'static>,+ ) {+ let ledger = crate::ingest_memory::active_ledger();+ let (entries, guard) = self.drain_shard_inner(shard_idx, skip_parse, ledger);+ (+ entries,+ guard.unwrap_or_else(crate::ingest_memory::Retained::disabled),+ )+ }++ #[cfg(test)]+ pub(crate) fn drain_shard_accounted_for_test(+ &self,+ shard_idx: usize,+ skip_parse: bool,+ ledger: &'static crate::ingest_memory::Ledger,+ ) -> (+ Vec<(u64, String, Arc<Value>, Arc<[u8]>)>,+ crate::ingest_memory::Retained<'static>,+ ) {+ let (entries, guard) = self.drain_shard_inner(shard_idx, skip_parse, Some(ledger));+ (entries, guard.expect("test ledger always creates guard"))+ }++ fn drain_shard_inner<'a>(+ &self,+ shard_idx: usize,+ skip_parse: bool,+ ledger: Option<&'a crate::ingest_memory::Ledger>,+ ) -> (+ Vec<(u64, String, Arc<Value>, Arc<[u8]>)>,+ Option<crate::ingest_memory::Retained<'a>>,+ ) {// Swap the shard's maps out under the write lock (pointer moves,// O(1)) and deallocate them AFTER the lock is released, on a// detached thread. Pre-fix the reset assignments freed the@@ -1511,8 +1620,9 @@ impl ShardedFtsMemtable {// drain stalled every in-flight bulk request. 32 flushes per// 1 M docs × ~95 ms = a fixed ~3 s Amdahl serial term that// capped 8-client ingest at ~3.1× single-client throughput.- let (drained, dead) = {+ let (drained, dead, drain_guard) = {let mut g = self.shards[shard_idx].write();+ let removed_bytes = g.total_bytes;let d: Vec<MemEntry> = std::mem::take(&mut g.docs);let dead_index = std::mem::take(&mut g.index);let dead_dv = std::mem::take(&mut g.doc_values);@@ -1520,6 +1630,17 @@ impl ShardedFtsMemtable {let dead_afl = std::mem::take(&mut g.avg_field_lengths);let dead_dii = std::mem::take(&mut g.doc_id_index);g.total_bytes = 0;+ // Created while the authoritative shard lock is still held so+ // the drained lifetime includes detached-map handoff and parsing.+ // Periodic active-vs-drained snapshots remain best-effort because+ // the two authorities are sampled separately.+ let drain_guard = ledger.map(|ledger| {+ crate::ingest_memory::Retained::for_ledger(+ ledger,+ crate::ingest_memory::Category::FlushDrained,+ removed_bytes,+ )+ });// Flush == merge: purge delete-aware ghost collection stats.g.ghost_docs = 0;let dead_gfl = std::mem::take(&mut g.ghost_field_len);@@ -1529,6 +1650,7 @@ impl ShardedFtsMemtable {(dead_index, dead_dv, dead_fl, dead_afl, dead_dii, dead_gfl, dead_gdf,),+ drain_guard,)};// Free the dead maps off the flush critical path too — the@@ -1558,7 +1680,7 @@ impl ShardedFtsMemtable {}).collect();out.sort_by_key(|(seq, _, _, _)| *seq);- out+ (out, drain_guard)}/// Check if a shard's first entry was inserted via the raw-bytesdiff --git a/engine/crates/xerj-server/Cargo.toml b/engine/crates/xerj-server/Cargo.tomlindex 98e8fcb0c..d0a756590 100644--- a/engine/crates/xerj-server/Cargo.toml+++ b/engine/crates/xerj-server/Cargo.toml@@ -76,6 +76,8 @@ debug-profiling = [[target.'cfg(not(target_env = "msvc"))'.dependencies]tikv-jemallocator = "0.6"+tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }+libc = "0.2"[target.'cfg(all(target_os = "linux", not(target_env = "msvc")))'.dependencies]jemalloc_pprof = { version = "0.8", features = ["symbolize"], optional = true }diff --git a/engine/crates/xerj-server/src/ingest_memory_trace.rs b/engine/crates/xerj-server/src/ingest_memory_trace.rsnew file mode 100644index 000000000..a7db3d8e6--- /dev/null+++ b/engine/crates/xerj-server/src/ingest_memory_trace.rs@@ -0,0 +1,411 @@+use std::io::Write;+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::{mpsc, Arc, Condvar, Mutex, Weak};+use std::thread::JoinHandle;+use std::time::Duration;++use xerj_engine::ingest_memory::{+ self, AllocatorSnapshot, EventKind, ProcessSnapshot, TraceMode, LEDGER,+};+use xerj_engine::Engine;++const CHANNEL_CAPACITY: usize = 256;++pub struct TraceRuntime {+ stop: Arc<AtomicBool>,+ wake: Arc<(Mutex<()>, Condvar)>,+ sampler: Option<JoinHandle<()>>,+ writer: Option<JoinHandle<()>>,+ dropped: Arc<AtomicU64>,+}++impl Drop for TraceRuntime {+ fn drop(&mut self) {+ self.stop.store(true, Ordering::Release);+ self.wake.1.notify_all();+ if let Some(handle) = self.sampler.take() {+ let _ = handle.join();+ }+ if let Some(handle) = self.writer.take() {+ let _ = handle.join();+ }+ let dropped = self.dropped.load(Ordering::Relaxed);+ if dropped != 0 {+ tracing::warn!(+ target: "xerj::ingest_memory",+ dropped,+ "ingest-memory trace events dropped by bounded sink"+ );+ }+ }+}++pub fn spawn(engine: &Arc<Engine>) -> Option<TraceRuntime> {+ let config = ingest_memory::config();+ if config.mode == TraceMode::Off {+ return None;+ }++ let (tx, rx) = mpsc::sync_channel::<String>(CHANNEL_CAPACITY);+ let stop = Arc::new(AtomicBool::new(false));+ let dropped = Arc::new(AtomicU64::new(0));+ let writer_dropped = Arc::clone(&dropped);+ let writer = std::thread::Builder::new()+ .name("xerj-ingest-memory-writer".into())+ .spawn(move || writer_loop(rx, writer_dropped))+ .ok()?;++ let weak = Arc::downgrade(engine);+ let sampler_stop = Arc::clone(&stop);+ let wake = Arc::new((Mutex::new(()), Condvar::new()));+ let sampler_wake = Arc::clone(&wake);+ let sampler_dropped = Arc::clone(&dropped);+ let sampler = std::thread::Builder::new()+ .name("xerj-ingest-memory-sampler".into())+ .spawn(move || {+ send_snapshot(&tx, &sampler_dropped, EventKind::TraceStart, false);+ sample_loop(+ weak,+ tx,+ sampler_stop,+ sampler_wake,+ sampler_dropped,+ Duration::from_millis(config.sample_ms),+ );+ })+ .ok()?;++ Some(TraceRuntime {+ stop,+ wake,+ sampler: Some(sampler),+ writer: Some(writer),+ dropped,+ })+}++fn sample_loop(+ engine: Weak<Engine>,+ tx: mpsc::SyncSender<String>,+ stop: Arc<AtomicBool>,+ wake: Arc<(Mutex<()>, Condvar)>,+ dropped: Arc<AtomicU64>,+ period: Duration,+) {+ while !stop.load(Ordering::Acquire) {+ let Some(engine) = engine.upgrade() else {+ break;+ };+ refresh_snapshot_state(&engine);+ send_snapshot(&tx, &dropped, EventKind::TraceSnapshot, true);+ wait_for_stop_or_period(&stop, &wake, period);+ }++ // The shutdown path force-flushes every index before dropping+ // `TraceRuntime`. That can change the sampled memtable gauge after the+ // final periodic snapshot. Refresh synchronously here so `trace_stop`+ // describes the post-flush state even when shutdown happens immediately+ // after ingest, without waiting for another sampler period.+ if let Some(engine) = engine.upgrade() {+ refresh_snapshot_state(&engine);+ }+ send_snapshot(&tx, &dropped, EventKind::TraceStop, false);+}++fn refresh_snapshot_state(engine: &Engine) {+ refresh_snapshot_state_with(+ engine.total_memtable_bytes(),+ allocator_snapshot(),+ xerj_engine::governor::current_rss_bytes(),+ process_cpu_time_ns(),+ );+}++fn refresh_snapshot_state_with(+ memtable_bytes: u64,+ allocator: AllocatorSnapshot,+ rss: u64,+ cpu_time_ns: Option<u64>,+) {+ LEDGER.observe(+ ingest_memory::Category::MemtableActive,+ memtable_bytes as usize,+ );+ let process = ProcessSnapshot {+ rss: (rss != 0).then_some(rss),+ rss_ok: rss != 0,+ cpu_time_ns,+ cpu_time_ok: cpu_time_ns.is_some(),+ };+ LEDGER.update_physical(allocator, process);+}++fn wait_for_stop_or_period(stop: &AtomicBool, wake: &(Mutex<()>, Condvar), period: Duration) {+ let guard = wake.0.lock().unwrap_or_else(|e| e.into_inner());+ let _ = wake+ .1+ .wait_timeout_while(guard, period, |_| !stop.load(Ordering::Acquire));+}++fn send_snapshot(+ tx: &mpsc::SyncSender<String>,+ dropped: &AtomicU64,+ kind: EventKind,+ sampled: bool,+) {+ let Some(event) = ingest_memory::snapshot_event(kind, sampled) else {+ return;+ };+ match serde_json::to_string(&event) {+ Ok(line) => {+ if tx.try_send(line).is_err() {+ dropped.fetch_add(1, Ordering::Relaxed);+ LEDGER.record_dropped_event();+ }+ }+ Err(_) => {+ dropped.fetch_add(1, Ordering::Relaxed);+ LEDGER.record_dropped_event();+ }+ }+}++#[cfg(unix)]+fn process_cpu_time_ns() -> Option<u64> {+ let mut time = libc::timespec {+ tv_sec: 0,+ tv_nsec: 0,+ };+ // SAFETY: `time` is a valid writable timespec and the process CPU clock+ // requires no caller-managed lifetime.+ if unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, &mut time) } != 0 {+ return None;+ }+ let seconds = u64::try_from(time.tv_sec).ok()?;+ let nanos = u64::try_from(time.tv_nsec).ok()?;+ seconds.checked_mul(1_000_000_000)?.checked_add(nanos)+}++#[cfg(not(unix))]+fn process_cpu_time_ns() -> Option<u64> {+ None+}++fn writer_loop(rx: mpsc::Receiver<String>, dropped: Arc<AtomicU64>) {+ let output =+ std::env::var("XERJ_INGEST_MEMORY_OUTPUT").unwrap_or_else(|_| "tracing".to_string());+ if output == "tracing" {+ loop {+ match rx.recv() {+ Ok(line) => tracing::info!(target: "xerj::ingest_memory", "{line}"),+ Err(_) => break,+ }+ }+ while let Ok(line) = rx.try_recv() {+ tracing::info!(target: "xerj::ingest_memory", "{line}");+ }+ return;+ }++ let mut file = match open_output(&output) {+ Ok(file) => file,+ Err(error) => {+ tracing::warn!(+ target: "xerj::ingest_memory",+ %error,+ path = %output,+ "could not open ingest-memory NDJSON output"+ );+ return;+ }+ };+ loop {+ match rx.recv() {+ Ok(line) => {+ if let Err(error) = writeln!(file, "{line}") {+ record_sink_error(&dropped, &output, &error);+ break;+ }+ }+ Err(_) => break,+ }+ }+ while let Ok(line) = rx.try_recv() {+ if let Err(error) = writeln!(file, "{line}") {+ record_sink_error(&dropped, &output, &error);+ break;+ }+ }+}++fn open_output(path: &str) -> std::io::Result<std::fs::File> {+ let mut options = std::fs::OpenOptions::new();+ options.create(true).append(true);+ #[cfg(unix)]+ {+ use std::os::unix::fs::OpenOptionsExt;+ // `mode` applies only when this call creates the file. An existing+ // operator-managed file keeps its permissions and append semantics.+ options.mode(0o600);+ }+ options.open(path)+}++fn record_sink_error(dropped: &AtomicU64, path: &str, error: &std::io::Error) {+ dropped.fetch_add(1, Ordering::Relaxed);+ LEDGER.record_dropped_event();+ tracing::warn!(+ target: "xerj::ingest_memory",+ %error,+ path,+ "could not write ingest-memory NDJSON output; stopping file sink"+ );+}++#[cfg(not(target_env = "msvc"))]+fn allocator_snapshot() -> AllocatorSnapshot {+ use tikv_jemalloc_ctl::{epoch, stats};+ if epoch::advance().is_err() {+ return AllocatorSnapshot::default();+ }+ match (+ stats::allocated::read(),+ stats::active::read(),+ stats::resident::read(),+ ) {+ (Ok(allocated), Ok(active), Ok(resident)) => AllocatorSnapshot {+ allocated: Some(allocated as u64),+ active: Some(active as u64),+ resident: Some(resident as u64),+ epoch_ok: true,+ },+ _ => AllocatorSnapshot::default(),+ }+}++#[cfg(target_env = "msvc")]+fn allocator_snapshot() -> AllocatorSnapshot {+ AllocatorSnapshot::default()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn bounded_sink_reports_overflow_without_blocking() {+ let (tx, _rx) = mpsc::sync_channel(1);+ let dropped = AtomicU64::new(0);+ let line = r#"{"schema":"xerj.ingest_memory.v1"}"#.to_string();+ assert!(tx.try_send(line.clone()).is_ok());+ if tx.try_send(line).is_err() {+ dropped.fetch_add(1, Ordering::Relaxed);+ }+ assert_eq!(dropped.load(Ordering::Relaxed), 1);+ }++ #[test]+ fn allocator_probe_is_explicitly_available_or_null() {+ let snapshot = allocator_snapshot();+ if snapshot.epoch_ok {+ assert!(snapshot.allocated.is_some());+ assert!(snapshot.active.is_some());+ assert!(snapshot.resident.is_some());+ } else {+ assert_eq!(snapshot.allocated, None);+ assert_eq!(snapshot.active, None);+ assert_eq!(snapshot.resident, None);+ }+ }++ #[test]+ fn process_cpu_probe_is_explicitly_available_or_null() {+ let first = process_cpu_time_ns();+ let second = process_cpu_time_ns();+ match (first, second) {+ (Some(first), Some(second)) => assert!(second >= first),+ (None, None) => {}+ _ => panic!("CPU-time probe support changed within one process"),+ }+ }++ #[test]+ fn sampler_wait_is_interruptible_even_at_max_period() {+ let stop = Arc::new(AtomicBool::new(false));+ let wake = Arc::new((Mutex::new(()), Condvar::new()));+ let thread_stop = Arc::clone(&stop);+ let thread_wake = Arc::clone(&wake);+ let started = std::time::Instant::now();+ let handle = std::thread::spawn(move || {+ wait_for_stop_or_period(&thread_stop, &thread_wake, Duration::from_secs(60));+ });+ stop.store(true, Ordering::Release);+ wake.1.notify_all();+ handle.join().unwrap();+ assert!(started.elapsed() < Duration::from_secs(1));+ }++ #[test]+ fn stop_refresh_observes_changed_gauge_without_waiting_for_period() {+ LEDGER.observe(ingest_memory::Category::MemtableActive, 8192);++ let started = std::time::Instant::now();+ refresh_snapshot_state_with(+ 0,+ AllocatorSnapshot {+ allocated: Some(11),+ active: Some(12),+ resident: Some(13),+ epoch_ok: true,+ },+ 14,+ Some(15),+ );++ assert!(started.elapsed() < Duration::from_secs(1));+ let memtable = LEDGER.gauge(+ ingest_memory::Category::MemtableActive,+ ingest_memory::Measurement::Estimated,+ );+ assert_eq!(memtable.current, 0);+ }++ #[cfg(unix)]+ #[test]+ fn output_file_is_private_when_created_and_preserves_existing_mode() {+ use std::os::unix::fs::PermissionsExt;++ let dir = tempfile::tempdir().unwrap();+ let created = dir.path().join("created.ndjson");+ drop(open_output(created.to_str().unwrap()).unwrap());+ assert_eq!(+ std::fs::metadata(&created).unwrap().permissions().mode() & 0o777,+ 0o600+ );++ let existing = dir.path().join("existing.ndjson");+ std::fs::write(&existing, b"existing\n").unwrap();+ std::fs::set_permissions(&existing, std::fs::Permissions::from_mode(0o640)).unwrap();+ let mut file = open_output(existing.to_str().unwrap()).unwrap();+ writeln!(file, "appended").unwrap();+ drop(file);+ assert_eq!(+ std::fs::metadata(&existing).unwrap().permissions().mode() & 0o777,+ 0o640+ );+ assert_eq!(+ std::fs::read_to_string(existing).unwrap(),+ "existing\nappended\n"+ );+ }++ #[cfg(target_os = "linux")]+ #[test]+ fn output_write_failure_is_accounted() {+ let mut file = open_output("/dev/full").unwrap();+ let dropped = AtomicU64::new(0);+ let error = writeln!(file, "cannot fit").unwrap_err();+ record_sink_error(&dropped, "/dev/full", &error);+ assert_eq!(dropped.load(Ordering::Relaxed), 1);+ }+}diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex 5f8814b81..e577e6132 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -77,6 +77,7 @@ use xerj_engine::Engine;#[cfg(all(target_os = "linux", feature = "debug-profiling"))]mod debug_profiling;mod grpc;+mod ingest_memory_trace;// ─────────────────────────────────────────────────────────────────────────────// CLI@@ -209,6 +210,12 @@ fn print_help() {XERJ_ONNX_TOKENIZER matching tokenizer.json path\n\XERJ_COMPAT_DISTRIBUTION elasticsearch|opensearch — same as --compat-distribution\n\XERJ_COMPAT_VERSION same as --compat-version\n\+ XERJ_INGEST_MEMORY_TRACE off|summary (default: off); summary emits bounded\n\+ periodic NDJSON diagnostics, never per-item events\n\+ XERJ_INGEST_MEMORY_SAMPLE_MS sampler period, clamped to 25..60000 ms\n\+ XERJ_INGEST_MEMORY_OUTPUT tracing|<PATH> (default: tracing); append NDJSON to path\n\+ Merge/cache fields are reserved/unavailable in v1.\n\+ Active+drained may transiently overlap across sampling.\n\\n\WIRE-COMPAT IDENTITY — three ways to run this, pick what fits:\n\\n\@@ -1482,6 +1489,8 @@ async fn async_main() -> Result<()> {// is the structural guard against the 112 GiB OOM class — writes get// a 429 circuit_breaking_exception before the kernel OOM-kills us.state.engine.spawn_resource_sampler();++ let _ingest_memory_trace = ingest_memory_trace::spawn(&state.engine);// 13. Start servers concurrentlylet rest_tls = tls_config.clone();let rest = tokio::spawn(async move {