AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(read-under-write): kill the quadratic merge + O(N) uncached-read stack behind the 15s p99 tail
xerj-orge270e66fd387
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 29ec1f316..1ee32a447 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -3743,6 +3743,7 @@ dependencies = ["chrono-tz","criterion","dashmap",+ "libc","memchr","parking_lot","rayon",diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.tomlindex 1b04beded..d93ee1b06 100644--- a/engine/crates/xerj-engine/Cargo.toml+++ b/engine/crates/xerj-engine/Cargo.toml@@ -30,6 +30,7 @@ dashmap = { workspace = true }arc-swap = { workspace = true }parking_lot = "0.12"rayon = { workspace = true }+libc = "0.2"uuid = { workspace = true }sha2 = "0.10" # v0.9 9-P4 — audit log hash chainchrono = { workspace = true }diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex 1798404f1..832c1c62a 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -176,7 +176,9 @@ pub async fn process_bulk_with_opts(}let t_parse = std::time::Instant::now();- let parse_results: Vec<ParseOutcome> = pairs+ // Dedicated ingest pool: keeps bulk-parse bursts off the global rayon+ // pool that the search path fans out on (see `crate::ingest_pool`).+ let parse_results: Vec<ParseOutcome> = crate::ingest_pool().install(|| pairs.par_iter().map(|(_line_idx, action_line, doc_line, item_index)| {// M5.12 — FAST-PATH manual parse of the action line.@@ -540,7 +542,7 @@ pub async fn process_bulk_with_opts(dynamic_templates: dynamic_templates_per_item,})})- .collect();+ .collect());let parse_ms = t_parse.elapsed().as_millis() as u64;let t_group = std::time::Instant::now();diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rsindex 57d2e3973..239335e70 100644--- a/engine/crates/xerj-engine/src/fast_aggs.rs+++ b/engine/crates/xerj-engine/src/fast_aggs.rs@@ -72,13 +72,65 @@ struct SegEntry {docs: u32,}+/// Memtable docs for the fast path.+///+/// `Owned` is the original brute-parity representation: every buffered doc+/// deep-cloned with `_id` / `_index` / `_seq_no` injected. It is only+/// needed when the agg tree can actually OBSERVE those meta fields+/// (`top_hits`, or an agg targeting `_id`/`_index`/`_seq_no`). For every+/// other shape we use `Shared`: Arc clones of the buffered sources — no+/// deep clone, no injection. Under a bulk writer the memtable holds 10⁴-10⁵+/// docs, and the per-request Owned build (~100-300 ms of `clone_subtree` ++/// version-map lookups) dominated terms/cardinality latency.+enum MemDocs {+ Owned(Vec<Value>),+ Shared { ids: Vec<String>, srcs: Vec<std::sync::Arc<Value>> },+}++impl MemDocs {+ fn len(&self) -> usize {+ match self {+ MemDocs::Owned(v) => v.len(),+ MemDocs::Shared { srcs, .. } => srcs.len(),+ }+ }+ fn is_empty(&self) -> bool {+ self.len() == 0+ }+ fn iter(&self) -> Box<dyn Iterator<Item = &Value> + '_> {+ match self {+ MemDocs::Owned(v) => Box::new(v.iter()),+ MemDocs::Shared { srcs, .. } => Box::new(srcs.iter().map(|a| a.as_ref())),+ }+ }+ /// Materialise doc `i` brute-corpus-style (meta fields injected).+ fn owned(&self, i: usize, idx: &Index) -> Option<Value> {+ match self {+ MemDocs::Owned(v) => v.get(i).cloned(),+ MemDocs::Shared { ids, srcs } => {+ let mut v: Value = srcs.get(i)?.as_ref().clone();+ let id = ids.get(i)?;+ if let Some(o) = v.as_object_mut() {+ o.entry("_id".to_string())+ .or_insert_with(|| Value::String(id.clone()));+ o.entry("_index".to_string())+ .or_insert_with(|| Value::String(idx.name.to_string()));+ if let Some(seq) = idx.lookup_seq_no(id) {+ o.entry("_seq_no".to_string()).or_insert_with(|| json!(seq));+ }+ }+ Some(v)+ }+ }+ }+}+pub(super) struct FastCtx<'a> {idx: &'a Index,segs: Vec<SegEntry>,- /// Memtable docs materialised brute-style: `_source` + `_id`, `_index`,- /// `_seq_no` injected. Empty when the memtable is empty (steady state- /// for a flushed corpus).- mem_docs: Vec<Value>,+ /// Memtable docs (see `MemDocs`). Empty when the memtable is empty+ /// (steady state for a flushed corpus).+ mem_docs: MemDocs,/// Schema fields mapped `boolean`. Their `.dv` columns are numeric/// 0/1 (indistinguishable from real integers at the column level), but/// the brute path renders them as "false"/"true" term keys — so the@@ -232,8 +284,12 @@ impl Index {if total_physical < FAST_AGG_MIN_DOCS {return None;}+ // Exact ghost signal — see the matching gate in `search_inner` and+ // `VersionMap::ghost_events` (the old live-vs-physical arithmetic+ // false-positived on physical-count drift and disabled this fast+ // path under append-only write load).let deletes_present = snap.segments.iter().any(|m| m.has_tombstones)- || self.live_doc_count() < total_physical;+ || self.store.version_map.ghost_events() > 0;if deletes_present {return None;}@@ -270,25 +326,47 @@ impl Index {// Memtable docs, materialised brute-style (see `search_inner`'s// full-corpus assembly): `_id` + `_index` + `_seq_no` injected.- let mem_docs: Vec<Value> = if mem_physical > 0 {- self.memtable- .all_docs_with_sources()- .into_iter()- .map(|(id, mut v)| {- if let Some(o) = v.as_object_mut() {- o.entry("_id".to_string())- .or_insert_with(|| Value::String(id.clone()));- o.entry("_index".to_string())- .or_insert_with(|| Value::String(self.name.to_string()));- if let Some(seq) = self.lookup_seq_no(&id) {- o.entry("_seq_no".to_string()).or_insert_with(|| json!(seq));+ // Owned (deep-clone + meta-field injection) only when the agg tree+ // can observe meta fields; Arc-shared otherwise. Conservative+ // substring probe — false positives merely fall back to the exact+ // legacy build.+ let needs_owned_mem = {+ let raw = aggs_def.to_string();+ raw.contains("top_hits")+ || raw.contains("\"_id\"")+ || raw.contains("\"_index\"")+ || raw.contains("\"_seq_no\"")+ };+ let mem_docs: MemDocs = if mem_physical == 0 {+ MemDocs::Owned(Vec::new())+ } else if needs_owned_mem {+ MemDocs::Owned(+ self.memtable+ .all_docs_with_sources()+ .into_iter()+ .map(|(id, mut v)| {+ if let Some(o) = v.as_object_mut() {+ o.entry("_id".to_string())+ .or_insert_with(|| Value::String(id.clone()));+ o.entry("_index".to_string())+ .or_insert_with(|| Value::String(self.name.to_string()));+ if let Some(seq) = self.lookup_seq_no(&id) {+ o.entry("_seq_no".to_string()).or_insert_with(|| json!(seq));+ }}- }- v- })- .collect()+ v+ })+ .collect(),+ )} else {- Vec::new()+ let pairs = self.memtable.all_docs_with_sources_arc();+ let mut ids = Vec::with_capacity(pairs.len());+ let mut srcs = Vec::with_capacity(pairs.len());+ for (id, src) in pairs {+ ids.push(id);+ srcs.push(src);+ }+ MemDocs::Shared { ids, srcs }};let ctx = FastCtx {@@ -376,7 +454,7 @@ impl<'a> FastCtx<'a> {fn fetch_doc(&self, r: DocRef) -> Option<Value> {match r {- DocRef::Mem(i) => self.mem_docs.get(i).cloned(),+ DocRef::Mem(i) => self.mem_docs.owned(i, self.idx),DocRef::Seg(si, row) => self.fetch_seg_doc(si, row),}}@@ -616,6 +694,7 @@ impl<'a> FastCtx<'a> {self.exec_metric_top(metric_kind_of(agg_type)?, params)}"terms" => self.exec_terms(params, sub),+ "cardinality" => self.exec_cardinality(params, sub),"range" => self.exec_range(params, sub),"date_range" => self.exec_date_range(params, sub),"filter" => self.exec_filter(params, sub),@@ -683,7 +762,7 @@ impl<'a> FastCtx<'a> {field: field.to_string(),meta: None,};- for d in &self.mem_docs {+ for d in self.mem_docs.iter() {Self::fold_mem_metric(d, &spec, &mut acc);}Some(Self::emit_metric(kind, &acc))@@ -808,6 +887,124 @@ impl<'a> FastCtx<'a> {// ── terms ────────────────────────────────────────────────────────────+ /// `cardinality` — exact distinct count from the keyword term+ /// dictionaries, matching the brute path bit-for-bit (xerj's+ /// cardinality is exact, not HLL — see aggs.rs `run_cardinality`).+ ///+ /// Pre-fix `cardinality` was NOT on the fast path, so every uncached+ /// `size:0 + match_all + cardinality` request fell into the+ /// `need_full_corpus` branch of `search_inner`: decompress + JSON-parse+ /// EVERY stored doc in the index (~22 s at 3.5 M docs). Steady-state+ /// dashboards never noticed because the query cache absorbed it, but a+ /// background bulk writer bumps `dataset_version` on every batch — 100 %+ /// cache misses — and cardinality became the single biggest CPU bomb in+ /// the read-under-write collapse.+ ///+ /// Keyword-shaped columns only (plus schema-boolean 0/1 numerics):+ /// numeric distinct values can't be re-rendered into the brute path's+ /// string keys without risking formatting drift → bail to brute.+ fn exec_cardinality(&self, params: &Value, sub: Option<&Value>) -> Option<Value> {+ if sub.is_some() {+ return None;+ }+ // `precision_threshold` is accepted-and-ignored by the brute path+ // (exact count regardless), so accepting it here is bit-identical.+ if !params_only(params, &["field", "missing", "precision_threshold"]) {+ return None;+ }+ let field = params.get("field").and_then(Value::as_str)?;+ let is_bool = self.bool_fields.contains(field);+ match self.seg_field_kind(field) {+ Ok(Some(ColKind::Keyword)) | Ok(None) => {}+ Ok(Some(ColKind::Numeric)) if is_bool => {}+ _ => return None,+ }+ // Same `missing` placeholder semantics as `run_cardinality`.+ let missing_placeholder: Option<String> =+ params.get("missing").and_then(|v| match v {+ Value::String(s) => Some(s.clone()),+ Value::Number(n) => Some(n.to_string()),+ Value::Bool(b) => Some(b.to_string()),+ _ => None,+ });++ let mut distinct: std::collections::HashSet<String> =+ std::collections::HashSet::new();+ const BOOL_TERMS: [&str; 2] = ["false", "true"];+ for seg in &self.segs {+ match seg.cols.get(field) {+ Some(Column::Keyword(k)) => {+ for (ord, &cnt) in k.per_ord_count.iter().enumerate() {+ if cnt > 0 {+ distinct.insert(k.terms[ord].clone());+ }+ }+ if let Some(ph) = &missing_placeholder {+ if k.null_bitmap.len() > 0 {+ distinct.insert(ph.clone());+ }+ }+ }+ Some(Column::Numeric(n)) if is_bool => {+ // Same 0/1 purity check as exec_terms: stray numbers+ // would render as "0"/"2.5" on the brute path → bail.+ let zeros = n.range_count(0.0, 0.0, true, true);+ let ones = n.range_count(1.0, 1.0, true, true);+ if zeros + ones != n.live_count {+ return None;+ }+ if zeros > 0 {+ distinct.insert(BOOL_TERMS[0].to_string());+ }+ if ones > 0 {+ distinct.insert(BOOL_TERMS[1].to_string());+ }+ if let Some(ph) = &missing_placeholder {+ if n.null_bitmap.len() > 0 {+ distinct.insert(ph.clone());+ }+ }+ }+ Some(_) => return None,+ None => {+ // Field absent from the whole segment: every doc counts+ // as missing (brute: empty vals → placeholder).+ if let Some(ph) = &missing_placeholder {+ if seg.docs > 0 {+ distinct.insert(ph.clone());+ }+ }+ }+ }+ }+ // Memtable docs go through the exact brute extractor so string+ // rendering is identical.+ for d in self.mem_docs.iter() {+ let vals = crate::aggs::extract_field_values(d, field);+ if vals.is_empty() {+ if let Some(ph) = &missing_placeholder {+ distinct.insert(ph.clone());+ }+ continue;+ }+ for v in vals {+ distinct.insert(v);+ }+ }++ // Identical output shape to `run_cardinality` including the+ // internal `__xy_*` fields the cross-index merge consumes.+ let values: Vec<Value> = distinct+ .iter()+ .map(|s| Value::String(s.clone()))+ .collect();+ Some(json!({+ "value": distinct.len(),+ "__xy_agg__": "cardinality",+ "__xy_values__": values,+ }))+ }+fn exec_terms(&self, params: &Value, sub: Option<&Value>) -> Option<Value> {if !params_only(params, &["field", "size", "order", "shard_size", "min_doc_count"]) {return None;@@ -1112,7 +1309,7 @@ impl<'a> FastCtx<'a> {}}}- for doc in &self.mem_docs {+ for doc in self.mem_docs.iter() {let Some(v) = extract_numeric(doc, field) else { continue };let matches = match (from, to) {(Some(f), Some(t)) => v >= f && v < t,@@ -1258,7 +1455,7 @@ impl<'a> FastCtx<'a> {}}}- for doc in &self.mem_docs {+ for doc in self.mem_docs.iter() {let v = doc.get(field);let Some(ms) = v.and_then(parse_date_ms) else { continue };let pass_from = from_ms.map(|f| ms >= f).unwrap_or(true);@@ -1363,7 +1560,7 @@ impl<'a> FastCtx<'a> {}}}- for doc in &self.mem_docs {+ for doc in self.mem_docs.iter() {if doc_matches_filter(doc, filter_query) {count += doc_count_weight(doc);for (mi, spec) in plan.metrics.iter().enumerate() {@@ -1451,7 +1648,7 @@ impl<'a> FastCtx<'a> {}}}- for doc in &self.mem_docs {+ for doc in self.mem_docs.iter() {if doc_matches_filter(doc, filter_query) {count += doc_count_weight(doc);for (mi, spec) in plan.metrics.iter().enumerate() {@@ -1519,7 +1716,7 @@ impl<'a> FastCtx<'a> {}}}- for doc in &self.mem_docs {+ for doc in self.mem_docs.iter() {let matches: Vec<bool> = filters_map.values().map(|q| doc_matches_filter(doc, q))@@ -1649,7 +1846,7 @@ impl<'a> FastCtx<'a> {*counts.entry(key).or_insert(0) += c;}}- for doc in &self.mem_docs {+ for doc in self.mem_docs.iter() {let weight = doc_count_weight(doc);// Cross-product across multi-valued fields — brute semantics.let mut keys: Vec<Vec<String>> = vec![vec![]];@@ -1738,7 +1935,7 @@ impl<'a> FastCtx<'a> {// match_all every doc scores equal, so the brute sampler's// stable score sort preserves exactly this prefix.let mut sample: Vec<Value> = Vec::with_capacity(shard_size);- for d in &self.mem_docs {+ for d in self.mem_docs.iter() {if sample.len() >= shard_size {break;}@@ -1982,7 +2179,7 @@ impl<'a> FastCtx<'a> {}// Memtable docs — brute multi-value date extraction with per-doc// bucket dedup.- for doc in &self.mem_docs {+ for doc in self.mem_docs.iter() {let raws = extract_date_ms_values(doc, field);let weight = doc_count_weight(doc);let mut seen: Vec<i64> = Vec::with_capacity(raws.len());diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 4a4336f0e..f587b83e1 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -364,6 +364,17 @@ pub struct Index {/// the merge-completion site alongside dv_cache/stored_value_cache).stored_slices_cache: Arc<dashmap::DashMap<String, Arc<StoredSlices>>>,stored_slices_cache_bytes: Arc<AtomicU64>,+ /// Per-segment cache of the DECOMPRESSED stored section bytes for the+ /// UNSORTED scan path (`scan_stored_section_into`). The bounded+ /// collector already stops the scan O(from+size) docs in, but every+ /// query still paid the full zstd/LZ4 decompress of the FIRST segment+ /// it touched — for a merged multi-million-doc segment that is+ /// 0.5-1.5 s per query, and under a bulk writer (100 % query-cache+ /// misses) it WAS the match_all/bool/range read-under-write tail.+ /// Budgeted like `stored_slices_cache`; entries evicted by id at the+ /// merge-completion site. Segments are immutable → no invalidation.+ decoded_stored_cache: Arc<dashmap::DashMap<String, Arc<Vec<u8>>>>,+ decoded_stored_cache_bytes: Arc<AtomicU64>,/// Regexp term-dictionary expansion cache — keyed by/// `(segment_id, field, pattern)`, holds the exact union doc count/// plus the first `REGEXP_EXPANSION_POS_CAP` matching doc positions@@ -535,6 +546,8 @@ impl Index {stored_value_cache: Arc::new(dashmap::DashMap::new()),stored_slices_cache: Arc::new(dashmap::DashMap::new()),stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),+ decoded_stored_cache: Arc::new(dashmap::DashMap::new()),+ decoded_stored_cache_bytes: Arc::new(AtomicU64::new(0)),regexp_expand_cache: Arc::new(dashmap::DashMap::new()),fast_date_cache: Arc::new(dashmap::DashMap::new()),query_cache: Arc::new(dashmap::DashMap::new()),@@ -630,6 +643,16 @@ impl Index {let memtable_doc_count = memtable.doc_count() as u64;let total_doc_count = segment_doc_count + memtable_doc_count;+ // Conservative ghost init: at rest (no in-flight flushes) a live+ // count below the physical count means superseded copies exist+ // from a pre-restart history the WAL replay above cannot see.+ // Flag them so the delete-aware slow paths stay on — the exact+ // per-event tracking (`VersionMap::ghost_events`) only observes+ // events from this process lifetime onward.+ if (store.version_map.live_count() as u64) < total_doc_count {+ store.version_map.force_ghost_event();+ }+// See create_with_settings — doc count is a sanity cap, byte threshold drives flushes.let flush_doc_threshold = 500_000usize;let flush_byte_threshold = (config.storage.flush_size_mb as usize).saturating_mul(1024 * 1024);@@ -690,6 +713,8 @@ impl Index {stored_value_cache: Arc::new(dashmap::DashMap::new()),stored_slices_cache: Arc::new(dashmap::DashMap::new()),stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),+ decoded_stored_cache: Arc::new(dashmap::DashMap::new()),+ decoded_stored_cache_bytes: Arc::new(AtomicU64::new(0)),regexp_expand_cache: Arc::new(dashmap::DashMap::new()),fast_date_cache: Arc::new(dashmap::DashMap::new()),query_cache: Arc::new(dashmap::DashMap::new()),@@ -1353,15 +1378,21 @@ impl Index {// the memtable entry can share the allocation (no deep clone at// insert time — pre-fix `insert` did `Arc::new(source.clone())`// per doc under the shard write lock).- let sources: Vec<Arc<Value>> = docs- .par_iter()- .map(|(_, bytes)| {- Arc::new(- serde_json::from_slice::<Value>(bytes)- .unwrap_or_else(|_| Value::Object(serde_json::Map::new())),- )- })- .collect();+ //+ // Runs on the dedicated ingest pool (`crate::ingest_pool`) so bulk+ // parse bursts never queue ahead of search/agg par_iters on the+ // global rayon pool — see the read-under-write collapse notes on+ // `ingest_pool()`.+ let sources: Vec<Arc<Value>> = crate::ingest_pool().install(|| {+ docs.par_iter()+ .map(|(_, bytes)| {+ Arc::new(+ serde_json::from_slice::<Value>(bytes)+ .unwrap_or_else(|_| Value::Object(serde_json::Map::new())),+ )+ })+ .collect()+ });// 2. Dynamic mapping: evolve schema serially, in doc order.for source in &sources {@@ -1390,40 +1421,47 @@ impl Index {let analyzer = mem.default_analyzer().expect("standard analyzer always present");- let analyzed: Vec<Vec<(String, Vec<xerj_fts::analyzer::Token>)>> = sources- .par_iter()- .map(|source| crate::memtable::analyze_doc(source.as_ref(), schema, &analyzer))- .collect();+ let analyzed: Vec<Vec<(String, Vec<xerj_fts::analyzer::Token>)>> =+ crate::ingest_pool().install(|| {+ sources+ .par_iter()+ .map(|source| {+ crate::memtable::analyze_doc(source.as_ref(), schema, &analyzer)+ })+ .collect()+ });let n_shards = mem.shard_count().max(1);let mut buckets: Vec<Vec<usize>> = vec![Vec::new(); n_shards];for (i, (id, _)) in docs.iter().enumerate() {buckets[mem.shard_for_dynamic(id)].push(i);}- buckets.par_iter().enumerate().for_each(|(shard, idxs)| {- if idxs.is_empty() {- return;- }- mem.with_shard_mut(shard, |m| {- for &i in idxs {- let (id, bytes) = &docs[i];- // `remove` first so a same-id overwrite within- // the batch leaves exactly one live entry- // (mirrors index_document).- m.remove(id);- // Same sizing formula as `insert` (`raw * 3 + 64`)- // with the raw NDJSON byte length standing in for- // `source.to_string().len()` — keeps the size- // re-serialisation off the lock too.- let size = (bytes.len() + id.len()) * 3 + 64;- m.insert_analyzed(- seq_nos[i],- id.clone(),- Arc::clone(&sources[i]),- &analyzed[i],- size,- );+ crate::ingest_pool().install(|| {+ buckets.par_iter().enumerate().for_each(|(shard, idxs)| {+ if idxs.is_empty() {+ return;}+ mem.with_shard_mut(shard, |m| {+ for &i in idxs {+ let (id, bytes) = &docs[i];+ // `remove` first so a same-id overwrite within+ // the batch leaves exactly one live entry+ // (mirrors index_document).+ m.remove(id);+ // Same sizing formula as `insert` (`raw * 3 + 64`)+ // with the raw NDJSON byte length standing in for+ // `source.to_string().len()` — keeps the size+ // re-serialisation off the lock too.+ let size = (bytes.len() + id.len()) * 3 + 64;+ m.insert_analyzed(+ seq_nos[i],+ id.clone(),+ Arc::clone(&sources[i]),+ &analyzed[i],+ size,+ );+ }+ });});});}@@ -2163,6 +2201,12 @@ impl Index {let metas_for_task = metas;tokio::task::spawn_blocking(move || -> Option<MergeOutput> {+ // Entire merge encode runs on the deprioritised ingest+ // pool (nice +10): the decompress + byte-copy + FTS/DV+ // rebuild below saturates cores for seconds, and on the+ // normal-priority blocking pool it stole scheduler time+ // from foreground searches (read-under-write tail).+ crate::ingest_pool().install(move || -> Option<MergeOutput> {use serde_json::value::RawValue;// V4 M4.8 — STREAMING BYTE-COPY MERGE.@@ -2358,10 +2402,17 @@ impl Index {fts_writer.configure_field(field_name.clone(), cfg.clone());}if !fts_input.is_empty() {- fts_writer.add_documents_parallel(&fts_input);- if let Err(e) = fts_writer.finish() {- tracing::warn!("merge: FTS build failed: {e}");- }+ // Dedicated ingest pool: a merged-segment FTS build+ // is the single longest rayon job in the system —+ // on the global pool it queued every concurrent+ // search's par_iter behind it (read-under-write+ // collapse; see `crate::ingest_pool`).+ crate::ingest_pool().install(|| {+ fts_writer.add_documents_parallel(&fts_input);+ if let Err(e) = fts_writer.finish() {+ tracing::warn!("merge: FTS build failed: {e}");+ }+ });}}@@ -2418,6 +2469,7 @@ impl Index {}Some((batch_for_task, merged_meta, live_doc_count as usize, ids_pairs))+ })})};@@ -2537,6 +2589,10 @@ impl Index {self.stored_slices_cache_bytes.fetch_sub(slices.retained_bytes(), Ordering::Relaxed);}+ if let Some((_, bytes)) = self.decoded_stored_cache.remove(id.as_str()) {+ self.decoded_stored_cache_bytes+ .fetch_sub(bytes.len() as u64, Ordering::Relaxed);+ }self.fast_date_cache.retain(|(seg, _), _| seg != id.as_str());}}@@ -3681,6 +3737,12 @@ impl Index {// the working set is far smaller and the cheap O(n)// truncate happens once.if let (Some(key), Ok(ref r)) = (cache_key, &result) {+ // Never cache a timed-out partial result — the dataset+ // version doesn't change on timeout, so a poisoned entry+ // would be served until the next write.+ if r.timed_out {+ return result;+ }if self.query_cache.len() > 1024 {// Cheap eviction: clear the whole cache rather// than implement LRU. It rebuilds on the next@@ -3729,6 +3791,32 @@ impl Index {)));}+ // ── Cooperative deadline ──────────────────────────────────────────+ // The `tokio::time::timeout` wrapper in `search()` is DEAD CODE on+ // the multi-thread runtime: the search body runs inside+ // `block_in_place(block_on(..))`, so the wrapper's first poll only+ // returns when the whole search is already finished — it can never+ // fire mid-flight. Live-verified during the read-under-write+ // benchmark: 2 118 slow queries (p50 19 s, max 39 s) and ZERO+ // "search timed out" log lines; every piled-up search ran to+ // completion even after its client disconnected, which is what let+ // the backlog snowball keep the CPU pegged for minutes after load+ // stopped. Instead we compute a wall-clock deadline here and have+ // the O(N) loops below check it cooperatively, returning partial+ // results with `timed_out: true` (ES semantics).+ let search_deadline: std::time::Instant = std::time::Instant::now()+ + std::time::Duration::from_millis(request.timeout_ms.unwrap_or(30_000));+ let mut deadline_exceeded = false;+ // Slow-query phase attribution (logged for >1 s queries).+ let phase_t0 = std::time::Instant::now();+ let mut phase_marks: Vec<(&'static str, u64)> = Vec::new();+ let mut dbg_segs = 0u32;+ let mut dbg_decode_ms = 0u64;+ let mut dbg_scan_ms = 0u64;+ let mut dbg_fts_ms = 0u64;+ let mut dbg_walked = 0u64;+ let mut dbg_admitted = 0u64;+// size=0 is valid and means "return no hits but still run aggs / return total".let size = request.size;let from = request.from;@@ -4059,11 +4147,11 @@ impl Index {enum MemSnapshot {Empty,FtsHits(Vec<(String, f32)>), // (doc_id, score) from inverted index- AllDocIds(Vec<String>), // MatchAll: IDs only, sources filled later+ AllDocIds(Vec<String>, u64), // MatchAll: bounded IDs + uncollected remainder count/// Fast DocValues hit: (doc_id, internal_index) pairs — source is NOT/// cloned here; it will be fetched only for the final page of hits.DocValuesHits(Vec<(String, usize)>),- DocsForScan(Vec<(String, Value)>), // doc_id + source for term-level scan+ DocsForScan(Vec<(String, Arc<Value>)>), // doc_id + shared source for term-level scan}// Search the memtable (unflushed docs only). Flushed docs are searched@@ -4091,9 +4179,18 @@ impl Index {// fallback with empty inputs.MemSnapshot::Empty} else if matches!(query, QueryNode::MatchAll) {- // Return all doc IDs from the memtable (size > 0 path).- let ids: Vec<String> = mem.all_doc_ids();- MemSnapshot::AllDocIds(ids)+ // size > 0 match_all. Field-sorted requests must offer+ // EVERY buffered doc to the top-N heap; unsorted requests+ // only ever admit the first `materialisation_limit` ids,+ // so clone just those and carry the remainder as a count.+ if sort_topk.is_some() {+ let ids: Vec<String> = mem.all_doc_ids();+ MemSnapshot::AllDocIds(ids, 0)+ } else {+ let (ids, total) = mem.doc_ids_bounded(materialisation_limit);+ let extra = total.saturating_sub(ids.len() as u64);+ MemSnapshot::AllDocIds(ids, extra)+ }} else if is_doc_scan_query(query) {// count_only + Term/Range short-circuits BEFORE// `try_doc_values_query` because the DV-term path walks@@ -4121,9 +4218,13 @@ impl Index {// the hit list (size > 0 or sort-on-field).MemSnapshot::DocValuesHits(hits)} else {- // Full source materialisation — only reached for- // composite queries or when sources are needed.- let docs: Vec<(String, Value)> = mem.all_docs_with_sources();+ // Composite queries (e.g. bool) that the memtable DV+ // path can't resolve: share the buffered sources via+ // Arc — the old deep-clone of the ENTIRE memtable per+ // query (plus a second per-doc clone for `_id`+ // injection below) was the single hottest search-side+ // cost in the read-under-write thread dumps.+ let docs: Vec<(String, Arc<Value>)> = mem.all_docs_with_sources_arc();MemSnapshot::DocsForScan(docs)}} else {@@ -4161,6 +4262,7 @@ impl Index {}// Lock dropped here};+ phase_marks.push(("mem_snapshot", phase_t0.elapsed().as_millis() as u64));match mem_snapshot {MemSnapshot::Empty => {}@@ -4177,7 +4279,11 @@ impl Index {});}}- MemSnapshot::AllDocIds(ids) => {+ MemSnapshot::AllDocIds(ids, uncollected) => {+ // Bounded id page: the remainder still counts toward+ // hits.total (match_all's total is later overwritten by+ // the authoritative live count anyway).+ total_count += uncollected;for doc_id in ids {admit_hit!(count doc_id);}@@ -4197,38 +4303,57 @@ impl Index {}MemSnapshot::DocsForScan(docs) => {// Lock already released — scan docs without holding the memtable lock.+ //+ // `_id` injection (a full per-doc Object clone) is only+ // observable when the query tree can actually read `_id`+ // — a nested Ids query. Detect that ONCE via the query's+ // serialised form (conservative: false positives merely+ // keep the legacy per-doc clone) instead of cloning every+ // buffered doc on every request.+ let query_may_read_id: bool = serde_json::to_string(query)+ .map(|s| s.contains("_id") || s.contains("Ids") || s.contains("ids"))+ .unwrap_or(true);for (doc_id, source) in docs {let matched = if let QueryNode::Ids { values } = query {values.iter().any(|v| v == doc_id.as_str())+ } else if !query_may_read_id || source.get("_id").is_some() {+ doc_matches_query(query, &source)} else {// Inject `_id` so deeply-nested Ids queries (e.g.// function_score → ids) can resolve it from source.- let source_with_id = if source.get("_id").is_some() {- source.clone()- } else if let Some(obj) = source.as_object() {+ let source_with_id = if let Some(obj) = source.as_object() {let mut o = obj.clone();o.insert("_id".to_string(), serde_json::Value::String(doc_id.clone()));serde_json::Value::Object(o)} else {- source.clone()+ (*source).clone()};doc_matches_query(query, &source_with_id)};if matched {- // Only clone source if we'll actually materialise it.+ // Materialise (score + owned source) ONLY when the+ // hit can actually enter the bounded collector /+ // top-N heap; everything else is a bare count.+ // Identical outcome to unconditionally building the+ // Hit: `admit_hit!` drops post-cap hits anyway.if count_only {total_count += 1;- } else {+ } else if sort_topk.is_some()+ || (all_hits.len() < materialisation_limit+ && !seen_ids.contains(&doc_id))+ {let score = score_query_against_doc(query, &source);admit_hit!(Hit {id: doc_id,score,- source,+ source: (*source).clone(),sort: Vec::new(),explain: None,highlight: None,matched_queries: Vec::new(),});+ } else {+ total_count += 1;}}}@@ -4345,6 +4470,7 @@ impl Index {// result is stashed in `precomputed_aggs` and consumed at the// agg-section below.let mut precomputed_aggs: Option<Value> = None;+ phase_marks.push(("mem_admit", phase_t0.elapsed().as_millis() as u64));if is_match_all && size == 0 && request.aggs.is_some() && request.min_score.is_none() {if let Some(aggs_def) = &request.aggs {let r_opt = self@@ -4374,13 +4500,19 @@ impl Index {// inflate the per-segment/memtable physical tally above the version-map// live count). MatchAll is unaffected — overwritten with the// delete-aware live_doc_count() regardless of this flag.- let deletes_present: bool = {- let seg_physical: u64 = snap.segments.iter().map(|m| m.doc_count).sum();- let mem_physical = self.memtable.doc_count() as u64;- snap.segments.iter().any(|m| m.has_tombstones)- || self.live_doc_count() < seg_physical + mem_physical- };-+ // EXACT ghost signal (see `VersionMap::ghost_events`): flushed+ // tombstones OR any overwrite/delete event ever recorded. The old+ // `live_count() < seg_physical + mem_physical` arithmetic also+ // tripped on unrelated physical-count drift — live-verified: two+ // duplicate docs per merged segment turned the gate permanently ON+ // for a pure append-only corpus, disabling the F1 bounded scan and+ // forcing every size>0 term/range/bool into a full O(N) stored+ // scan. That was the dominant per-query cost in the+ // read-under-write collapse.+ let deletes_present: bool = snap.segments.iter().any(|m| m.has_tombstones)+ || self.store.version_map.ghost_events() > 0;++ phase_marks.push(("fast_aggs+gates", phase_t0.elapsed().as_millis() as u64));if precomputed_aggs.is_some() {// Skip the entire segment loop — the fast agg path already// computed everything we need. `total_count` is set above.@@ -4475,6 +4607,7 @@ impl Index {};for meta in &snap.segments {+ dbg_segs += 1;// F1: once the exact total is authoritative AND the bounded// collector is full, remaining segments can neither add a// materialised hit (page is full) nor change `hits.total`@@ -4486,6 +4619,13 @@ impl Index {if count_authoritative && all_hits.len() >= materialisation_limit {break;}+ // Cooperative deadline: stop fanning into further segments+ // once the request timeout has elapsed — partial results+ // with `timed_out: true`, exactly like ES.+ if std::time::Instant::now() >= search_deadline {+ deadline_exceeded = true;+ break;+ }let seg_id = meta.id.clone();let fts_dir = segments_dir.clone();@@ -4496,11 +4636,13 @@ impl Index {// Try FTS path first if we have an FTS query.let mut fts_handled = false;+ let t_fts = std::time::Instant::now();let reader_opt = if needs_fts {FtsIndexReader::open(&fts_dir, &seg_id, &field_refs).ok()} else {None};+ dbg_fts_ms += t_fts.elapsed().as_millis() as u64;if let Some(reader) = reader_opt {let reader = Arc::new(reader);let searcher =@@ -4785,6 +4927,38 @@ impl Index {&mut total_count,);}+ } else if let Some(cached_bytes) = (!sorted_candidates_path)+ .then(|| {+ self.decoded_stored_cache+ .get(seg_id.as_str())+ .map(|e| Arc::clone(e.value()))+ })+ .flatten()+ {+ // Decoded-bytes cache hit: skip open + decompress+ // entirely. With the in-section early stop, the+ // whole per-segment cost is now O(from+size) doc+ // parses instead of O(section) decompress+parse.+ let t_scan = std::time::Instant::now();+ self.scan_stored_section_into(+ &cached_bytes,+ query,+ is_match_all,+ count_only,+ materialisation_limit,+ count_authoritative,+ &mut total_count,+ &mut all_hits,+ &mut seen_ids,+ pre_filter.as_ref(),+ sort_topk.as_mut(),+ None,+ Some(search_deadline),+ &mut deadline_exceeded,+ &mut dbg_walked,+ &mut dbg_admitted,+ );+ dbg_scan_ms += t_scan.elapsed().as_millis() as u64;} else {// Merge-race hardening (2026-07): a snapshot// segment that fails to open is a query error, not@@ -4793,11 +4967,14 @@ impl Index {// With the read lease held by `snap`, retired// segment files persist, so this only fires on// genuine corruption.+ let t_dec = std::time::Instant::now();let seg_reader = self.store.open_segment_arc(&seg_id)?;if let Ok(Some(stored_bytes_raw)) = seg_reader.section(SectionType::Stored) {if let Ok(stored_bytes) =xerj_storage::stored_codec::decode_stored(stored_bytes_raw){+ dbg_decode_ms += t_dec.elapsed().as_millis() as u64;+ let t_scan = std::time::Instant::now();// Record per-doc offsets on the sorted path so// the decompressed section can be cached for// the next query (subject to the budget).@@ -4818,7 +4995,12 @@ impl Index {pre_filter.as_ref(),sort_topk.as_mut(),if want_cache { Some(&mut offsets) } else { None },+ Some(search_deadline),+ &mut deadline_exceeded,+ &mut dbg_walked,+ &mut dbg_admitted,);+ dbg_scan_ms += t_scan.elapsed().as_millis() as u64;// Cache only a COMPLETE offsets map (a// malformed section bails early) and only// within the retained-bytes budget.@@ -4841,6 +5023,26 @@ impl Index {self.stored_slices_cache_bytes.fetch_add(sz, Ordering::Relaxed);}+ } else if !want_cache {+ // Unsorted path: retain the decompressed+ // section so the NEXT query on this+ // segment skips the decompress (the+ // read-under-write match_all/bool/range+ // tail fix — see `decoded_stored_cache`).+ let sz = stored_bytes.len() as u64;+ if self+ .decoded_stored_cache_bytes+ .load(Ordering::Relaxed)+ .saturating_add(sz)+ <= DECODED_STORED_CACHE_BUDGET+ && self+ .decoded_stored_cache+ .insert(seg_id.clone(), Arc::new(stored_bytes))+ .is_none()+ {+ self.decoded_stored_cache_bytes+ .fetch_add(sz, Ordering::Relaxed);+ }}}}@@ -4858,6 +5060,7 @@ impl Index {// faithfully reflects full query semantics (flattened / subobjects:false// / passthrough field resolution the doc-values shortcut can't model),// so we keep it and never overwrite.+ phase_marks.push(("segment_loop", phase_t0.elapsed().as_millis() as u64));let scan_hit_cap: bool = all_hits.len() >= materialisation_limit;// De-duplication is already enforced by `seen_ids` inside the@@ -5308,6 +5511,15 @@ impl Index {.collect();let snap_bg = self.store.snapshot();for seg in &snap_bg.segments {+ // Cooperative deadline: the full-corpus assembly is the+ // most expensive path in the engine (decompress + parse+ // EVERY stored doc). Stop pulling further segments once+ // past the request timeout — aggs become partial and the+ // response carries `timed_out: true`.+ if std::time::Instant::now() >= search_deadline {+ deadline_exceeded = true;+ break;+ }// Merge-race hardening (2026-07): open failures are// errors, not skips — a skipped segment silently drops// its docs from every aggregation bucket. The read@@ -5388,6 +5600,7 @@ impl Index {vec![]};+ phase_marks.push(("hydrate+corpus", phase_t0.elapsed().as_millis() as u64));// --- Run aggregations over the full matched set (before pagination) ---let agg_start = std::time::Instant::now();let agg_result = if let Some(aggs_def) = &request.aggs {@@ -5576,6 +5789,43 @@ impl Index {None};+ let inner_ms = phase_t0.elapsed().as_millis() as u64;+ if inner_ms >= 1_000 {+ phase_marks.push(("aggs+page", inner_ms));+ // Cumulative marks → per-phase deltas for the log line.+ let mut prev = 0u64;+ let breakdown: Vec<String> = phase_marks+ .iter()+ .map(|(name, at)| {+ let d = at.saturating_sub(prev);+ prev = *at;+ format!("{name}={d}ms")+ })+ .collect();+ warn!(+ total_ms = inner_ms,+ index = self.name.as_str(),+ phases = %breakdown.join(" "),+ gates = %format!(+ "needs_fts={} shortcut={:?} deletes={} ghosts={} total={} lim={} segs={} dec={}ms scan={}ms ftsopen={}ms walked={} admitted={} allhits_cap={}",+ query_needs_fts,+ shortcut_count,+ deletes_present,+ self.store.version_map.ghost_events(),+ total,+ materialisation_limit,+ dbg_segs,+ dbg_decode_ms,+ dbg_scan_ms,+ dbg_fts_ms,+ dbg_walked,+ dbg_admitted,+ scan_hit_cap+ ),+ "slow search_inner phase breakdown"+ );+ }+Ok(SearchResult {hits: page,total: TotalHits {@@ -5584,7 +5834,10 @@ impl Index {},took_ms: 0,aggs: agg_result,- timed_out: false,+ // Set when any cooperative deadline check fired: the hits /+ // total / aggs above are partial, exactly like an ES shard+ // timeout. `search()` refuses to cache timed-out results.+ timed_out: deadline_exceeded,profile,max_score: population_max_score,})@@ -7284,6 +7537,16 @@ impl Index {// verify `offsets_out.len() == segment doc_count` before caching —// a malformed section bails early and leaves the vec incomplete.mut offsets_out: Option<&mut Vec<(u32, u32)>>,+ // Cooperative timeout: when `Some`, the walk polls the wall clock+ // every 4 096 docs and aborts (setting `*timed_out`) once past the+ // deadline. One giant merged segment is otherwise a multi-second+ // uninterruptible unit — the between-segment deadline check in+ // `search_inner` can't bound it.+ deadline: Option<std::time::Instant>,+ timed_out: &mut bool,+ // Diagnostics: docs brace-walked and hits admitted by THIS call.+ dbg_walked: &mut u64,+ dbg_admitted: &mut u64,) {// Fast path: scan one doc at a time with a hand-rolled array splitter.// The stored section is always shaped as `[{...}, {...}, ...]` (a@@ -7307,6 +7570,32 @@ impl Index {let mut doc_pos: u32 = 0;loop {+ // F1b — IN-SEGMENT early stop. The between-segment F1 break in+ // `search_inner` skips segments once the page is full and the+ // total is authoritative, but a single giant merged segment+ // still paid a FULL walk here: brace-scan + simd_json parse of+ // every stored doc (~11 s for an uncached size=10 `match_all`+ // at 3.5 M docs — completely masked at steady state by the+ // query cache, catastrophic under a bulk writer where every+ // request is a cache miss). Once the bounded collector is+ // full, nothing later in this section can change the page+ // (admit order is positional) and `total_count` is overwritten+ // by the shortcut/live count anyway — so stop scanning.+ //+ // Excluded, for the same reasons as the outer break:+ // - `count_only`: the scan tally IS the total when no+ // shortcut resolved (count_authoritative already implies a+ // shortcut, but keep the guard belt-and-braces);+ // - `sort_topk`: top-N needs every matching doc's sort key;+ // - `offsets_out`: the offsets cache must cover the section.+ if count_authoritative+ && !count_only+ && sort_topk.is_none()+ && offsets_out.is_none()+ && all_hits.len() >= materialisation_limit+ {+ return;+ }// Skip whitespace/commas between docs.while i < n && (bytes[i].is_ascii_whitespace() || bytes[i] == b',') {i += 1;@@ -7361,6 +7650,18 @@ impl Index {let cur_pos = doc_pos;doc_pos += 1;+ *dbg_walked += 1;++ // Cooperative timeout poll — every 4 096 docs (~1 µs of clock+ // reads per million docs; the parse below costs ~3 µs/doc).+ if doc_pos & 0xFFF == 0 {+ if let Some(dl) = deadline {+ if std::time::Instant::now() >= dl {+ *timed_out = true;+ return;+ }+ }+ }// Pre-filter: if the caller supplied a matching set and this// position isn't in it, skip the parse entirely. We already@@ -7415,6 +7716,7 @@ impl Index {}*total_count += 1;+ *dbg_admitted += 1;if count_only {continue;@@ -7657,6 +7959,10 @@ async fn do_flush_shard(// 50-100 ms segment write that follows, and 100K-doc/s ingest// bench numbers are unaffected.let build_fts = move |meta: &xerj_storage::segment::SegmentMeta| -> xerj_storage::Result<()> {+ // Whole side-car build runs on the dedicated ingest pool so a+ // flush burst can't queue search/agg par_iters behind it on the+ // global rayon pool (read-under-write; see `crate::ingest_pool`).+ crate::ingest_pool().install(|| {// ── Doc-values side-car (always built) ────────────────────{let columns = build_doc_value_columns(@@ -7687,6 +7993,7 @@ async fn do_flush_shard(tracing::warn!("flush: FTS build failed: {e}");}}+ });let _ = &segments_dir;let _ = &drained_fts;@@ -7694,7 +8001,11 @@ async fn do_flush_shard(};// Hand the pre-drained storage entries to the finaliser.- let meta = match store.finalize_flush_with_publisher(storage_drained, build_fts)? {+ // Segment encode + side-car build on the deprioritised ingest pool+ // (build_fts's inner install() is a same-pool no-op).+ let meta = match crate::ingest_pool()+ .install(|| store.finalize_flush_with_publisher(storage_drained, build_fts))?+ {Some(m) => m,None => {tracing::warn!("storage finalize returned None — unexpected");@@ -12105,6 +12416,11 @@ impl StoredSlices {/// path); merge eviction returns the dropped segment's bytes to the budget.const STORED_SLICES_CACHE_BUDGET: u64 = 512 * 1024 * 1024;+/// Budget for `decoded_stored_cache` (raw decompressed stored sections,+/// ~1× the corpus JSON size). Overflow → inserts are refused and the+/// query falls back to the per-query decompress (pre-cache behaviour).+const DECODED_STORED_CACHE_BUDGET: u64 = 4 * 1024 * 1024 * 1024;+/// Normalize one `search_after` cursor value so it compares against the/// normalized `hit.sort` values `compute_sort_values` produces (date-shaped/// strings → epoch-ms / epoch-ns numbers). `fmt_hint` is the SortField'sdiff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rsindex bb001cc70..14d49781f 100644--- a/engine/crates/xerj-engine/src/lib.rs+++ b/engine/crates/xerj-engine/src/lib.rs@@ -26,6 +26,50 @@ pub use engine::{Engine, HealthStatus, IndexInfo};pub use index::{Index, IndexResponse, IndexStats, FieldEncodingInfo, LogFormat, EnrichTable, detect_log_format, resolve_field_alias, resolve_date_math};pub use memtable::FtsMemtable;+// ── Ingest/flush/merge rayon pool ────────────────────────────────────────────++/// Dedicated rayon pool for ingest-side CPU work: bulk-body parse, P2.1+/// per-batch doc parse/analyze/insert, flush FTS+DV side-car builds, and+/// merge FTS rebuilds.+///+/// Pre-fix all of that ran on the GLOBAL rayon pool — the same pool the+/// search path uses for segment fan-out (`search_segments`) and the+/// fast-agg dv-warm par_iter. A single long flush/merge job therefore+/// queued every search's rayon work behind it: with a background bulk+/// writer running, foreground `match_all` p99 was measured at 15 s+ while+/// steady-state reads were ~2 ms (BEAT_ES_MASTER_PLAN Phase 2, read-under-+/// write). Ingest work now runs here; the global pool stays free for+/// search/agg fan-out, so neither side can starve the other's queue (the+/// OS scheduler time-slices the two pools fairly).+///+/// Sized to all available cores: ingest throughput is unchanged when no+/// searches run, and under mixed load the kernel — not the rayon job+/// queue — arbitrates.+pub(crate) fn ingest_pool() -> &'static rayon::ThreadPool {+ static POOL: std::sync::OnceLock<rayon::ThreadPool> = std::sync::OnceLock::new();+ POOL.get_or_init(|| {+ let n = std::thread::available_parallelism()+ .map(|n| n.get())+ .unwrap_or(8);+ rayon::ThreadPoolBuilder::new()+ .num_threads(n)+ .thread_name(|i| format!("xerj-ingest-{i}"))+ // Deprioritise ingest CPU vs foreground reads. Flush/merge+ // side-car builds saturate every core for seconds at a time;+ // at equal priority a ~20 ms search waits behind them for+ // hundreds of ms (measured p50 0.2-1.9 s under a bulk+ // writer). nice(10) lets CFS schedule search threads ahead+ // of background encode work; with no reads running the pool+ // still gets every core, so writer-only throughput is+ // unchanged.+ .start_handler(|_| unsafe {+ let _ = libc::nice(10);+ })+ .build()+ .expect("failed to build ingest rayon pool")+ })+}+// ── Analyzer helpers ──────────────────────────────────────────────────────────use std::sync::Arc;diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex d7929a7df..f3b219571 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -464,6 +464,25 @@ impl ShardedFtsMemtable {out}+ /// Bounded variant: clone at most `limit` doc ids and return the TOTAL+ /// buffered doc count alongside. The unsorted match_all page only+ /// needs `from+size+ε` ids, but `all_doc_ids` cloned every buffered id+ /// on every request — ~1 s per query at a 300 k-doc memtable under+ /// sustained bulk load (the `mem_admit` phase of the read-under-write+ /// breakdown).+ pub fn doc_ids_bounded(&self, limit: usize) -> (Vec<String>, u64) {+ let mut out: Vec<String> = Vec::with_capacity(limit.min(4096));+ let mut total: u64 = 0;+ for s in &self.shards {+ let g = s.read();+ total += g.doc_count() as u64;+ if out.len() < limit {+ out.extend(g.doc_ids_up_to(limit - out.len()));+ }+ }+ (out, total)+ }+/// Return every (doc_id, source) pair. Aggregates across all/// shards; insertion order is preserved within a shard but not/// across shards. Callers that need global order should sort@@ -476,6 +495,15 @@ impl ShardedFtsMemtable {out}+ /// Arc-sharing twin of `all_docs_with_sources` — no deep clones.+ pub fn all_docs_with_sources_arc(&self) -> Vec<(String, Arc<Value>)> {+ let mut out = Vec::new();+ for s in &self.shards {+ out.extend(s.read().all_docs_with_sources_arc());+ }+ out+ }+/// Drain every shard, merge-sort by seq_no, and return the/// combined (doc_id, text_fields, source) stream. This is the/// single entry point used by the flush path — the sort@@ -1631,6 +1659,12 @@ impl FtsMemtable {self.docs.iter().map(|e| e.doc_id.clone()).collect()}+ /// First `n` doc ids (insertion order) — see+ /// `ShardedFtsMemtable::doc_ids_bounded`.+ pub fn doc_ids_up_to(&self, n: usize) -> Vec<String> {+ self.docs.iter().take(n).map(|e| e.doc_id.clone()).collect()+ }+/// Resolve a MemEntry's source Value — if `source` is Null but/// `source_bytes` is non-empty, lazily parse the bytes. This is/// the M5.11 deferred-parse path used by `insert_raw_bytes_with_seq`.@@ -1642,19 +1676,56 @@ impl FtsMemtable {}}- /// Return the full original source JSON for a document by ID.- pub fn get_doc_source_as_value(&self, doc_id: &str) -> Option<Value> {+ /// Arc-sharing twin of `resolve_source`: no deep clone for the common+ /// HTTP-bulk path where the parsed source is already Arc-stored.+ fn resolve_source_arc(entry: &MemEntry) -> Arc<Value> {+ if entry.source.is_null() && !entry.source_bytes.is_empty() {+ Arc::new(serde_json::from_slice(&entry.source_bytes).unwrap_or(Value::Null))+ } else {+ Arc::clone(&entry.source)+ }+ }++ /// All (doc_id, source) pairs WITHOUT deep-cloning the source trees.+ /// The fast-agg path was deep-cloning the entire memtable per agg+ /// request via `all_docs_with_sources` (~100-300 ms/query at 1e5+ /// buffered docs under a bulk writer).+ pub fn all_docs_with_sources_arc(&self) -> Vec<(String, Arc<Value>)> {self.docs.iter()- .find(|e| e.doc_id == doc_id)- .map(Self::resolve_source)+ .map(|e| (e.doc_id.clone(), Self::resolve_source_arc(e)))+ .collect()+ }++ /// Return the full original source JSON for a document by ID.+ /// O(1) doc lookup via `doc_id_index` (maintained by every insert /+ /// remove path), with a linear-scan fallback in case an entry is ever+ /// missing from the index.+ ///+ /// Pre-fix this was an unconditional `docs.iter().find(..)` — a full+ /// linear walk of the shard per call. `fill_memtable_sources`+ /// hydrates up to `materialisation_limit` (256) hits per search, so+ /// with a bulk writer keeping the memtable at 10⁵ docs every search+ /// paid 256 × O(shard) string compares; `get_doc_source_as_value` was+ /// the single hottest frame in the read-under-write profile.+ fn entry_for(&self, doc_id: &str) -> Option<&MemEntry> {+ if let Some(&i) = self.doc_id_index.get(doc_id) {+ if let Some(e) = self.docs.get(i) {+ if e.doc_id == doc_id {+ return Some(e);+ }+ }+ }+ self.docs.iter().find(|e| e.doc_id == doc_id)+ }++ pub fn get_doc_source_as_value(&self, doc_id: &str) -> Option<Value> {+ self.entry_for(doc_id).map(Self::resolve_source)}/// Return the Arc-wrapped source for a document by ID.pub fn get_doc_source_arc(&self, doc_id: &str) -> Option<Arc<Value>> {- self.docs- .iter()- .find(|e| e.doc_id == doc_id)+ self.entry_for(doc_id).map(|e| {if e.source.is_null() && !e.source_bytes.is_empty() {Arc::new(serde_json::from_slice(&e.source_bytes).unwrap_or(Value::Null))@@ -1681,9 +1752,10 @@ impl FtsMemtable {.collect()}- /// Check if a document exists in the memtable.+ /// Check if a document exists in the memtable. O(1) via+ /// `doc_id_index`, with linear fallback (see `entry_for`).pub fn contains(&self, doc_id: &str) -> bool {- self.docs.iter().any(|e| e.doc_id == doc_id)+ self.entry_for(doc_id).is_some()}/// Number of documents in the memtable.diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rsindex b5f4a3cfd..4f7932e84 100644--- a/engine/crates/xerj-fts/src/index.rs+++ b/engine/crates/xerj-fts/src/index.rs@@ -699,12 +699,14 @@ impl FtsIndexWriter {for term in &sorted_terms {if let Some((offset, _skip)) = field_data.postings.encode_term(term, &mut post_data) {- // Calculate doc_freq and ttf from the writer's internal stats+ // Calculate doc_freq and ttf from the writer's internal stats.+ // Direct map lookup — the old `term_stats().find(..)` linear+ // scan made this loop O(terms² × postings): a merged segment+ // with ~1 distinct term/doc (float/date fields) took HOURS of+ // CPU at 3M docs and pinned rayon workers for the duration.let (doc_freq, ttf) = field_data.postings- .term_stats()- .find(|(t, _, _)| *t == term.as_str())- .map(|(_, df, ttf)| (df, ttf))+ .stats_for(term).unwrap_or((0, 0));let end_offset = post_data.len() as u64;diff --git a/engine/crates/xerj-fts/src/postings.rs b/engine/crates/xerj-fts/src/postings.rsindex 620067d1e..a8df4b8bb 100644--- a/engine/crates/xerj-fts/src/postings.rs+++ b/engine/crates/xerj-fts/src/postings.rs@@ -176,6 +176,24 @@ impl PostingsWriter {});}+ /// Direct `(doc_frequency, total_term_frequency)` lookup for ONE term.+ ///+ /// O(log T + postings(term)) — use this in per-term loops. The segment+ /// writer (`write_field_static`) previously resolved stats via+ /// `term_stats().find(term)`, which walks the term map from the start+ /// *and sums every earlier term's postings* on each call — O(T² × P)+ /// across the write loop. Invisible on 625-doc flush segments, but a+ /// merged segment over a float field (≈1 distinct term per doc) spun+ /// 100 % CPU for HOURS at 3 M docs, pinning rayon workers and starving+ /// every search + flush behind it (the read-under-write collapse).+ pub fn stats_for(&self, term: &str) -> Option<(u32, u64)> {+ self.postings.get(term).map(|postings| {+ let doc_freq = postings.len() as u32;+ let ttf: u64 = postings.iter().map(|p| p.term_freq as u64).sum();+ (doc_freq, ttf)+ })+ }+/// Returns an iterator over `(term, doc_frequency, total_term_frequency)`/// without consuming self (for building the FST term dictionary).pub fn term_stats(&self) -> impl Iterator<Item = (&str, u32, u64)> {diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rsindex a0447aba2..10185384a 100644--- a/engine/crates/xerj-storage/src/version_map.rs+++ b/engine/crates/xerj-storage/src/version_map.rs@@ -84,12 +84,31 @@ pub struct VersionMap {/// transiently-interleaved reader can never underflow the type;/// `live_count()` clamps at 0.live: AtomicI64,+ /// Monotonic count of "ghost-producing" events: overwrites of an+ /// existing doc id and deletes. Each such event leaves a physically+ /// present but superseded/tombstoned copy in the memtable or a+ /// segment until a merge purges it.+ ///+ /// The search fast paths (`deletes_present` gates in `search_inner` /+ /// `fast_aggs`) used to INFER this state from+ /// `live_count() < Σ segment doc_count + memtable doc_count` — but+ /// that arithmetic also trips on any unrelated physical-count drift+ /// (live-verified: two duplicate docs per merged segment flipped the+ /// gate permanently on a pure append-only workload, forcing every+ /// size>0 term/range/bool query into a full O(N) stored scan — the+ /// core of the read-under-write collapse). This counter is the+ /// exact signal: zero ⇔ no update/delete has ever produced a ghost.+ ///+ /// Monotonic by design (never decremented on merge): once an index+ /// has seen updates, the delete-aware slow paths stay on. That is+ /// conservative but always correct.+ ghost_events: std::sync::atomic::AtomicU64,}impl VersionMap {/// Create an empty version map.pub fn new() -> Self {- Self { inner: DashMap::new(), live: AtomicI64::new(0) }+ Self { inner: DashMap::new(), live: AtomicI64::new(0), ghost_events: std::sync::atomic::AtomicU64::new(0) }}/// Apply a live-count delta for an entry transition.@@ -128,6 +147,16 @@ impl VersionMap {let old = self.inner.insert(doc_id, VersionEntry { seq_no, segment_id: segment_id.into(), deleted });+ // Overwrite or delete — a superseded/tombstoned physical copy now+ // exists somewhere until merged away. A same-seq_no re-set is NOT+ // an overwrite: the flush path repoints every drained doc's entry+ // from `__memtable__` to its segment id with the seq_no unchanged+ // (live-verified: counting those flipped the gate ON for every+ // flushed doc and re-disabled all fast paths under pure appends).+ let is_overwrite = old.as_ref().map_or(false, |e| e.seq_no != seq_no);+ if is_overwrite || deleted {+ self.ghost_events.fetch_add(1, Ordering::Relaxed);+ }self.live_delta(old.map_or(false, |e| !e.deleted), !deleted);}@@ -215,6 +244,7 @@ impl VersionMap {}let was_live = !occ.get().deleted;occ.insert(VersionEntry { seq_no: new_seq_no, segment_id: new_segment_id, deleted });+ self.ghost_events.fetch_add(1, Ordering::Relaxed);self.live_delta(was_live, !deleted);Ok(new_seq_no)}@@ -228,6 +258,9 @@ impl VersionMap {});}vac.insert(VersionEntry { seq_no: new_seq_no, segment_id: new_segment_id, deleted });+ if deleted {+ self.ghost_events.fetch_add(1, Ordering::Relaxed);+ }self.live_delta(false, !deleted);Ok(new_seq_no)}@@ -254,6 +287,7 @@ impl VersionMap {segment_id: segment_id.into(),deleted: true,};+ self.ghost_events.fetch_add(1, Ordering::Relaxed);self.live_delta(true, false);Ok(true)}@@ -324,6 +358,22 @@ impl VersionMap {pub fn live_count(&self) -> usize {self.live.load(Ordering::Relaxed).max(0) as usize}++ /// Number of overwrite/delete events ever seen (see `ghost_events`+ /// field docs). Zero ⇔ append-only history ⇔ physical postings and+ /// stored copies are 1:1 with live docs (barring storage-level bugs,+ /// which corrupt the brute counting paths identically).+ pub fn ghost_events(&self) -> u64 {+ self.ghost_events.load(Ordering::Relaxed)+ }++ /// Conservatively mark the index as containing ghosts. Called once at+ /// open time when the at-rest arithmetic (`live < physical`) indicates+ /// superseded copies from a pre-restart history the WAL replay cannot+ /// see.+ pub fn force_ghost_event(&self) {+ self.ghost_events.fetch_add(1, Ordering::Relaxed);+ }}impl Default for VersionMap {