Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(read-under-write): kill the quadratic merge + O(N) uncached-read stack behind the 15s p99 tail

xerj-orge270e66fd387
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 29ec1f316..1ee32a447 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -3743,6 +3743,7 @@ dependencies = [
"chrono-tz",
"criterion",
"dashmap",
+ "libc",
"memchr",
"parking_lot",
"rayon",
diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.toml
index 1b04beded..d93ee1b06 100644
--- a/engine/crates/xerj-engine/Cargo.toml
+++ b/engine/crates/xerj-engine/Cargo.toml
@@ -30,6 +30,7 @@ dashmap = { workspace = true }
arc-swap = { workspace = true }
parking_lot = "0.12"
rayon = { workspace = true }
+libc = "0.2"
uuid = { workspace = true }
sha2 = "0.10" # v0.9 9-P4 — audit log hash chain
chrono = { workspace = true }
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index 1798404f1..832c1c62a 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -176,7 +176,9 @@ pub async fn process_bulk_with_opts(
}
let t_parse = std::time::Instant::now();
- let parse_results: Vec<ParseOutcome> = pairs
+ // Dedicated ingest pool: keeps bulk-parse bursts off the global rayon
+ // pool that the search path fans out on (see `crate::ingest_pool`).
+ let parse_results: Vec<ParseOutcome> = crate::ingest_pool().install(|| pairs
.par_iter()
.map(|(_line_idx, action_line, doc_line, item_index)| {
// M5.12 — FAST-PATH manual parse of the action line.
@@ -540,7 +542,7 @@ pub async fn process_bulk_with_opts(
dynamic_templates: dynamic_templates_per_item,
})
})
- .collect();
+ .collect());
let parse_ms = t_parse.elapsed().as_millis() as u64;
let t_group = std::time::Instant::now();
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index 57d2e3973..239335e70 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -72,13 +72,65 @@ struct SegEntry {
docs: u32,
}
+/// Memtable docs for the fast path.
+///
+/// `Owned` is the original brute-parity representation: every buffered doc
+/// deep-cloned with `_id` / `_index` / `_seq_no` injected. It is only
+/// needed when the agg tree can actually OBSERVE those meta fields
+/// (`top_hits`, or an agg targeting `_id`/`_index`/`_seq_no`). For every
+/// other shape we use `Shared`: Arc clones of the buffered sources — no
+/// deep clone, no injection. Under a bulk writer the memtable holds 10⁴-10⁵
+/// docs, and the per-request Owned build (~100-300 ms of `clone_subtree` +
+/// version-map lookups) dominated terms/cardinality latency.
+enum MemDocs {
+ Owned(Vec<Value>),
+ Shared { ids: Vec<String>, srcs: Vec<std::sync::Arc<Value>> },
+}
+
+impl MemDocs {
+ fn len(&self) -> usize {
+ match self {
+ MemDocs::Owned(v) => v.len(),
+ MemDocs::Shared { srcs, .. } => srcs.len(),
+ }
+ }
+ fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+ fn iter(&self) -> Box<dyn Iterator<Item = &Value> + '_> {
+ match self {
+ MemDocs::Owned(v) => Box::new(v.iter()),
+ MemDocs::Shared { srcs, .. } => Box::new(srcs.iter().map(|a| a.as_ref())),
+ }
+ }
+ /// Materialise doc `i` brute-corpus-style (meta fields injected).
+ fn owned(&self, i: usize, idx: &Index) -> Option<Value> {
+ match self {
+ MemDocs::Owned(v) => v.get(i).cloned(),
+ MemDocs::Shared { ids, srcs } => {
+ let mut v: Value = srcs.get(i)?.as_ref().clone();
+ let id = ids.get(i)?;
+ if let Some(o) = v.as_object_mut() {
+ o.entry("_id".to_string())
+ .or_insert_with(|| Value::String(id.clone()));
+ o.entry("_index".to_string())
+ .or_insert_with(|| Value::String(idx.name.to_string()));
+ if let Some(seq) = idx.lookup_seq_no(id) {
+ o.entry("_seq_no".to_string()).or_insert_with(|| json!(seq));
+ }
+ }
+ Some(v)
+ }
+ }
+ }
+}
+
pub(super) struct FastCtx<'a> {
idx: &'a Index,
segs: Vec<SegEntry>,
- /// Memtable docs materialised brute-style: `_source` + `_id`, `_index`,
- /// `_seq_no` injected. Empty when the memtable is empty (steady state
- /// for a flushed corpus).
- mem_docs: Vec<Value>,
+ /// Memtable docs (see `MemDocs`). Empty when the memtable is empty
+ /// (steady state for a flushed corpus).
+ mem_docs: MemDocs,
/// Schema fields mapped `boolean`. Their `.dv` columns are numeric
/// 0/1 (indistinguishable from real integers at the column level), but
/// the brute path renders them as "false"/"true" term keys — so the
@@ -232,8 +284,12 @@ impl Index {
if total_physical < FAST_AGG_MIN_DOCS {
return None;
}
+ // Exact ghost signal — see the matching gate in `search_inner` and
+ // `VersionMap::ghost_events` (the old live-vs-physical arithmetic
+ // false-positived on physical-count drift and disabled this fast
+ // path under append-only write load).
let deletes_present = snap.segments.iter().any(|m| m.has_tombstones)
- || self.live_doc_count() < total_physical;
+ || self.store.version_map.ghost_events() > 0;
if deletes_present {
return None;
}
@@ -270,25 +326,47 @@ impl Index {
// Memtable docs, materialised brute-style (see `search_inner`'s
// full-corpus assembly): `_id` + `_index` + `_seq_no` injected.
- let mem_docs: Vec<Value> = if mem_physical > 0 {
- self.memtable
- .all_docs_with_sources()
- .into_iter()
- .map(|(id, mut v)| {
- if let Some(o) = v.as_object_mut() {
- o.entry("_id".to_string())
- .or_insert_with(|| Value::String(id.clone()));
- o.entry("_index".to_string())
- .or_insert_with(|| Value::String(self.name.to_string()));
- if let Some(seq) = self.lookup_seq_no(&id) {
- o.entry("_seq_no".to_string()).or_insert_with(|| json!(seq));
+ // Owned (deep-clone + meta-field injection) only when the agg tree
+ // can observe meta fields; Arc-shared otherwise. Conservative
+ // substring probe — false positives merely fall back to the exact
+ // legacy build.
+ let needs_owned_mem = {
+ let raw = aggs_def.to_string();
+ raw.contains("top_hits")
+ || raw.contains("\"_id\"")
+ || raw.contains("\"_index\"")
+ || raw.contains("\"_seq_no\"")
+ };
+ let mem_docs: MemDocs = if mem_physical == 0 {
+ MemDocs::Owned(Vec::new())
+ } else if needs_owned_mem {
+ MemDocs::Owned(
+ self.memtable
+ .all_docs_with_sources()
+ .into_iter()
+ .map(|(id, mut v)| {
+ if let Some(o) = v.as_object_mut() {
+ o.entry("_id".to_string())
+ .or_insert_with(|| Value::String(id.clone()));
+ o.entry("_index".to_string())
+ .or_insert_with(|| Value::String(self.name.to_string()));
+ if let Some(seq) = self.lookup_seq_no(&id) {
+ o.entry("_seq_no".to_string()).or_insert_with(|| json!(seq));
+ }
}
- }
- v
- })
- .collect()
+ v
+ })
+ .collect(),
+ )
} else {
- Vec::new()
+ let pairs = self.memtable.all_docs_with_sources_arc();
+ let mut ids = Vec::with_capacity(pairs.len());
+ let mut srcs = Vec::with_capacity(pairs.len());
+ for (id, src) in pairs {
+ ids.push(id);
+ srcs.push(src);
+ }
+ MemDocs::Shared { ids, srcs }
};
let ctx = FastCtx {
@@ -376,7 +454,7 @@ impl<'a> FastCtx<'a> {
fn fetch_doc(&self, r: DocRef) -> Option<Value> {
match r {
- DocRef::Mem(i) => self.mem_docs.get(i).cloned(),
+ DocRef::Mem(i) => self.mem_docs.owned(i, self.idx),
DocRef::Seg(si, row) => self.fetch_seg_doc(si, row),
}
}
@@ -616,6 +694,7 @@ impl<'a> FastCtx<'a> {
self.exec_metric_top(metric_kind_of(agg_type)?, params)
}
"terms" => self.exec_terms(params, sub),
+ "cardinality" => self.exec_cardinality(params, sub),
"range" => self.exec_range(params, sub),
"date_range" => self.exec_date_range(params, sub),
"filter" => self.exec_filter(params, sub),
@@ -683,7 +762,7 @@ impl<'a> FastCtx<'a> {
field: field.to_string(),
meta: None,
};
- for d in &self.mem_docs {
+ for d in self.mem_docs.iter() {
Self::fold_mem_metric(d, &spec, &mut acc);
}
Some(Self::emit_metric(kind, &acc))
@@ -808,6 +887,124 @@ impl<'a> FastCtx<'a> {
// ── terms ────────────────────────────────────────────────────────────
+ /// `cardinality` — exact distinct count from the keyword term
+ /// dictionaries, matching the brute path bit-for-bit (xerj's
+ /// cardinality is exact, not HLL — see aggs.rs `run_cardinality`).
+ ///
+ /// Pre-fix `cardinality` was NOT on the fast path, so every uncached
+ /// `size:0 + match_all + cardinality` request fell into the
+ /// `need_full_corpus` branch of `search_inner`: decompress + JSON-parse
+ /// EVERY stored doc in the index (~22 s at 3.5 M docs). Steady-state
+ /// dashboards never noticed because the query cache absorbed it, but a
+ /// background bulk writer bumps `dataset_version` on every batch — 100 %
+ /// cache misses — and cardinality became the single biggest CPU bomb in
+ /// the read-under-write collapse.
+ ///
+ /// Keyword-shaped columns only (plus schema-boolean 0/1 numerics):
+ /// numeric distinct values can't be re-rendered into the brute path's
+ /// string keys without risking formatting drift → bail to brute.
+ fn exec_cardinality(&self, params: &Value, sub: Option<&Value>) -> Option<Value> {
+ if sub.is_some() {
+ return None;
+ }
+ // `precision_threshold` is accepted-and-ignored by the brute path
+ // (exact count regardless), so accepting it here is bit-identical.
+ if !params_only(params, &["field", "missing", "precision_threshold"]) {
+ return None;
+ }
+ let field = params.get("field").and_then(Value::as_str)?;
+ let is_bool = self.bool_fields.contains(field);
+ match self.seg_field_kind(field) {
+ Ok(Some(ColKind::Keyword)) | Ok(None) => {}
+ Ok(Some(ColKind::Numeric)) if is_bool => {}
+ _ => return None,
+ }
+ // Same `missing` placeholder semantics as `run_cardinality`.
+ let missing_placeholder: Option<String> =
+ params.get("missing").and_then(|v| match v {
+ Value::String(s) => Some(s.clone()),
+ Value::Number(n) => Some(n.to_string()),
+ Value::Bool(b) => Some(b.to_string()),
+ _ => None,
+ });
+
+ let mut distinct: std::collections::HashSet<String> =
+ std::collections::HashSet::new();
+ const BOOL_TERMS: [&str; 2] = ["false", "true"];
+ for seg in &self.segs {
+ match seg.cols.get(field) {
+ Some(Column::Keyword(k)) => {
+ for (ord, &cnt) in k.per_ord_count.iter().enumerate() {
+ if cnt > 0 {
+ distinct.insert(k.terms[ord].clone());
+ }
+ }
+ if let Some(ph) = &missing_placeholder {
+ if k.null_bitmap.len() > 0 {
+ distinct.insert(ph.clone());
+ }
+ }
+ }
+ Some(Column::Numeric(n)) if is_bool => {
+ // Same 0/1 purity check as exec_terms: stray numbers
+ // would render as "0"/"2.5" on the brute path → bail.
+ let zeros = n.range_count(0.0, 0.0, true, true);
+ let ones = n.range_count(1.0, 1.0, true, true);
+ if zeros + ones != n.live_count {
+ return None;
+ }
+ if zeros > 0 {
+ distinct.insert(BOOL_TERMS[0].to_string());
+ }
+ if ones > 0 {
+ distinct.insert(BOOL_TERMS[1].to_string());
+ }
+ if let Some(ph) = &missing_placeholder {
+ if n.null_bitmap.len() > 0 {
+ distinct.insert(ph.clone());
+ }
+ }
+ }
+ Some(_) => return None,
+ None => {
+ // Field absent from the whole segment: every doc counts
+ // as missing (brute: empty vals → placeholder).
+ if let Some(ph) = &missing_placeholder {
+ if seg.docs > 0 {
+ distinct.insert(ph.clone());
+ }
+ }
+ }
+ }
+ }
+ // Memtable docs go through the exact brute extractor so string
+ // rendering is identical.
+ for d in self.mem_docs.iter() {
+ let vals = crate::aggs::extract_field_values(d, field);
+ if vals.is_empty() {
+ if let Some(ph) = &missing_placeholder {
+ distinct.insert(ph.clone());
+ }
+ continue;
+ }
+ for v in vals {
+ distinct.insert(v);
+ }
+ }
+
+ // Identical output shape to `run_cardinality` including the
+ // internal `__xy_*` fields the cross-index merge consumes.
+ let values: Vec<Value> = distinct
+ .iter()
+ .map(|s| Value::String(s.clone()))
+ .collect();
+ Some(json!({
+ "value": distinct.len(),
+ "__xy_agg__": "cardinality",
+ "__xy_values__": values,
+ }))
+ }
+
fn exec_terms(&self, params: &Value, sub: Option<&Value>) -> Option<Value> {
if !params_only(params, &["field", "size", "order", "shard_size", "min_doc_count"]) {
return None;
@@ -1112,7 +1309,7 @@ impl<'a> FastCtx<'a> {
}
}
}
- for doc in &self.mem_docs {
+ for doc in self.mem_docs.iter() {
let Some(v) = extract_numeric(doc, field) else { continue };
let matches = match (from, to) {
(Some(f), Some(t)) => v >= f && v < t,
@@ -1258,7 +1455,7 @@ impl<'a> FastCtx<'a> {
}
}
}
- for doc in &self.mem_docs {
+ for doc in self.mem_docs.iter() {
let v = doc.get(field);
let Some(ms) = v.and_then(parse_date_ms) else { continue };
let pass_from = from_ms.map(|f| ms >= f).unwrap_or(true);
@@ -1363,7 +1560,7 @@ impl<'a> FastCtx<'a> {
}
}
}
- for doc in &self.mem_docs {
+ for doc in self.mem_docs.iter() {
if doc_matches_filter(doc, filter_query) {
count += doc_count_weight(doc);
for (mi, spec) in plan.metrics.iter().enumerate() {
@@ -1451,7 +1648,7 @@ impl<'a> FastCtx<'a> {
}
}
}
- for doc in &self.mem_docs {
+ for doc in self.mem_docs.iter() {
if doc_matches_filter(doc, filter_query) {
count += doc_count_weight(doc);
for (mi, spec) in plan.metrics.iter().enumerate() {
@@ -1519,7 +1716,7 @@ impl<'a> FastCtx<'a> {
}
}
}
- for doc in &self.mem_docs {
+ for doc in self.mem_docs.iter() {
let matches: Vec<bool> = filters_map
.values()
.map(|q| doc_matches_filter(doc, q))
@@ -1649,7 +1846,7 @@ impl<'a> FastCtx<'a> {
*counts.entry(key).or_insert(0) += c;
}
}
- for doc in &self.mem_docs {
+ for doc in self.mem_docs.iter() {
let weight = doc_count_weight(doc);
// Cross-product across multi-valued fields — brute semantics.
let mut keys: Vec<Vec<String>> = vec![vec![]];
@@ -1738,7 +1935,7 @@ impl<'a> FastCtx<'a> {
// match_all every doc scores equal, so the brute sampler's
// stable score sort preserves exactly this prefix.
let mut sample: Vec<Value> = Vec::with_capacity(shard_size);
- for d in &self.mem_docs {
+ for d in self.mem_docs.iter() {
if sample.len() >= shard_size {
break;
}
@@ -1982,7 +2179,7 @@ impl<'a> FastCtx<'a> {
}
// Memtable docs — brute multi-value date extraction with per-doc
// bucket dedup.
- for doc in &self.mem_docs {
+ for doc in self.mem_docs.iter() {
let raws = extract_date_ms_values(doc, field);
let weight = doc_count_weight(doc);
let mut seen: Vec<i64> = Vec::with_capacity(raws.len());
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 4a4336f0e..f587b83e1 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -364,6 +364,17 @@ pub struct Index {
/// the merge-completion site alongside dv_cache/stored_value_cache).
stored_slices_cache: Arc<dashmap::DashMap<String, Arc<StoredSlices>>>,
stored_slices_cache_bytes: Arc<AtomicU64>,
+ /// Per-segment cache of the DECOMPRESSED stored section bytes for the
+ /// UNSORTED scan path (`scan_stored_section_into`). The bounded
+ /// collector already stops the scan O(from+size) docs in, but every
+ /// query still paid the full zstd/LZ4 decompress of the FIRST segment
+ /// it touched — for a merged multi-million-doc segment that is
+ /// 0.5-1.5 s per query, and under a bulk writer (100 % query-cache
+ /// misses) it WAS the match_all/bool/range read-under-write tail.
+ /// Budgeted like `stored_slices_cache`; entries evicted by id at the
+ /// merge-completion site. Segments are immutable → no invalidation.
+ decoded_stored_cache: Arc<dashmap::DashMap<String, Arc<Vec<u8>>>>,
+ decoded_stored_cache_bytes: Arc<AtomicU64>,
/// Regexp term-dictionary expansion cache — keyed by
/// `(segment_id, field, pattern)`, holds the exact union doc count
/// plus the first `REGEXP_EXPANSION_POS_CAP` matching doc positions
@@ -535,6 +546,8 @@ impl Index {
stored_value_cache: Arc::new(dashmap::DashMap::new()),
stored_slices_cache: Arc::new(dashmap::DashMap::new()),
stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),
+ decoded_stored_cache: Arc::new(dashmap::DashMap::new()),
+ decoded_stored_cache_bytes: Arc::new(AtomicU64::new(0)),
regexp_expand_cache: Arc::new(dashmap::DashMap::new()),
fast_date_cache: Arc::new(dashmap::DashMap::new()),
query_cache: Arc::new(dashmap::DashMap::new()),
@@ -630,6 +643,16 @@ impl Index {
let memtable_doc_count = memtable.doc_count() as u64;
let total_doc_count = segment_doc_count + memtable_doc_count;
+ // Conservative ghost init: at rest (no in-flight flushes) a live
+ // count below the physical count means superseded copies exist
+ // from a pre-restart history the WAL replay above cannot see.
+ // Flag them so the delete-aware slow paths stay on — the exact
+ // per-event tracking (`VersionMap::ghost_events`) only observes
+ // events from this process lifetime onward.
+ if (store.version_map.live_count() as u64) < total_doc_count {
+ store.version_map.force_ghost_event();
+ }
+
// See create_with_settings — doc count is a sanity cap, byte threshold drives flushes.
let flush_doc_threshold = 500_000usize;
let flush_byte_threshold = (config.storage.flush_size_mb as usize).saturating_mul(1024 * 1024);
@@ -690,6 +713,8 @@ impl Index {
stored_value_cache: Arc::new(dashmap::DashMap::new()),
stored_slices_cache: Arc::new(dashmap::DashMap::new()),
stored_slices_cache_bytes: Arc::new(AtomicU64::new(0)),
+ decoded_stored_cache: Arc::new(dashmap::DashMap::new()),
+ decoded_stored_cache_bytes: Arc::new(AtomicU64::new(0)),
regexp_expand_cache: Arc::new(dashmap::DashMap::new()),
fast_date_cache: Arc::new(dashmap::DashMap::new()),
query_cache: Arc::new(dashmap::DashMap::new()),
@@ -1353,15 +1378,21 @@ impl Index {
// the memtable entry can share the allocation (no deep clone at
// insert time — pre-fix `insert` did `Arc::new(source.clone())`
// per doc under the shard write lock).
- let sources: Vec<Arc<Value>> = docs
- .par_iter()
- .map(|(_, bytes)| {
- Arc::new(
- serde_json::from_slice::<Value>(bytes)
- .unwrap_or_else(|_| Value::Object(serde_json::Map::new())),
- )
- })
- .collect();
+ //
+ // Runs on the dedicated ingest pool (`crate::ingest_pool`) so bulk
+ // parse bursts never queue ahead of search/agg par_iters on the
+ // global rayon pool — see the read-under-write collapse notes on
+ // `ingest_pool()`.
+ let sources: Vec<Arc<Value>> = crate::ingest_pool().install(|| {
+ docs.par_iter()
+ .map(|(_, bytes)| {
+ Arc::new(
+ serde_json::from_slice::<Value>(bytes)
+ .unwrap_or_else(|_| Value::Object(serde_json::Map::new())),
+ )
+ })
+ .collect()
+ });
// 2. Dynamic mapping: evolve schema serially, in doc order.
for source in &sources {
@@ -1390,40 +1421,47 @@ impl Index {
let analyzer = mem
.default_analyzer()
.expect("standard analyzer always present");
- let analyzed: Vec<Vec<(String, Vec<xerj_fts::analyzer::Token>)>> = sources
- .par_iter()
- .map(|source| crate::memtable::analyze_doc(source.as_ref(), schema, &analyzer))
- .collect();
+ let analyzed: Vec<Vec<(String, Vec<xerj_fts::analyzer::Token>)>> =
+ crate::ingest_pool().install(|| {
+ sources
+ .par_iter()
+ .map(|source| {
+ crate::memtable::analyze_doc(source.as_ref(), schema, &analyzer)
+ })
+ .collect()
+ });
let n_shards = mem.shard_count().max(1);
let mut buckets: Vec<Vec<usize>> = vec![Vec::new(); n_shards];
for (i, (id, _)) in docs.iter().enumerate() {
buckets[mem.shard_for_dynamic(id)].push(i);
}
- buckets.par_iter().enumerate().for_each(|(shard, idxs)| {
- if idxs.is_empty() {
- return;
- }
- mem.with_shard_mut(shard, |m| {
- for &i in idxs {
- let (id, bytes) = &docs[i];
- // `remove` first so a same-id overwrite within
- // the batch leaves exactly one live entry
- // (mirrors index_document).
- m.remove(id);
- // Same sizing formula as `insert` (`raw * 3 + 64`)
- // with the raw NDJSON byte length standing in for
- // `source.to_string().len()` — keeps the size
- // re-serialisation off the lock too.
- let size = (bytes.len() + id.len()) * 3 + 64;
- m.insert_analyzed(
- seq_nos[i],
- id.clone(),
- Arc::clone(&sources[i]),
- &analyzed[i],
- size,
- );
+ crate::ingest_pool().install(|| {
+ buckets.par_iter().enumerate().for_each(|(shard, idxs)| {
+ if idxs.is_empty() {
+ return;
}
+ mem.with_shard_mut(shard, |m| {
+ for &i in idxs {
+ let (id, bytes) = &docs[i];
+ // `remove` first so a same-id overwrite within
+ // the batch leaves exactly one live entry
+ // (mirrors index_document).
+ m.remove(id);
+ // Same sizing formula as `insert` (`raw * 3 + 64`)
+ // with the raw NDJSON byte length standing in for
+ // `source.to_string().len()` — keeps the size
+ // re-serialisation off the lock too.
+ let size = (bytes.len() + id.len()) * 3 + 64;
+ m.insert_analyzed(
+ seq_nos[i],
+ id.clone(),
+ Arc::clone(&sources[i]),
+ &analyzed[i],
+ size,
+ );
+ }
+ });
});
});
}
@@ -2163,6 +2201,12 @@ impl Index {
let metas_for_task = metas;
tokio::task::spawn_blocking(move || -> Option<MergeOutput> {
+ // Entire merge encode runs on the deprioritised ingest
+ // pool (nice +10): the decompress + byte-copy + FTS/DV
+ // rebuild below saturates cores for seconds, and on the
+ // normal-priority blocking pool it stole scheduler time
+ // from foreground searches (read-under-write tail).
+ crate::ingest_pool().install(move || -> Option<MergeOutput> {
use serde_json::value::RawValue;
// V4 M4.8 — STREAMING BYTE-COPY MERGE.
@@ -2358,10 +2402,17 @@ impl Index {
fts_writer.configure_field(field_name.clone(), cfg.clone());
}
if !fts_input.is_empty() {
- fts_writer.add_documents_parallel(&fts_input);
- if let Err(e) = fts_writer.finish() {
- tracing::warn!("merge: FTS build failed: {e}");
- }
+ // Dedicated ingest pool: a merged-segment FTS build
+ // is the single longest rayon job in the system —
+ // on the global pool it queued every concurrent
+ // search's par_iter behind it (read-under-write
+ // collapse; see `crate::ingest_pool`).
+ crate::ingest_pool().install(|| {
+ fts_writer.add_documents_parallel(&fts_input);
+ if let Err(e) = fts_writer.finish() {
+ tracing::warn!("merge: FTS build failed: {e}");
+ }
+ });
}
}
@@ -2418,6 +2469,7 @@ impl Index {
}
Some((batch_for_task, merged_meta, live_doc_count as usize, ids_pairs))
+ })
})
};
@@ -2537,6 +2589,10 @@ impl Index {
self.stored_slices_cache_bytes
.fetch_sub(slices.retained_bytes(), Ordering::Relaxed);
}
+ if let Some((_, bytes)) = self.decoded_stored_cache.remove(id.as_str()) {
+ self.decoded_stored_cache_bytes
+ .fetch_sub(bytes.len() as u64, Ordering::Relaxed);
+ }
self.fast_date_cache.retain(|(seg, _), _| seg != id.as_str());
}
}
@@ -3681,6 +3737,12 @@ impl Index {
// the working set is far smaller and the cheap O(n)
// truncate happens once.
if let (Some(key), Ok(ref r)) = (cache_key, &result) {
+ // Never cache a timed-out partial result — the dataset
+ // version doesn't change on timeout, so a poisoned entry
+ // would be served until the next write.
+ if r.timed_out {
+ return result;
+ }
if self.query_cache.len() > 1024 {
// Cheap eviction: clear the whole cache rather
// than implement LRU. It rebuilds on the next
@@ -3729,6 +3791,32 @@ impl Index {
)));
}
+ // ── Cooperative deadline ──────────────────────────────────────────
+ // The `tokio::time::timeout` wrapper in `search()` is DEAD CODE on
+ // the multi-thread runtime: the search body runs inside
+ // `block_in_place(block_on(..))`, so the wrapper's first poll only
+ // returns when the whole search is already finished — it can never
+ // fire mid-flight. Live-verified during the read-under-write
+ // benchmark: 2 118 slow queries (p50 19 s, max 39 s) and ZERO
+ // "search timed out" log lines; every piled-up search ran to
+ // completion even after its client disconnected, which is what let
+ // the backlog snowball keep the CPU pegged for minutes after load
+ // stopped. Instead we compute a wall-clock deadline here and have
+ // the O(N) loops below check it cooperatively, returning partial
+ // results with `timed_out: true` (ES semantics).
+ let search_deadline: std::time::Instant = std::time::Instant::now()
+ + std::time::Duration::from_millis(request.timeout_ms.unwrap_or(30_000));
+ let mut deadline_exceeded = false;
+ // Slow-query phase attribution (logged for >1 s queries).
+ let phase_t0 = std::time::Instant::now();
+ let mut phase_marks: Vec<(&'static str, u64)> = Vec::new();
+ let mut dbg_segs = 0u32;
+ let mut dbg_decode_ms = 0u64;
+ let mut dbg_scan_ms = 0u64;
+ let mut dbg_fts_ms = 0u64;
+ let mut dbg_walked = 0u64;
+ let mut dbg_admitted = 0u64;
+
// size=0 is valid and means "return no hits but still run aggs / return total".
let size = request.size;
let from = request.from;
@@ -4059,11 +4147,11 @@ impl Index {
enum MemSnapshot {
Empty,
FtsHits(Vec<(String, f32)>), // (doc_id, score) from inverted index
- AllDocIds(Vec<String>), // MatchAll: IDs only, sources filled later
+ AllDocIds(Vec<String>, u64), // MatchAll: bounded IDs + uncollected remainder count
/// Fast DocValues hit: (doc_id, internal_index) pairs — source is NOT
/// cloned here; it will be fetched only for the final page of hits.
DocValuesHits(Vec<(String, usize)>),
- DocsForScan(Vec<(String, Value)>), // doc_id + source for term-level scan
+ DocsForScan(Vec<(String, Arc<Value>)>), // doc_id + shared source for term-level scan
}
// Search the memtable (unflushed docs only). Flushed docs are searched
@@ -4091,9 +4179,18 @@ impl Index {
// fallback with empty inputs.
MemSnapshot::Empty
} else if matches!(query, QueryNode::MatchAll) {
- // Return all doc IDs from the memtable (size > 0 path).
- let ids: Vec<String> = mem.all_doc_ids();
- MemSnapshot::AllDocIds(ids)
+ // size > 0 match_all. Field-sorted requests must offer
+ // EVERY buffered doc to the top-N heap; unsorted requests
+ // only ever admit the first `materialisation_limit` ids,
+ // so clone just those and carry the remainder as a count.
+ if sort_topk.is_some() {
+ let ids: Vec<String> = mem.all_doc_ids();
+ MemSnapshot::AllDocIds(ids, 0)
+ } else {
+ let (ids, total) = mem.doc_ids_bounded(materialisation_limit);
+ let extra = total.saturating_sub(ids.len() as u64);
+ MemSnapshot::AllDocIds(ids, extra)
+ }
} else if is_doc_scan_query(query) {
// count_only + Term/Range short-circuits BEFORE
// `try_doc_values_query` because the DV-term path walks
@@ -4121,9 +4218,13 @@ impl Index {
// the hit list (size > 0 or sort-on-field).
MemSnapshot::DocValuesHits(hits)
} else {
- // Full source materialisation — only reached for
- // composite queries or when sources are needed.
- let docs: Vec<(String, Value)> = mem.all_docs_with_sources();
+ // Composite queries (e.g. bool) that the memtable DV
+ // path can't resolve: share the buffered sources via
+ // Arc — the old deep-clone of the ENTIRE memtable per
+ // query (plus a second per-doc clone for `_id`
+ // injection below) was the single hottest search-side
+ // cost in the read-under-write thread dumps.
+ let docs: Vec<(String, Arc<Value>)> = mem.all_docs_with_sources_arc();
MemSnapshot::DocsForScan(docs)
}
} else {
@@ -4161,6 +4262,7 @@ impl Index {
}
// Lock dropped here
};
+ phase_marks.push(("mem_snapshot", phase_t0.elapsed().as_millis() as u64));
match mem_snapshot {
MemSnapshot::Empty => {}
@@ -4177,7 +4279,11 @@ impl Index {
});
}
}
- MemSnapshot::AllDocIds(ids) => {
+ MemSnapshot::AllDocIds(ids, uncollected) => {
+ // Bounded id page: the remainder still counts toward
+ // hits.total (match_all's total is later overwritten by
+ // the authoritative live count anyway).
+ total_count += uncollected;
for doc_id in ids {
admit_hit!(count doc_id);
}
@@ -4197,38 +4303,57 @@ impl Index {
}
MemSnapshot::DocsForScan(docs) => {
// Lock already released — scan docs without holding the memtable lock.
+ //
+ // `_id` injection (a full per-doc Object clone) is only
+ // observable when the query tree can actually read `_id`
+ // — a nested Ids query. Detect that ONCE via the query's
+ // serialised form (conservative: false positives merely
+ // keep the legacy per-doc clone) instead of cloning every
+ // buffered doc on every request.
+ let query_may_read_id: bool = serde_json::to_string(query)
+ .map(|s| s.contains("_id") || s.contains("Ids") || s.contains("ids"))
+ .unwrap_or(true);
for (doc_id, source) in docs {
let matched = if let QueryNode::Ids { values } = query {
values.iter().any(|v| v == doc_id.as_str())
+ } else if !query_may_read_id || source.get("_id").is_some() {
+ doc_matches_query(query, &source)
} else {
// Inject `_id` so deeply-nested Ids queries (e.g.
// function_score → ids) can resolve it from source.
- let source_with_id = if source.get("_id").is_some() {
- source.clone()
- } else if let Some(obj) = source.as_object() {
+ let source_with_id = if let Some(obj) = source.as_object() {
let mut o = obj.clone();
o.insert("_id".to_string(), serde_json::Value::String(doc_id.clone()));
serde_json::Value::Object(o)
} else {
- source.clone()
+ (*source).clone()
};
doc_matches_query(query, &source_with_id)
};
if matched {
- // Only clone source if we'll actually materialise it.
+ // Materialise (score + owned source) ONLY when the
+ // hit can actually enter the bounded collector /
+ // top-N heap; everything else is a bare count.
+ // Identical outcome to unconditionally building the
+ // Hit: `admit_hit!` drops post-cap hits anyway.
if count_only {
total_count += 1;
- } else {
+ } else if sort_topk.is_some()
+ || (all_hits.len() < materialisation_limit
+ && !seen_ids.contains(&doc_id))
+ {
let score = score_query_against_doc(query, &source);
admit_hit!(Hit {
id: doc_id,
score,
- source,
+ source: (*source).clone(),
sort: Vec::new(),
explain: None,
highlight: None,
matched_queries: Vec::new(),
});
+ } else {
+ total_count += 1;
}
}
}
@@ -4345,6 +4470,7 @@ impl Index {
// result is stashed in `precomputed_aggs` and consumed at the
// agg-section below.
let mut precomputed_aggs: Option<Value> = None;
+ phase_marks.push(("mem_admit", phase_t0.elapsed().as_millis() as u64));
if is_match_all && size == 0 && request.aggs.is_some() && request.min_score.is_none() {
if let Some(aggs_def) = &request.aggs {
let r_opt = self
@@ -4374,13 +4500,19 @@ impl Index {
// inflate the per-segment/memtable physical tally above the version-map
// live count). MatchAll is unaffected — overwritten with the
// delete-aware live_doc_count() regardless of this flag.
- let deletes_present: bool = {
- let seg_physical: u64 = snap.segments.iter().map(|m| m.doc_count).sum();
- let mem_physical = self.memtable.doc_count() as u64;
- snap.segments.iter().any(|m| m.has_tombstones)
- || self.live_doc_count() < seg_physical + mem_physical
- };
-
+ // EXACT ghost signal (see `VersionMap::ghost_events`): flushed
+ // tombstones OR any overwrite/delete event ever recorded. The old
+ // `live_count() < seg_physical + mem_physical` arithmetic also
+ // tripped on unrelated physical-count drift — live-verified: two
+ // duplicate docs per merged segment turned the gate permanently ON
+ // for a pure append-only corpus, disabling the F1 bounded scan and
+ // forcing every size>0 term/range/bool into a full O(N) stored
+ // scan. That was the dominant per-query cost in the
+ // read-under-write collapse.
+ let deletes_present: bool = snap.segments.iter().any(|m| m.has_tombstones)
+ || self.store.version_map.ghost_events() > 0;
+
+ phase_marks.push(("fast_aggs+gates", phase_t0.elapsed().as_millis() as u64));
if precomputed_aggs.is_some() {
// Skip the entire segment loop — the fast agg path already
// computed everything we need. `total_count` is set above.
@@ -4475,6 +4607,7 @@ impl Index {
};
for meta in &snap.segments {
+ dbg_segs += 1;
// F1: once the exact total is authoritative AND the bounded
// collector is full, remaining segments can neither add a
// materialised hit (page is full) nor change `hits.total`
@@ -4486,6 +4619,13 @@ impl Index {
if count_authoritative && all_hits.len() >= materialisation_limit {
break;
}
+ // Cooperative deadline: stop fanning into further segments
+ // once the request timeout has elapsed — partial results
+ // with `timed_out: true`, exactly like ES.
+ if std::time::Instant::now() >= search_deadline {
+ deadline_exceeded = true;
+ break;
+ }
let seg_id = meta.id.clone();
let fts_dir = segments_dir.clone();
@@ -4496,11 +4636,13 @@ impl Index {
// Try FTS path first if we have an FTS query.
let mut fts_handled = false;
+ let t_fts = std::time::Instant::now();
let reader_opt = if needs_fts {
FtsIndexReader::open(&fts_dir, &seg_id, &field_refs).ok()
} else {
None
};
+ dbg_fts_ms += t_fts.elapsed().as_millis() as u64;
if let Some(reader) = reader_opt {
let reader = Arc::new(reader);
let searcher =
@@ -4785,6 +4927,38 @@ impl Index {
&mut total_count,
);
}
+ } else if let Some(cached_bytes) = (!sorted_candidates_path)
+ .then(|| {
+ self.decoded_stored_cache
+ .get(seg_id.as_str())
+ .map(|e| Arc::clone(e.value()))
+ })
+ .flatten()
+ {
+ // Decoded-bytes cache hit: skip open + decompress
+ // entirely. With the in-section early stop, the
+ // whole per-segment cost is now O(from+size) doc
+ // parses instead of O(section) decompress+parse.
+ let t_scan = std::time::Instant::now();
+ self.scan_stored_section_into(
+ &cached_bytes,
+ query,
+ is_match_all,
+ count_only,
+ materialisation_limit,
+ count_authoritative,
+ &mut total_count,
+ &mut all_hits,
+ &mut seen_ids,
+ pre_filter.as_ref(),
+ sort_topk.as_mut(),
+ None,
+ Some(search_deadline),
+ &mut deadline_exceeded,
+ &mut dbg_walked,
+ &mut dbg_admitted,
+ );
+ dbg_scan_ms += t_scan.elapsed().as_millis() as u64;
} else {
// Merge-race hardening (2026-07): a snapshot
// segment that fails to open is a query error, not
@@ -4793,11 +4967,14 @@ impl Index {
// With the read lease held by `snap`, retired
// segment files persist, so this only fires on
// genuine corruption.
+ let t_dec = std::time::Instant::now();
let seg_reader = self.store.open_segment_arc(&seg_id)?;
if let Ok(Some(stored_bytes_raw)) = seg_reader.section(SectionType::Stored) {
if let Ok(stored_bytes) =
xerj_storage::stored_codec::decode_stored(stored_bytes_raw)
{
+ dbg_decode_ms += t_dec.elapsed().as_millis() as u64;
+ let t_scan = std::time::Instant::now();
// Record per-doc offsets on the sorted path so
// the decompressed section can be cached for
// the next query (subject to the budget).
@@ -4818,7 +4995,12 @@ impl Index {
pre_filter.as_ref(),
sort_topk.as_mut(),
if want_cache { Some(&mut offsets) } else { None },
+ Some(search_deadline),
+ &mut deadline_exceeded,
+ &mut dbg_walked,
+ &mut dbg_admitted,
);
+ dbg_scan_ms += t_scan.elapsed().as_millis() as u64;
// Cache only a COMPLETE offsets map (a
// malformed section bails early) and only
// within the retained-bytes budget.
@@ -4841,6 +5023,26 @@ impl Index {
self.stored_slices_cache_bytes
.fetch_add(sz, Ordering::Relaxed);
}
+ } else if !want_cache {
+ // Unsorted path: retain the decompressed
+ // section so the NEXT query on this
+ // segment skips the decompress (the
+ // read-under-write match_all/bool/range
+ // tail fix — see `decoded_stored_cache`).
+ let sz = stored_bytes.len() as u64;
+ if self
+ .decoded_stored_cache_bytes
+ .load(Ordering::Relaxed)
+ .saturating_add(sz)
+ <= DECODED_STORED_CACHE_BUDGET
+ && self
+ .decoded_stored_cache
+ .insert(seg_id.clone(), Arc::new(stored_bytes))
+ .is_none()
+ {
+ self.decoded_stored_cache_bytes
+ .fetch_add(sz, Ordering::Relaxed);
+ }
}
}
}
@@ -4858,6 +5060,7 @@ impl Index {
// faithfully reflects full query semantics (flattened / subobjects:false
// / passthrough field resolution the doc-values shortcut can't model),
// so we keep it and never overwrite.
+ phase_marks.push(("segment_loop", phase_t0.elapsed().as_millis() as u64));
let scan_hit_cap: bool = all_hits.len() >= materialisation_limit;
// De-duplication is already enforced by `seen_ids` inside the
@@ -5308,6 +5511,15 @@ impl Index {
.collect();
let snap_bg = self.store.snapshot();
for seg in &snap_bg.segments {
+ // Cooperative deadline: the full-corpus assembly is the
+ // most expensive path in the engine (decompress + parse
+ // EVERY stored doc). Stop pulling further segments once
+ // past the request timeout — aggs become partial and the
+ // response carries `timed_out: true`.
+ if std::time::Instant::now() >= search_deadline {
+ deadline_exceeded = true;
+ break;
+ }
// Merge-race hardening (2026-07): open failures are
// errors, not skips — a skipped segment silently drops
// its docs from every aggregation bucket. The read
@@ -5388,6 +5600,7 @@ impl Index {
vec![]
};
+ phase_marks.push(("hydrate+corpus", phase_t0.elapsed().as_millis() as u64));
// --- Run aggregations over the full matched set (before pagination) ---
let agg_start = std::time::Instant::now();
let agg_result = if let Some(aggs_def) = &request.aggs {
@@ -5576,6 +5789,43 @@ impl Index {
None
};
+ let inner_ms = phase_t0.elapsed().as_millis() as u64;
+ if inner_ms >= 1_000 {
+ phase_marks.push(("aggs+page", inner_ms));
+ // Cumulative marks → per-phase deltas for the log line.
+ let mut prev = 0u64;
+ let breakdown: Vec<String> = phase_marks
+ .iter()
+ .map(|(name, at)| {
+ let d = at.saturating_sub(prev);
+ prev = *at;
+ format!("{name}={d}ms")
+ })
+ .collect();
+ warn!(
+ total_ms = inner_ms,
+ index = self.name.as_str(),
+ phases = %breakdown.join(" "),
+ gates = %format!(
+ "needs_fts={} shortcut={:?} deletes={} ghosts={} total={} lim={} segs={} dec={}ms scan={}ms ftsopen={}ms walked={} admitted={} allhits_cap={}",
+ query_needs_fts,
+ shortcut_count,
+ deletes_present,
+ self.store.version_map.ghost_events(),
+ total,
+ materialisation_limit,
+ dbg_segs,
+ dbg_decode_ms,
+ dbg_scan_ms,
+ dbg_fts_ms,
+ dbg_walked,
+ dbg_admitted,
+ scan_hit_cap
+ ),
+ "slow search_inner phase breakdown"
+ );
+ }
+
Ok(SearchResult {
hits: page,
total: TotalHits {
@@ -5584,7 +5834,10 @@ impl Index {
},
took_ms: 0,
aggs: agg_result,
- timed_out: false,
+ // Set when any cooperative deadline check fired: the hits /
+ // total / aggs above are partial, exactly like an ES shard
+ // timeout. `search()` refuses to cache timed-out results.
+ timed_out: deadline_exceeded,
profile,
max_score: population_max_score,
})
@@ -7284,6 +7537,16 @@ impl Index {
// verify `offsets_out.len() == segment doc_count` before caching —
// a malformed section bails early and leaves the vec incomplete.
mut offsets_out: Option<&mut Vec<(u32, u32)>>,
+ // Cooperative timeout: when `Some`, the walk polls the wall clock
+ // every 4 096 docs and aborts (setting `*timed_out`) once past the
+ // deadline. One giant merged segment is otherwise a multi-second
+ // uninterruptible unit — the between-segment deadline check in
+ // `search_inner` can't bound it.
+ deadline: Option<std::time::Instant>,
+ timed_out: &mut bool,
+ // Diagnostics: docs brace-walked and hits admitted by THIS call.
+ dbg_walked: &mut u64,
+ dbg_admitted: &mut u64,
) {
// Fast path: scan one doc at a time with a hand-rolled array splitter.
// The stored section is always shaped as `[{...}, {...}, ...]` (a
@@ -7307,6 +7570,32 @@ impl Index {
let mut doc_pos: u32 = 0;
loop {
+ // F1b — IN-SEGMENT early stop. The between-segment F1 break in
+ // `search_inner` skips segments once the page is full and the
+ // total is authoritative, but a single giant merged segment
+ // still paid a FULL walk here: brace-scan + simd_json parse of
+ // every stored doc (~11 s for an uncached size=10 `match_all`
+ // at 3.5 M docs — completely masked at steady state by the
+ // query cache, catastrophic under a bulk writer where every
+ // request is a cache miss). Once the bounded collector is
+ // full, nothing later in this section can change the page
+ // (admit order is positional) and `total_count` is overwritten
+ // by the shortcut/live count anyway — so stop scanning.
+ //
+ // Excluded, for the same reasons as the outer break:
+ // - `count_only`: the scan tally IS the total when no
+ // shortcut resolved (count_authoritative already implies a
+ // shortcut, but keep the guard belt-and-braces);
+ // - `sort_topk`: top-N needs every matching doc's sort key;
+ // - `offsets_out`: the offsets cache must cover the section.
+ if count_authoritative
+ && !count_only
+ && sort_topk.is_none()
+ && offsets_out.is_none()
+ && all_hits.len() >= materialisation_limit
+ {
+ return;
+ }
// Skip whitespace/commas between docs.
while i < n && (bytes[i].is_ascii_whitespace() || bytes[i] == b',') {
i += 1;
@@ -7361,6 +7650,18 @@ impl Index {
let cur_pos = doc_pos;
doc_pos += 1;
+ *dbg_walked += 1;
+
+ // Cooperative timeout poll — every 4 096 docs (~1 µs of clock
+ // reads per million docs; the parse below costs ~3 µs/doc).
+ if doc_pos & 0xFFF == 0 {
+ if let Some(dl) = deadline {
+ if std::time::Instant::now() >= dl {
+ *timed_out = true;
+ return;
+ }
+ }
+ }
// Pre-filter: if the caller supplied a matching set and this
// position isn't in it, skip the parse entirely. We already
@@ -7415,6 +7716,7 @@ impl Index {
}
*total_count += 1;
+ *dbg_admitted += 1;
if count_only {
continue;
@@ -7657,6 +7959,10 @@ async fn do_flush_shard(
// 50-100 ms segment write that follows, and 100K-doc/s ingest
// bench numbers are unaffected.
let build_fts = move |meta: &xerj_storage::segment::SegmentMeta| -> xerj_storage::Result<()> {
+ // Whole side-car build runs on the dedicated ingest pool so a
+ // flush burst can't queue search/agg par_iters behind it on the
+ // global rayon pool (read-under-write; see `crate::ingest_pool`).
+ crate::ingest_pool().install(|| {
// ── Doc-values side-car (always built) ────────────────────
{
let columns = build_doc_value_columns(
@@ -7687,6 +7993,7 @@ async fn do_flush_shard(
tracing::warn!("flush: FTS build failed: {e}");
}
}
+ });
let _ = &segments_dir;
let _ = &drained_fts;
@@ -7694,7 +8001,11 @@ async fn do_flush_shard(
};
// Hand the pre-drained storage entries to the finaliser.
- let meta = match store.finalize_flush_with_publisher(storage_drained, build_fts)? {
+ // Segment encode + side-car build on the deprioritised ingest pool
+ // (build_fts's inner install() is a same-pool no-op).
+ let meta = match crate::ingest_pool()
+ .install(|| store.finalize_flush_with_publisher(storage_drained, build_fts))?
+ {
Some(m) => m,
None => {
tracing::warn!("storage finalize returned None — unexpected");
@@ -12105,6 +12416,11 @@ impl StoredSlices {
/// path); merge eviction returns the dropped segment's bytes to the budget.
const STORED_SLICES_CACHE_BUDGET: u64 = 512 * 1024 * 1024;
+/// Budget for `decoded_stored_cache` (raw decompressed stored sections,
+/// ~1× the corpus JSON size). Overflow → inserts are refused and the
+/// query falls back to the per-query decompress (pre-cache behaviour).
+const DECODED_STORED_CACHE_BUDGET: u64 = 4 * 1024 * 1024 * 1024;
+
/// Normalize one `search_after` cursor value so it compares against the
/// normalized `hit.sort` values `compute_sort_values` produces (date-shaped
/// strings → epoch-ms / epoch-ns numbers). `fmt_hint` is the SortField's
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index bb001cc70..14d49781f 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -26,6 +26,50 @@ pub use engine::{Engine, HealthStatus, IndexInfo};
pub use index::{Index, IndexResponse, IndexStats, FieldEncodingInfo, LogFormat, EnrichTable, detect_log_format, resolve_field_alias, resolve_date_math};
pub use memtable::FtsMemtable;
+// ── Ingest/flush/merge rayon pool ────────────────────────────────────────────
+
+/// Dedicated rayon pool for ingest-side CPU work: bulk-body parse, P2.1
+/// per-batch doc parse/analyze/insert, flush FTS+DV side-car builds, and
+/// merge FTS rebuilds.
+///
+/// Pre-fix all of that ran on the GLOBAL rayon pool — the same pool the
+/// search path uses for segment fan-out (`search_segments`) and the
+/// fast-agg dv-warm par_iter. A single long flush/merge job therefore
+/// queued every search's rayon work behind it: with a background bulk
+/// writer running, foreground `match_all` p99 was measured at 15 s+ while
+/// steady-state reads were ~2 ms (BEAT_ES_MASTER_PLAN Phase 2, read-under-
+/// write). Ingest work now runs here; the global pool stays free for
+/// search/agg fan-out, so neither side can starve the other's queue (the
+/// OS scheduler time-slices the two pools fairly).
+///
+/// Sized to all available cores: ingest throughput is unchanged when no
+/// searches run, and under mixed load the kernel — not the rayon job
+/// queue — arbitrates.
+pub(crate) fn ingest_pool() -> &'static rayon::ThreadPool {
+ static POOL: std::sync::OnceLock<rayon::ThreadPool> = std::sync::OnceLock::new();
+ POOL.get_or_init(|| {
+ let n = std::thread::available_parallelism()
+ .map(|n| n.get())
+ .unwrap_or(8);
+ rayon::ThreadPoolBuilder::new()
+ .num_threads(n)
+ .thread_name(|i| format!("xerj-ingest-{i}"))
+ // Deprioritise ingest CPU vs foreground reads. Flush/merge
+ // side-car builds saturate every core for seconds at a time;
+ // at equal priority a ~20 ms search waits behind them for
+ // hundreds of ms (measured p50 0.2-1.9 s under a bulk
+ // writer). nice(10) lets CFS schedule search threads ahead
+ // of background encode work; with no reads running the pool
+ // still gets every core, so writer-only throughput is
+ // unchanged.
+ .start_handler(|_| unsafe {
+ let _ = libc::nice(10);
+ })
+ .build()
+ .expect("failed to build ingest rayon pool")
+ })
+}
+
// ── Analyzer helpers ──────────────────────────────────────────────────────────
use std::sync::Arc;
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index d7929a7df..f3b219571 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -464,6 +464,25 @@ impl ShardedFtsMemtable {
out
}
+ /// Bounded variant: clone at most `limit` doc ids and return the TOTAL
+ /// buffered doc count alongside. The unsorted match_all page only
+ /// needs `from+size+ε` ids, but `all_doc_ids` cloned every buffered id
+ /// on every request — ~1 s per query at a 300 k-doc memtable under
+ /// sustained bulk load (the `mem_admit` phase of the read-under-write
+ /// breakdown).
+ pub fn doc_ids_bounded(&self, limit: usize) -> (Vec<String>, u64) {
+ let mut out: Vec<String> = Vec::with_capacity(limit.min(4096));
+ let mut total: u64 = 0;
+ for s in &self.shards {
+ let g = s.read();
+ total += g.doc_count() as u64;
+ if out.len() < limit {
+ out.extend(g.doc_ids_up_to(limit - out.len()));
+ }
+ }
+ (out, total)
+ }
+
/// Return every (doc_id, source) pair. Aggregates across all
/// shards; insertion order is preserved within a shard but not
/// across shards. Callers that need global order should sort
@@ -476,6 +495,15 @@ impl ShardedFtsMemtable {
out
}
+ /// Arc-sharing twin of `all_docs_with_sources` — no deep clones.
+ pub fn all_docs_with_sources_arc(&self) -> Vec<(String, Arc<Value>)> {
+ let mut out = Vec::new();
+ for s in &self.shards {
+ out.extend(s.read().all_docs_with_sources_arc());
+ }
+ out
+ }
+
/// Drain every shard, merge-sort by seq_no, and return the
/// combined (doc_id, text_fields, source) stream. This is the
/// single entry point used by the flush path — the sort
@@ -1631,6 +1659,12 @@ impl FtsMemtable {
self.docs.iter().map(|e| e.doc_id.clone()).collect()
}
+ /// First `n` doc ids (insertion order) — see
+ /// `ShardedFtsMemtable::doc_ids_bounded`.
+ pub fn doc_ids_up_to(&self, n: usize) -> Vec<String> {
+ self.docs.iter().take(n).map(|e| e.doc_id.clone()).collect()
+ }
+
/// Resolve a MemEntry's source Value — if `source` is Null but
/// `source_bytes` is non-empty, lazily parse the bytes. This is
/// the M5.11 deferred-parse path used by `insert_raw_bytes_with_seq`.
@@ -1642,19 +1676,56 @@ impl FtsMemtable {
}
}
- /// Return the full original source JSON for a document by ID.
- pub fn get_doc_source_as_value(&self, doc_id: &str) -> Option<Value> {
+ /// Arc-sharing twin of `resolve_source`: no deep clone for the common
+ /// HTTP-bulk path where the parsed source is already Arc-stored.
+ fn resolve_source_arc(entry: &MemEntry) -> Arc<Value> {
+ if entry.source.is_null() && !entry.source_bytes.is_empty() {
+ Arc::new(serde_json::from_slice(&entry.source_bytes).unwrap_or(Value::Null))
+ } else {
+ Arc::clone(&entry.source)
+ }
+ }
+
+ /// All (doc_id, source) pairs WITHOUT deep-cloning the source trees.
+ /// The fast-agg path was deep-cloning the entire memtable per agg
+ /// request via `all_docs_with_sources` (~100-300 ms/query at 1e5
+ /// buffered docs under a bulk writer).
+ pub fn all_docs_with_sources_arc(&self) -> Vec<(String, Arc<Value>)> {
self.docs
.iter()
- .find(|e| e.doc_id == doc_id)
- .map(Self::resolve_source)
+ .map(|e| (e.doc_id.clone(), Self::resolve_source_arc(e)))
+ .collect()
+ }
+
+ /// Return the full original source JSON for a document by ID.
+ /// O(1) doc lookup via `doc_id_index` (maintained by every insert /
+ /// remove path), with a linear-scan fallback in case an entry is ever
+ /// missing from the index.
+ ///
+ /// Pre-fix this was an unconditional `docs.iter().find(..)` — a full
+ /// linear walk of the shard per call. `fill_memtable_sources`
+ /// hydrates up to `materialisation_limit` (256) hits per search, so
+ /// with a bulk writer keeping the memtable at 10⁵ docs every search
+ /// paid 256 × O(shard) string compares; `get_doc_source_as_value` was
+ /// the single hottest frame in the read-under-write profile.
+ fn entry_for(&self, doc_id: &str) -> Option<&MemEntry> {
+ if let Some(&i) = self.doc_id_index.get(doc_id) {
+ if let Some(e) = self.docs.get(i) {
+ if e.doc_id == doc_id {
+ return Some(e);
+ }
+ }
+ }
+ self.docs.iter().find(|e| e.doc_id == doc_id)
+ }
+
+ pub fn get_doc_source_as_value(&self, doc_id: &str) -> Option<Value> {
+ self.entry_for(doc_id).map(Self::resolve_source)
}
/// Return the Arc-wrapped source for a document by ID.
pub fn get_doc_source_arc(&self, doc_id: &str) -> Option<Arc<Value>> {
- self.docs
- .iter()
- .find(|e| e.doc_id == doc_id)
+ self.entry_for(doc_id)
.map(|e| {
if e.source.is_null() && !e.source_bytes.is_empty() {
Arc::new(serde_json::from_slice(&e.source_bytes).unwrap_or(Value::Null))
@@ -1681,9 +1752,10 @@ impl FtsMemtable {
.collect()
}
- /// Check if a document exists in the memtable.
+ /// Check if a document exists in the memtable. O(1) via
+ /// `doc_id_index`, with linear fallback (see `entry_for`).
pub fn contains(&self, doc_id: &str) -> bool {
- self.docs.iter().any(|e| e.doc_id == doc_id)
+ self.entry_for(doc_id).is_some()
}
/// Number of documents in the memtable.
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index b5f4a3cfd..4f7932e84 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -699,12 +699,14 @@ impl FtsIndexWriter {
for term in &sorted_terms {
if let Some((offset, _skip)) = field_data.postings.encode_term(term, &mut post_data) {
- // Calculate doc_freq and ttf from the writer's internal stats
+ // Calculate doc_freq and ttf from the writer's internal stats.
+ // Direct map lookup — the old `term_stats().find(..)` linear
+ // scan made this loop O(terms² × postings): a merged segment
+ // with ~1 distinct term/doc (float/date fields) took HOURS of
+ // CPU at 3M docs and pinned rayon workers for the duration.
let (doc_freq, ttf) = field_data
.postings
- .term_stats()
- .find(|(t, _, _)| *t == term.as_str())
- .map(|(_, df, ttf)| (df, ttf))
+ .stats_for(term)
.unwrap_or((0, 0));
let end_offset = post_data.len() as u64;
diff --git a/engine/crates/xerj-fts/src/postings.rs b/engine/crates/xerj-fts/src/postings.rs
index 620067d1e..a8df4b8bb 100644
--- a/engine/crates/xerj-fts/src/postings.rs
+++ b/engine/crates/xerj-fts/src/postings.rs
@@ -176,6 +176,24 @@ impl PostingsWriter {
});
}
+ /// Direct `(doc_frequency, total_term_frequency)` lookup for ONE term.
+ ///
+ /// O(log T + postings(term)) — use this in per-term loops. The segment
+ /// writer (`write_field_static`) previously resolved stats via
+ /// `term_stats().find(term)`, which walks the term map from the start
+ /// *and sums every earlier term's postings* on each call — O(T² × P)
+ /// across the write loop. Invisible on 625-doc flush segments, but a
+ /// merged segment over a float field (≈1 distinct term per doc) spun
+ /// 100 % CPU for HOURS at 3 M docs, pinning rayon workers and starving
+ /// every search + flush behind it (the read-under-write collapse).
+ pub fn stats_for(&self, term: &str) -> Option<(u32, u64)> {
+ self.postings.get(term).map(|postings| {
+ let doc_freq = postings.len() as u32;
+ let ttf: u64 = postings.iter().map(|p| p.term_freq as u64).sum();
+ (doc_freq, ttf)
+ })
+ }
+
/// Returns an iterator over `(term, doc_frequency, total_term_frequency)`
/// without consuming self (for building the FST term dictionary).
pub fn term_stats(&self) -> impl Iterator<Item = (&str, u32, u64)> {
diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rs
index a0447aba2..10185384a 100644
--- a/engine/crates/xerj-storage/src/version_map.rs
+++ b/engine/crates/xerj-storage/src/version_map.rs
@@ -84,12 +84,31 @@ pub struct VersionMap {
/// transiently-interleaved reader can never underflow the type;
/// `live_count()` clamps at 0.
live: AtomicI64,
+ /// Monotonic count of "ghost-producing" events: overwrites of an
+ /// existing doc id and deletes. Each such event leaves a physically
+ /// present but superseded/tombstoned copy in the memtable or a
+ /// segment until a merge purges it.
+ ///
+ /// The search fast paths (`deletes_present` gates in `search_inner` /
+ /// `fast_aggs`) used to INFER this state from
+ /// `live_count() < Σ segment doc_count + memtable doc_count` — but
+ /// that arithmetic also trips on any unrelated physical-count drift
+ /// (live-verified: two duplicate docs per merged segment flipped the
+ /// gate permanently on a pure append-only workload, forcing every
+ /// size>0 term/range/bool query into a full O(N) stored scan — the
+ /// core of the read-under-write collapse). This counter is the
+ /// exact signal: zero ⇔ no update/delete has ever produced a ghost.
+ ///
+ /// Monotonic by design (never decremented on merge): once an index
+ /// has seen updates, the delete-aware slow paths stay on. That is
+ /// conservative but always correct.
+ ghost_events: std::sync::atomic::AtomicU64,
}
impl VersionMap {
/// Create an empty version map.
pub fn new() -> Self {
- Self { inner: DashMap::new(), live: AtomicI64::new(0) }
+ Self { inner: DashMap::new(), live: AtomicI64::new(0), ghost_events: std::sync::atomic::AtomicU64::new(0) }
}
/// Apply a live-count delta for an entry transition.
@@ -128,6 +147,16 @@ impl VersionMap {
let old = self
.inner
.insert(doc_id, VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
+ // Overwrite or delete — a superseded/tombstoned physical copy now
+ // exists somewhere until merged away. A same-seq_no re-set is NOT
+ // an overwrite: the flush path repoints every drained doc's entry
+ // from `__memtable__` to its segment id with the seq_no unchanged
+ // (live-verified: counting those flipped the gate ON for every
+ // flushed doc and re-disabled all fast paths under pure appends).
+ let is_overwrite = old.as_ref().map_or(false, |e| e.seq_no != seq_no);
+ if is_overwrite || deleted {
+ self.ghost_events.fetch_add(1, Ordering::Relaxed);
+ }
self.live_delta(old.map_or(false, |e| !e.deleted), !deleted);
}
@@ -215,6 +244,7 @@ impl VersionMap {
}
let was_live = !occ.get().deleted;
occ.insert(VersionEntry { seq_no: new_seq_no, segment_id: new_segment_id, deleted });
+ self.ghost_events.fetch_add(1, Ordering::Relaxed);
self.live_delta(was_live, !deleted);
Ok(new_seq_no)
}
@@ -228,6 +258,9 @@ impl VersionMap {
});
}
vac.insert(VersionEntry { seq_no: new_seq_no, segment_id: new_segment_id, deleted });
+ if deleted {
+ self.ghost_events.fetch_add(1, Ordering::Relaxed);
+ }
self.live_delta(false, !deleted);
Ok(new_seq_no)
}
@@ -254,6 +287,7 @@ impl VersionMap {
segment_id: segment_id.into(),
deleted: true,
};
+ self.ghost_events.fetch_add(1, Ordering::Relaxed);
self.live_delta(true, false);
Ok(true)
}
@@ -324,6 +358,22 @@ impl VersionMap {
pub fn live_count(&self) -> usize {
self.live.load(Ordering::Relaxed).max(0) as usize
}
+
+ /// Number of overwrite/delete events ever seen (see `ghost_events`
+ /// field docs). Zero ⇔ append-only history ⇔ physical postings and
+ /// stored copies are 1:1 with live docs (barring storage-level bugs,
+ /// which corrupt the brute counting paths identically).
+ pub fn ghost_events(&self) -> u64 {
+ self.ghost_events.load(Ordering::Relaxed)
+ }
+
+ /// Conservatively mark the index as containing ghosts. Called once at
+ /// open time when the at-rest arithmetic (`live < physical`) indicates
+ /// superseded copies from a pre-restart history the WAL replay cannot
+ /// see.
+ pub fn force_ghost_event(&self) {
+ self.ghost_events.fetch_add(1, Ordering::Relaxed);
+ }
}
impl Default for VersionMap {