Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(ingest): isolate merge encode on a dedicated small rayon pool — kills the 1M-scale merge-induced ingest stall

xerj-org9bb03126c618
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 73540072f..1bcb11247 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -2258,12 +2258,16 @@ impl Index {
let metas_for_task = metas;
tokio::task::spawn_blocking(move || -> Option<MergeOutput> {
- // Entire merge encode runs on the deprioritised ingest
- // pool (nice +10): the decompress + byte-copy + FTS/DV
- // rebuild below saturates cores for seconds, and on the
- // normal-priority blocking pool it stole scheduler time
- // from foreground searches (read-under-write tail).
- crate::ingest_pool().install(move || -> Option<MergeOutput> {
+ // Entire merge encode runs on the dedicated SMALL merge
+ // pool (nice +15, ncores/8 threads): the decompress +
+ // byte-copy + FTS/DV rebuild below saturates its workers
+ // for seconds. On the normal-priority blocking pool it
+ // stole scheduler time from foreground searches (read-
+ // under-write tail); on the shared ingest pool it queued
+ // every bulk request's parse/analyze/insert behind it,
+ // fully stalling 1 M×c8 ingest for ~13 s per merge pass
+ // (see `crate::merge_pool`).
+ crate::merge_pool().install(move || -> Option<MergeOutput> {
use serde_json::value::RawValue;
// V4 M4.8 — STREAMING BYTE-COPY MERGE.
@@ -2459,12 +2463,15 @@ impl Index {
fts_writer.configure_field(field_name.clone(), cfg.clone());
}
if !fts_input.is_empty() {
- // Dedicated ingest pool: a merged-segment FTS build
+ // Dedicated merge pool: a merged-segment FTS build
// is the single longest rayon job in the system —
// on the global pool it queued every concurrent
// search's par_iter behind it (read-under-write
- // collapse; see `crate::ingest_pool`).
- crate::ingest_pool().install(|| {
+ // collapse), and on the ingest pool it stalled
+ // every bulk request (see `crate::merge_pool`).
+ // Same-pool install here is a no-op re-entry —
+ // the whole batch already runs inside merge_pool.
+ crate::merge_pool().install(|| {
fts_writer.add_documents_parallel(&fts_input);
if let Err(e) = fts_writer.finish() {
tracing::warn!("merge: FTS build failed: {e}");
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 14d49781f..61bccb04d 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -70,6 +70,43 @@ pub(crate) fn ingest_pool() -> &'static rayon::ThreadPool {
})
}
+/// Dedicated rayon pool for **merge** encode work (byte-copy re-encode +
+/// FTS side-car rebuild of merged segments).
+///
+/// Pre-fix merges ran on `ingest_pool()` — the same pool every `_bulk`
+/// request's parse/analyze/insert par_iters use. A single 16-segment
+/// (~500 k-doc) merge saturates all N workers for many seconds, so every
+/// concurrent bulk request's `install()` queued behind it: measured on the
+/// 1 M×c8 ingest benchmark as a 4.7 s → 17.5 s FULL STALL of ingest
+/// (window throughput 370 k docs/s → 7.8 k docs/s) the moment the first
+/// background merge tick fired. 100 k-doc runs finish before the first
+/// 5 s merge tick, which is why the regression only appeared at 1 M scale.
+///
+/// Merges are pure background work with no client waiting on them — ES
+/// likewise runs merges on a small dedicated pool and throttles them under
+/// indexing pressure. Sizing: `max(2, ncores/8)` keeps a merge from ever
+/// occupying more than a sliver of the machine, while `nice(15)` (one step
+/// below ingest's `nice(10)`) lets both foreground reads AND ingest encode
+/// pre-empt merge threads when cores are scarce. Merges take a few times
+/// longer to converge — acceptable: the 5 s merge loop just picks up where
+/// it left off, and `merge_in_progress` already serialises passes.
+pub(crate) fn merge_pool() -> &'static rayon::ThreadPool {
+ static POOL: std::sync::OnceLock<rayon::ThreadPool> = std::sync::OnceLock::new();
+ POOL.get_or_init(|| {
+ let n = std::thread::available_parallelism()
+ .map(|n| (n.get() / 8).max(2))
+ .unwrap_or(2);
+ rayon::ThreadPoolBuilder::new()
+ .num_threads(n)
+ .thread_name(|i| format!("xerj-merge-{i}"))
+ .start_handler(|_| unsafe {
+ let _ = libc::nice(15);
+ })
+ .build()
+ .expect("failed to build merge rayon pool")
+ })
+}
+
// ── Analyzer helpers ──────────────────────────────────────────────────────────
use std::sync::Arc;