Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(flush): stagger the 16-wide flush storm off the readers/ingest

xerj-org11ded05a2d7a
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 25854974e..d602583f7 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -9054,17 +9054,36 @@ async fn do_flush_shard(
};
// Hand the pre-drained storage entries to the finaliser.
+ //
+ // The segment encode + side-car build (CPU-heavy, ~600 ms) runs on the
+ // deprioritised background pool (build_fts's inner install() is a
+ // same-pool no-op). Two flush-storm mitigations wrap it:
+ // 1. `flush_finalize_gate()` bounds how many shard finalizes run at
+ // once, staggering the 16-wide storm's core contention off the
+ // readers (see the gate's doc comment for the measured win).
+ // 2. `spawn_blocking` runs the (blocking) finalize on a blocking
+ // thread instead of inline, so it never parks a tokio async worker
+ // — a 16-wide storm used to park up to 16 workers, starving the
+ // concurrent bulk clients' HTTP handling and the reader's search
+ // dispatch.
let t_finalize = std::time::Instant::now();
- // Segment encode + side-car build on the deprioritised ingest pool
- // (build_fts's inner install() is a same-pool no-op).
- let meta = match crate::background_pool()
- .install(|| store.finalize_flush_with_publisher(storage_drained, build_fts))?
- {
- Some(m) => m,
- None => {
+ let _fin_permit = crate::flush_finalize_gate().acquire().await.ok();
+ let finalize_join = tokio::task::spawn_blocking(move || {
+ crate::background_pool()
+ .install(|| store.finalize_flush_with_publisher(storage_drained, build_fts))
+ })
+ .await;
+ let meta = match finalize_join {
+ Ok(Ok(Some(m))) => m,
+ Ok(Ok(None)) => {
tracing::warn!("storage finalize returned None — unexpected");
return Ok(());
}
+ Ok(Err(e)) => return Err(e.into()),
+ Err(join_e) => {
+ tracing::warn!("finalize spawn_blocking join failed: {join_e}");
+ return Ok(());
+ }
};
if prof {
eprintln!(
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 8886e6bbd..46426dad2 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -138,6 +138,45 @@ pub(crate) fn merge_pool() -> &'static rayon::ThreadPool {
})
}
+/// Global bound on how many shard flushes may run their (CPU-heavy,
+/// ~600 ms) segment finalize concurrently.
+///
+/// Under sustained ingest all 16 memtable shards cross their per-shard
+/// flush threshold at nearly the same instant (round-robin doc routing),
+/// so `maybe_spawn_flush` dispatches a 16-wide flush STORM. Pre-fix each
+/// finalize ran inline via `background_pool().install(...)`, which BLOCKS
+/// the calling tokio worker for the whole finalize; a 16-wide storm parked
+/// up to 16 async workers at once, starving the concurrent `_bulk`
+/// clients' HTTP handling and the 300/s reader's search dispatch. It also
+/// oversubscribed the 32-thread background pool 16-deep (512 queued rayon
+/// tasks), inflating per-finalize wall time into 700 ms+ outliers and the
+/// memtable-back-pressure that capped ingest.
+///
+/// The finalize now runs on `spawn_blocking` (never parks an async worker)
+/// under this semaphore, so at most N finalizes are in flight. Measured
+/// on 1M×c8 (32 cores): unbounded+spawn_blocking gained little (337k),
+/// but N=4 staggered the storm into a smoother band — ingest 338k→370k
+/// (+9%, peak 377k) with all five mixed read p99s neutral-to-better.
+/// Total flush CPU is unchanged; the win is scheduling, not less work.
+///
+/// N defaults to `ncores/8` (=4 on the 32-core bench, matching the
+/// measured optimum); `XERJ_FIN_CONC` overrides for tuning.
+pub(crate) fn flush_finalize_gate() -> &'static tokio::sync::Semaphore {
+ static GATE: std::sync::OnceLock<tokio::sync::Semaphore> = std::sync::OnceLock::new();
+ GATE.get_or_init(|| {
+ let n = std::env::var("XERJ_FIN_CONC")
+ .ok()
+ .and_then(|v| v.parse::<usize>().ok())
+ .filter(|&n| n > 0)
+ .unwrap_or_else(|| {
+ std::thread::available_parallelism()
+ .map(|c| (c.get() / 8).max(2))
+ .unwrap_or(4)
+ });
+ tokio::sync::Semaphore::new(n)
+ })
+}
+
// ── Analyzer helpers ──────────────────────────────────────────────────────────
use std::sync::Arc;