Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Merge mixed-tail fix (700d283) — 7 root causes: quadratic FTS write, pool isolation, cardinality fast path, cooperative timeouts, O(1) memtable

xerj-orgccc7c9e292da
diff --cc engine/crates/xerj-engine/src/index.rs
index f37bf9900,f587b83e1..480a4baf0
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@@ -7701,8 -7959,11 +8003,12 @@@ async fn do_flush_shard
// 50-100 ms segment write that follows, and 100K-doc/s ingest
// bench numbers are unaffected.
let build_fts = move |meta: &xerj_storage::segment::SegmentMeta| -> xerj_storage::Result<()> {
+ // Whole side-car build runs on the dedicated ingest pool so a
+ // flush burst can't queue search/agg par_iters behind it on the
+ // global rayon pool (read-under-write; see `crate::ingest_pool`).
+ crate::ingest_pool().install(|| {
// ── Doc-values side-car (always built) ────────────────────
+ let t_dv = std::time::Instant::now();
{
let columns = build_doc_value_columns(
drained_fts.iter().map(|(_id, _fields, src)| Some(src.as_ref())),
@@@ -7737,14 -7992,8 +8043,15 @@@
if let Err(e) = fts_writer.finish() {
tracing::warn!("flush: FTS build failed: {e}");
}
+ fts_finish_us = t_fin.elapsed().as_micros();
+ }
+ if prof {
+ eprintln!(
+ "XERJ_PROF flush-sidecar docs={} dv_us={} fts_add_us={} fts_finish_us={}",
+ fts_doc_count, dv_us, fts_add_us, fts_finish_us
+ );
}
+ });
let _ = &segments_dir;
let _ = &drained_fts;
@@@ -7752,8 -8001,11 +8059,12 @@@
};
// Hand the pre-drained storage entries to the finaliser.
+ let t_finalize = std::time::Instant::now();
- let meta = match store.finalize_flush_with_publisher(storage_drained, build_fts)? {
+ // Segment encode + side-car build on the deprioritised ingest pool
+ // (build_fts's inner install() is a same-pool no-op).
+ let meta = match crate::ingest_pool()
+ .install(|| store.finalize_flush_with_publisher(storage_drained, build_fts))?
+ {
Some(m) => m,
None => {
tracing::warn!("storage finalize returned None — unexpected");
diff --cc engine/crates/xerj-fts/src/postings.rs
index 4982cf498,a8df4b8bb..0f321b0f0
--- a/engine/crates/xerj-fts/src/postings.rs
+++ b/engine/crates/xerj-fts/src/postings.rs
@@@ -153,46 -153,47 +153,64 @@@ impl PostingsWriter
/// When the writer was constructed without positions, `position` is
/// ignored and no heap growth happens in the `positions` Vec.
pub fn add_occurrence(&mut self, term: &str, doc_id: u32, position: u32) {
- let list = self.postings.entry(term.to_owned()).or_default();
-
- if let Some(last) = list.last_mut() {
- if last.doc_id == doc_id {
- last.term_freq += 1;
- if self.store_positions {
- last.positions.push(position);
+ // Lookup-first to avoid the `term.to_owned()` String allocation
+ // that `entry()` forced on EVERY occurrence (13M+ allocs per 1M
+ // docs at flush time). Misses (first occurrence of a term) pay
+ // one extra BTreeMap descent — postings are Zipf-shaped, so hits
+ // dominate.
+ if let Some(list) = self.postings.get_mut(term) {
+ if let Some(last) = list.last_mut() {
+ if last.doc_id == doc_id {
+ last.term_freq += 1;
+ if self.store_positions {
+ last.positions.push(position);
+ }
+ return;
}
- return;
}
+ list.push(RawPosting {
+ doc_id,
+ term_freq: 1,
+ positions: if self.store_positions {
+ vec![position]
+ } else {
+ Vec::new()
+ },
+ });
+ return;
}
-
- list.push(RawPosting {
- doc_id,
- term_freq: 1,
- positions: if self.store_positions {
- vec![position]
- } else {
- Vec::new()
- },
- });
+ self.postings.insert(
+ term.to_owned(),
+ vec![RawPosting {
+ doc_id,
+ term_freq: 1,
+ positions: if self.store_positions {
+ vec![position]
+ } else {
+ Vec::new()
+ },
+ }],
+ );
}
+ /// Direct `(doc_frequency, total_term_frequency)` lookup for ONE term.
+ ///
+ /// O(log T + postings(term)) — use this in per-term loops. The segment
+ /// writer (`write_field_static`) previously resolved stats via
+ /// `term_stats().find(term)`, which walks the term map from the start
+ /// *and sums every earlier term's postings* on each call — O(T² × P)
+ /// across the write loop. Invisible on 625-doc flush segments, but a
+ /// merged segment over a float field (≈1 distinct term per doc) spun
+ /// 100 % CPU for HOURS at 3 M docs, pinning rayon workers and starving
+ /// every search + flush behind it (the read-under-write collapse).
+ pub fn stats_for(&self, term: &str) -> Option<(u32, u64)> {
+ self.postings.get(term).map(|postings| {
+ let doc_freq = postings.len() as u32;
+ let ttf: u64 = postings.iter().map(|p| p.term_freq as u64).sum();
+ (doc_freq, ttf)
+ })
+ }
+
/// Returns an iterator over `(term, doc_frequency, total_term_frequency)`
/// without consuming self (for building the FST term dictionary).
pub fn term_stats(&self) -> impl Iterator<Item = (&str, u32, u64)> {