AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
Merge mixed-tail fix (700d283) — 7 root causes: quadratic FTS write, pool isolation, cardinality fast path, cooperative timeouts, O(1) memtable
xerj-orgccc7c9e292da
diff --cc engine/crates/xerj-engine/src/index.rsindex f37bf9900,f587b83e1..480a4baf0--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@@ -7701,8 -7959,11 +8003,12 @@@ async fn do_flush_shard// 50-100 ms segment write that follows, and 100K-doc/s ingest// bench numbers are unaffected.let build_fts = move |meta: &xerj_storage::segment::SegmentMeta| -> xerj_storage::Result<()> {+ // Whole side-car build runs on the dedicated ingest pool so a+ // flush burst can't queue search/agg par_iters behind it on the+ // global rayon pool (read-under-write; see `crate::ingest_pool`).+ crate::ingest_pool().install(|| {// ── Doc-values side-car (always built) ────────────────────+ let t_dv = std::time::Instant::now();{let columns = build_doc_value_columns(drained_fts.iter().map(|(_id, _fields, src)| Some(src.as_ref())),@@@ -7737,14 -7992,8 +8043,15 @@@if let Err(e) = fts_writer.finish() {tracing::warn!("flush: FTS build failed: {e}");}+ fts_finish_us = t_fin.elapsed().as_micros();+ }+ if prof {+ eprintln!(+ "XERJ_PROF flush-sidecar docs={} dv_us={} fts_add_us={} fts_finish_us={}",+ fts_doc_count, dv_us, fts_add_us, fts_finish_us+ );}+ });let _ = &segments_dir;let _ = &drained_fts;@@@ -7752,8 -8001,11 +8059,12 @@@};// Hand the pre-drained storage entries to the finaliser.+ let t_finalize = std::time::Instant::now();- let meta = match store.finalize_flush_with_publisher(storage_drained, build_fts)? {+ // Segment encode + side-car build on the deprioritised ingest pool+ // (build_fts's inner install() is a same-pool no-op).+ let meta = match crate::ingest_pool()+ .install(|| store.finalize_flush_with_publisher(storage_drained, build_fts))?+ {Some(m) => m,None => {tracing::warn!("storage finalize returned None — unexpected");diff --cc engine/crates/xerj-fts/src/postings.rsindex 4982cf498,a8df4b8bb..0f321b0f0--- a/engine/crates/xerj-fts/src/postings.rs+++ b/engine/crates/xerj-fts/src/postings.rs@@@ -153,46 -153,47 +153,64 @@@ impl PostingsWriter/// When the writer was constructed without positions, `position` is/// ignored and no heap growth happens in the `positions` Vec.pub fn add_occurrence(&mut self, term: &str, doc_id: u32, position: u32) {- let list = self.postings.entry(term.to_owned()).or_default();-- if let Some(last) = list.last_mut() {- if last.doc_id == doc_id {- last.term_freq += 1;- if self.store_positions {- last.positions.push(position);+ // Lookup-first to avoid the `term.to_owned()` String allocation+ // that `entry()` forced on EVERY occurrence (13M+ allocs per 1M+ // docs at flush time). Misses (first occurrence of a term) pay+ // one extra BTreeMap descent — postings are Zipf-shaped, so hits+ // dominate.+ if let Some(list) = self.postings.get_mut(term) {+ if let Some(last) = list.last_mut() {+ if last.doc_id == doc_id {+ last.term_freq += 1;+ if self.store_positions {+ last.positions.push(position);+ }+ return;}- return;}+ list.push(RawPosting {+ doc_id,+ term_freq: 1,+ positions: if self.store_positions {+ vec![position]+ } else {+ Vec::new()+ },+ });+ return;}-- list.push(RawPosting {- doc_id,- term_freq: 1,- positions: if self.store_positions {- vec![position]- } else {- Vec::new()- },- });+ self.postings.insert(+ term.to_owned(),+ vec![RawPosting {+ doc_id,+ term_freq: 1,+ positions: if self.store_positions {+ vec![position]+ } else {+ Vec::new()+ },+ }],+ );}+ /// Direct `(doc_frequency, total_term_frequency)` lookup for ONE term.+ ///+ /// O(log T + postings(term)) — use this in per-term loops. The segment+ /// writer (`write_field_static`) previously resolved stats via+ /// `term_stats().find(term)`, which walks the term map from the start+ /// *and sums every earlier term's postings* on each call — O(T² × P)+ /// across the write loop. Invisible on 625-doc flush segments, but a+ /// merged segment over a float field (≈1 distinct term per doc) spun+ /// 100 % CPU for HOURS at 3 M docs, pinning rayon workers and starving+ /// every search + flush behind it (the read-under-write collapse).+ pub fn stats_for(&self, term: &str) -> Option<(u32, u64)> {+ self.postings.get(term).map(|postings| {+ let doc_freq = postings.len() as u32;+ let ttf: u64 = postings.iter().map(|p| p.term_freq as u64).sum();+ (doc_freq, ttf)+ })+ }+/// Returns an iterator over `(term, doc_frequency, total_term_frequency)`/// without consuming self (for building the FST term dictionary).pub fn term_stats(&self) -> impl Iterator<Item = (&str, u32, u64)> {