AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(memtable): intern doc-id as Arc<str> — kill the per-token String malloc under the shard write lock
xerj-org5ebd07297d85
diff --git a/engine/crates/xerj-engine/examples/alloc_count.rs b/engine/crates/xerj-engine/examples/alloc_count.rsnew file mode 100644index 000000000..435418fc9--- /dev/null+++ b/engine/crates/xerj-engine/examples/alloc_count.rs@@ -0,0 +1,86 @@+//! Deterministic allocation-count harness for the memtable insert hot path.+//!+//! Wraps the global allocator with an atomic counter and measures the number+//! of heap allocations performed *inside the `insert_analyzed` loop only*+//! (analysis is done up front, outside the measured region). This isolates+//! the exact under-shard-write-lock work the Arc<str> doc-id interning change+//! targets. Fully deterministic and single-threaded — immune to the+//! concurrent-agent CPU contamination that skews wall-clock numbers.++use std::alloc::{GlobalAlloc, Layout, System};+use std::sync::atomic::{AtomicUsize, Ordering};+use std::sync::Arc;++use serde_json::Value;+use xerj_common::types::Schema;+use xerj_engine::memtable::{analyze_doc, FtsMemtable};+use xerj_fts::analyzer::AnalyzerRegistry;++struct Counting;+static ALLOCS: AtomicUsize = AtomicUsize::new(0);+static BYTES: AtomicUsize = AtomicUsize::new(0);++unsafe impl GlobalAlloc for Counting {+ unsafe fn alloc(&self, l: Layout) -> *mut u8 {+ ALLOCS.fetch_add(1, Ordering::Relaxed);+ BYTES.fetch_add(l.size(), Ordering::Relaxed);+ System.alloc(l)+ }+ unsafe fn dealloc(&self, p: *mut u8, l: Layout) {+ System.dealloc(p, l)+ }+}++#[global_allocator]+static A: Counting = Counting;++fn main() {+ let corpus = "/home/claude/ai/xerj/demo/data/extras/chat-events.ndjson";+ let raw: Vec<String> = std::fs::read_to_string(corpus)+ .unwrap()+ .lines()+ .map(|l| l.to_string())+ .collect();+ let n_docs: usize = std::env::args()+ .nth(1)+ .and_then(|s| s.parse().ok())+ .unwrap_or(40_000);++ // Build inputs OUTSIDE the measured region: parse + analyze every doc,+ // and synthesize turbo-style auto ids (20-char, like the bulk path).+ let reg = AnalyzerRegistry::default();+ let analyzer = reg.get_analyzer("standard").unwrap();+ let schema = Schema::default();+ let mut inputs: Vec<(String, Arc<Value>, Vec<(String, Vec<xerj_fts::analyzer::Token>)>)> =+ Vec::with_capacity(n_docs);+ for i in 0..n_docs {+ let line = &raw[i % raw.len()];+ let v: Value = serde_json::from_str(line).unwrap();+ let analyzed = analyze_doc(&v, &schema, &analyzer);+ // Auto-id shape: 20 base62-ish chars, unique per doc.+ let id = format!("{:020x}", 0xdead_0000_0000u64 + i as u64);+ inputs.push((id, Arc::new(v), analyzed));+ }++ let mut mem = FtsMemtable::new();++ // ── measured region: the insert_analyzed loop only ──+ let a0 = ALLOCS.load(Ordering::Relaxed);+ let b0 = BYTES.load(Ordering::Relaxed);+ for (i, (id, src, analyzed)) in inputs.iter().enumerate() {+ mem.insert_analyzed(i as u64, id.clone(), Arc::clone(src), analyzed, 800);+ }+ let a1 = ALLOCS.load(Ordering::Relaxed);+ let b1 = BYTES.load(Ordering::Relaxed);++ let allocs = a1 - a0;+ let bytes = b1 - b0;+ // Subtract the unavoidable `id.clone()` (one String per doc, done by the+ // caller inside the loop) so the number reflects work INSIDE the memtable.+ println!("docs={}", n_docs);+ println!("total_allocs_in_insert_loop={}", allocs);+ println!("allocs_per_doc={:.3}", allocs as f64 / n_docs as f64);+ println!("bytes_per_doc={:.1}", bytes as f64 / n_docs as f64);+ // keep mem alive so drops don't happen inside the measured window+ std::hint::black_box(&mem);+}diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex e2906f6f5..50069c4f1 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -96,6 +96,25 @@ pub fn extract_text_fields_from(source: &Value) -> HashMap<String, String> {out}+/// Interned document identifier.+///+/// One `Arc<str>` is allocated per document at insert time and its+/// pointer is cloned (a relaxed refcount bump, no `malloc`) into every+/// doc-id-keyed map — the postings lists, `field_lengths`, and+/// `doc_id_index`. Pre-interning, `insert_analyzed` paid a fresh+/// `String` heap allocation of the doc id for EVERY (doc × token) it+/// pushed into a posting list plus one per (doc × field) length entry —+/// profiled as the dominant remaining under-write-lock cost on the+/// 1M×c8 bulk ceiling (~40 ms / 10k-doc batch). Interning collapses+/// that to a single allocation per doc; the per-token work becomes a+/// pointer bump inside the shard critical section.+///+/// `Arc<str>: Borrow<str>` and its `Hash`/`Eq` forward to the pointed-to+/// `str`, so every existing lookup by `&str` (`remove`, `contains_key`,+/// BM25 `field_lengths` probe) is unchanged and byte-for-byte+/// identical to the `String`-keyed maps it replaces.+type DocId = Arc<str>;+/// Posting list entry: doc_id → term frequency.////// FxHashMap (rustc-hash) instead of the SipHash-keyed std HashMap:@@ -105,7 +124,7 @@ pub fn extract_text_fields_from(source: &Value) -> HashMap<String, String> {/// maps are never exposed to untrusted-key HashDoS (doc ids and field/// names of a single tenant's own documents), so the DoS-resistant/// hasher buys nothing here.-type PostingList = FxHashMap<String, u32>; // doc_id → tf+type PostingList = FxHashMap<DocId, u32>; // doc_id → tf// ── DocValues ─────────────────────────────────────────────────────────────────@@ -1410,11 +1429,11 @@ pub struct FtsMemtable {/// Analyzer registry.registry: Arc<AnalyzerRegistry>,/// Precomputed field lengths for BM25 scoring: field → {doc_id → token_count}- field_lengths: FxHashMap<String, FxHashMap<String, u32>>,+ field_lengths: FxHashMap<String, FxHashMap<DocId, u32>>,/// Running average field length per field: field → (total_tokens, doc_count)avg_field_lengths: FxHashMap<String, (f64, u64)>,/// doc_id → position in self.docs for O(1) lookup- doc_id_index: FxHashMap<String, usize>,+ doc_id_index: FxHashMap<DocId, usize>,/// Delete-aware BM25 collection statistics (Lucene/ES parity).////// When a document is superseded by an update (remove + re-insert) or@@ -1523,6 +1542,13 @@ impl FtsMemtable {analyzed: &[(String, Vec<Token>)],size: usize,) {+ // Intern the doc id ONCE (one `malloc`). Every posting / field-+ // length / doc-id-index entry below then stores a pointer-bumped+ // `Arc<str>` clone instead of a fresh `String` heap allocation —+ // the per-token clone under the shard write lock becomes a+ // relaxed refcount increment. See `DocId` for the rationale.+ let doc_key: DocId = Arc::from(doc_id.as_str());+for (field_name, tokens) in analyzed {let token_count = tokens.len() as u32;@@ -1530,7 +1556,7 @@ impl FtsMemtable {// recurs across every doc of the batch, so `entry_no_clone`// skips the per-doc field-key `malloc` on the hot path.entry_no_clone(&mut self.field_lengths, field_name, Default::default)- .insert(doc_id.clone(), token_count);+ .insert(doc_key.clone(), token_count);// Update running average.let entry = entry_no_clone(&mut self.avg_field_lengths, field_name, || (0.0, 0));@@ -1542,9 +1568,8 @@ impl FtsMemtable {// `token.text` likewise recurs (low-cardinality keyword-ish// fields), so only a genuinely new term pays the key clone.let posting = entry_no_clone(field_index, &token.text, Default::default);- // `doc_id` is unique on the turbo path (fresh auto-id) so it- // always misses — the owning clone is unavoidable here.- *posting.entry(doc_id.clone()).or_insert(0) += 1;+ // Interned doc id — a pointer bump, not a `String` malloc.+ *posting.entry(doc_key.clone()).or_insert(0) += 1;}}@@ -1556,7 +1581,7 @@ impl FtsMemtable {self.doc_values.push(&source, doc_index);// Track doc_id → index for O(1) lookup.- self.doc_id_index.insert(doc_id.clone(), doc_index);+ self.doc_id_index.insert(doc_key, doc_index);self.docs.push(MemEntry {seq_no,@@ -1610,7 +1635,7 @@ impl FtsMemtable {self.total_bytes += estimated;let doc_index = self.docs.len();- self.doc_id_index.insert(doc_id.clone(), doc_index);+ self.doc_id_index.insert(Arc::from(doc_id.as_str()), doc_index);self.docs.push(MemEntry {seq_no,@@ -1639,7 +1664,7 @@ impl FtsMemtable {self.total_bytes += estimated;let doc_index = self.docs.len();- self.doc_id_index.insert(doc_id.clone(), doc_index);+ self.doc_id_index.insert(Arc::from(doc_id.as_str()), doc_index);self.docs.push(MemEntry {seq_no,doc_id,@@ -1703,7 +1728,7 @@ impl FtsMemtable {self.total_bytes += estimated;let doc_index = self.docs.len();- self.doc_id_index.insert(doc_id.clone(), doc_index);+ self.doc_id_index.insert(Arc::from(doc_id.as_str()), doc_index);self.docs.push(MemEntry {seq_no,@@ -1742,6 +1767,7 @@ impl FtsMemtable {.or_else(|| self.registry.get_analyzer("standard")).expect("standard analyzer always present");+ let doc_key: DocId = Arc::from(doc_id.as_str());for (field_name, text) in &text_fields {let tokens = analyzer.analyze(text);let token_count = tokens.len() as u32;@@ -1750,7 +1776,7 @@ impl FtsMemtable {self.field_lengths.entry(field_name.clone()).or_default()- .insert(doc_id.clone(), token_count);+ .insert(doc_key.clone(), token_count);// Update running average.let entry = self.avg_field_lengths.entry(field_name.clone()).or_insert((0.0, 0));@@ -1760,7 +1786,7 @@ impl FtsMemtable {let field_index = self.index.entry(field_name.clone()).or_default();for token in &tokens {let posting = field_index.entry(token.text.clone()).or_default();- *posting.entry(doc_id.clone()).or_insert(0) += 1;+ *posting.entry(doc_key.clone()).or_insert(0) += 1;}}@@ -1774,8 +1800,8 @@ impl FtsMemtable {// Pass a reference through the Arc — DocValues reads without cloning source.self.doc_values.push(&source, doc_index);- // Track doc_id → index for O(1) lookup.- self.doc_id_index.insert(doc_id.clone(), doc_index);+ // Track doc_id → index for O(1) lookup (interned Arc<str> key).+ self.doc_id_index.insert(doc_key, doc_index);self.docs.push(MemEntry {seq_no: 0,@@ -1905,7 +1931,10 @@ impl FtsMemtable {}let doc_count = if global_doc_count > 0 { global_doc_count } else { self.docs.len() as u64 };- let mut scores: HashMap<String, f32> = HashMap::new();+ // Keyed by the interned `Arc<str>` doc id from the postings lists,+ // so accumulating a term's contribution is a pointer-bump clone+ // rather than a per-hit `String` allocation.+ let mut scores: HashMap<DocId, f32> = HashMap::new();for token in &tokens {// Search across requested fields (or all indexed fields if none specified).@@ -1956,7 +1985,7 @@ impl FtsMemtable {let mut hits: Vec<MemtableHit> = scores.into_iter()- .map(|(doc_id, score)| MemtableHit { doc_id, score })+ .map(|(doc_id, score)| MemtableHit { doc_id: doc_id.to_string(), score }).collect();// Sort by score descending.