Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(engine): full-corpus agg path deep-cloned the memtable under lock

Vincenzo Lombardo6a54cf58106d
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 0233b97b7..de54a1510 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -10902,11 +10902,21 @@ impl Index {
let all_docs: Vec<Value> = if need_full_corpus {
// `_id` is injected onto each source so `top_hits` / `_id`-keyed
// aggs over the corpus still work (the fast path never provided it).
+ // Arc-share out of the memtable (cheap refcount bump under the
+ // per-shard read lock — same fix `all_docs_with_sources_arc`
+ // already applies for the fast-agg path, see its doc comment),
+ // then deep-clone into owned `Value` AFTER the lock is released.
+ // `run_aggs_with_all` needs owned `Value`, but this was previously
+ // deep-cloning the whole memtable WHILE holding the shard lock —
+ // the dominant `hydrate+corpus` cost under concurrent dashboard
+ // load, where several panel queries serialize on the same
+ // per-shard RwLock while each does a full O(doc) clone under it.
let mut docs: Vec<Value> = self
.memtable
- .all_docs_with_sources()
+ .all_docs_with_sources_arc()
.into_iter()
- .map(|(id, mut v)| {
+ .map(|(id, v)| {
+ let mut v = (*v).clone();
if let Some(o) = v.as_object_mut() {
o.entry("_id".to_string())
.or_insert_with(|| Value::String(id));