Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(ingest) P1.1: route auto-id bulk through a correctness-fixed turbo path

xerj-org1a9f75bda8c2
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index ed040ecf3..1798404f1 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -663,6 +663,17 @@ pub async fn process_bulk_with_opts(
String,
Vec<(usize, String, std::sync::Arc<[u8]>)>,
> = HashMap::new();
+ // AUTO-ID plain-index actions (no explicit `_id`). These are
+ // guaranteed new — they can NEVER overwrite — so every one is a
+ // "created"/201. That makes them safe to push through the single
+ // `index_batch_turbo_raw` batch call, whose response is hardcoded
+ // to "created". Explicit-id index actions stay in `index_batches`
+ // and run the per-doc loop because they may overwrite and so need
+ // created-vs-updated / 201-vs-200 semantics turbo can't express.
+ let mut auto_id_batches: HashMap<
+ String,
+ Vec<(usize, String, std::sync::Arc<[u8]>)>,
+ > = HashMap::new();
let mut non_index_actions: Vec<ParsedAction> = Vec::new();
// Precompute which target indices declare any strict-format date
@@ -717,13 +728,23 @@ pub async fn process_bulk_with_opts(
&& !index_has_strict_date(&action.target_index, &mut index_needs_date_validation)
&& !index_has_dynamic_copy(&action.target_index, &mut index_needs_dynamic_copy);
if is_plain_index {
- let id = action
- .doc_id
- .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
- index_batches
- .entry(action.target_index)
- .or_default()
- .push((action.item_index, id, action.doc_bytes));
+ if action.doc_id.is_none() {
+ // Auto-id: brand-new doc, always "created"/201. Route the
+ // whole group through ONE `index_batch_turbo_raw` call.
+ let id = uuid::Uuid::new_v4().to_string();
+ auto_id_batches
+ .entry(action.target_index)
+ .or_default()
+ .push((action.item_index, id, action.doc_bytes));
+ } else {
+ // Explicit id: may overwrite → keep the per-doc path so
+ // created/updated + 201/200 is resolved correctly.
+ let id = action.doc_id.unwrap();
+ index_batches
+ .entry(action.target_index)
+ .or_default()
+ .push((action.item_index, id, action.doc_bytes));
+ }
} else {
non_index_actions.push(action);
}
@@ -732,6 +753,80 @@ pub async fn process_bulk_with_opts(
let group_ms = t_group.elapsed().as_millis() as u64;
let t_exec = std::time::Instant::now();
+
+ // ── Auto-id index actions: ONE turbo batch per target index ──────────
+ //
+ // `index_batch_turbo_raw` appends the whole group with a single WAL
+ // batch + one parsed FTS-memtable insert per doc (the now-correct,
+ // single-representation path). It returns one `IndexResponse` per
+ // input doc IN ORDER, so we map them straight back onto the item
+ // slots — all 201/"created" since auto-id docs can't overwrite.
+ for (index_name, batch) in auto_id_batches {
+ let idx = match engine.get_or_create_index(&index_name) {
+ Ok(i) => i,
+ Err(e) => {
+ for (item_idx, id, _) in batch {
+ items[item_idx] = Some(BulkItemResult {
+ action: "index".into(),
+ index: index_name.clone(),
+ id,
+ status: 500,
+ result: None,
+ error: Some(e.to_string()),
+ get_source: None,
+ });
+ errors = true;
+ }
+ continue;
+ }
+ };
+
+ // Keep item indices parallel to the docs we hand to turbo so we
+ // can re-associate responses (turbo preserves input order).
+ let item_indices: Vec<usize> = batch.iter().map(|(i, _, _)| *i).collect();
+ let turbo_docs: Vec<(String, std::sync::Arc<[u8]>)> = batch
+ .into_iter()
+ .map(|(_, id, bytes)| (id, bytes))
+ .collect();
+
+ match idx.index_batch_turbo_raw(turbo_docs).await {
+ Ok(responses) => {
+ for (k, resp) in responses.into_iter().enumerate() {
+ let item_idx = item_indices[k];
+ items[item_idx] = Some(BulkItemResult {
+ action: "index".into(),
+ index: index_name.clone(),
+ id: resp.id,
+ status: 201,
+ result: Some("created".into()),
+ error: None,
+ get_source: None,
+ });
+ }
+ }
+ Err(e) => {
+ // Whole-batch failure (e.g. ResourceExhausted → 429):
+ // mark every item in the batch with the same status.
+ let status = match &e {
+ EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. }) => 429,
+ _ => 500,
+ };
+ for item_idx in item_indices {
+ items[item_idx] = Some(BulkItemResult {
+ action: "index".into(),
+ index: index_name.clone(),
+ id: String::new(),
+ status,
+ result: None,
+ error: Some(e.to_string()),
+ get_source: None,
+ });
+ }
+ errors = true;
+ }
+ }
+ }
+
// Execute turbo batches (index-only).
for (index_name, batch) in index_batches {
let idx = match engine.get_or_create_index(&index_name) {
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 3ea7b23aa..54b7df9ec 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -1244,27 +1244,27 @@ impl Index {
let seq_nos = self.store.wal_append_batch(&wal_refs)?;
- // Push each doc as a raw-bytes memtable entry. Route the whole
- // batch by the first doc_id's hash so we hold ONE shard lock
- // for the entire loop (same strategy as the Value-carrying
- // turbo path).
- // Two memtable inserts per doc:
- // 1. Storage memtable (raw bytes) — for _source retrieval
- // 2. FTS memtable (parsed fields) — for search + aggregations
- let shard_idx = if docs.is_empty() {
- 0
- } else {
- self.memtable.shard_for_dynamic(&docs[0].0)
- };
- // Storage memtable: raw bytes for _source.
- self.memtable.with_shard_mut(shard_idx, |mem| {
- for (i, (id, bytes)) in docs.iter().enumerate() {
- mem.remove(id);
- mem.insert_raw_bytes_with_seq(seq_nos[i], id.clone(), Arc::clone(bytes));
- }
- });
-
- // FTS memtable: parsed field values for search + aggs.
+ // Insert each doc EXACTLY ONCE into the engine FTS memtable,
+ // routed to the doc's OWN shard — bit-for-bit identical to the
+ // per-doc `index_document` path.
+ //
+ // The earlier "ultra-turbo" shape pushed every doc TWICE: once
+ // as a raw-bytes entry pinned to docs[0]'s shard (intended for
+ // _source) and again via `insert` on the doc's own shard (for
+ // search/aggs). That double-represented each doc in
+ // `memtable.docs` — often on two different shards — so the
+ // brute-force agg corpus (`all_docs_with_sources`) and every
+ // terms/stats/cardinality/date_histogram aggregation counted
+ // each UNFLUSHED doc twice (140-test regression).
+ //
+ // A single `insert` discharges BOTH responsibilities: it stores
+ // the parsed `source` so `_source` GET (`get_doc_source_as_value`)
+ // resolves it, AND it builds the inverted index + columnar
+ // doc-values for search + aggregations. `wal_append_batch`
+ // above already wrote the WAL frames and set the version_map
+ // (so `_count`/`hits.total` via `version_map.live_count()` and
+ // GET visibility are correct), exactly like `self.store.index`
+ // does on the per-doc path.
let mut responses = Vec::with_capacity(batch_len);
for (i, (id, bytes)) in docs.iter().enumerate() {
let source = serde_json::from_slice::<Value>(bytes)
@@ -1273,11 +1273,15 @@ impl Index {
// Dynamic mapping: add newly-seen fields to schema.
self.evolve_schema_from_doc(&source).await;
- // Index field values.
- let schema_guard = self.schema.read().await;
- let mem = &*self.memtable;
- mem.insert(id.clone(), &source, &schema_guard.schema, seq_nos[i]);
- drop(schema_guard);
+ // Index field values into the FTS memtable once, on the
+ // doc's own shard. `remove` first so a same-id overwrite
+ // leaves exactly one live entry (mirrors index_document).
+ {
+ let schema_guard = self.schema.read().await;
+ let mem = &*self.memtable;
+ mem.remove(id);
+ mem.insert(id.clone(), &source, &schema_guard.schema, seq_nos[i]);
+ }
let version = self.doc_count.fetch_add(1, Ordering::Relaxed) + 1;
responses.push(IndexResponse {