AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(ingest) P1.1: route auto-id bulk through a correctness-fixed turbo path
xerj-org1a9f75bda8c2
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex ed040ecf3..1798404f1 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -663,6 +663,17 @@ pub async fn process_bulk_with_opts(String,Vec<(usize, String, std::sync::Arc<[u8]>)>,> = HashMap::new();+ // AUTO-ID plain-index actions (no explicit `_id`). These are+ // guaranteed new — they can NEVER overwrite — so every one is a+ // "created"/201. That makes them safe to push through the single+ // `index_batch_turbo_raw` batch call, whose response is hardcoded+ // to "created". Explicit-id index actions stay in `index_batches`+ // and run the per-doc loop because they may overwrite and so need+ // created-vs-updated / 201-vs-200 semantics turbo can't express.+ let mut auto_id_batches: HashMap<+ String,+ Vec<(usize, String, std::sync::Arc<[u8]>)>,+ > = HashMap::new();let mut non_index_actions: Vec<ParsedAction> = Vec::new();// Precompute which target indices declare any strict-format date@@ -717,13 +728,23 @@ pub async fn process_bulk_with_opts(&& !index_has_strict_date(&action.target_index, &mut index_needs_date_validation)&& !index_has_dynamic_copy(&action.target_index, &mut index_needs_dynamic_copy);if is_plain_index {- let id = action- .doc_id- .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());- index_batches- .entry(action.target_index)- .or_default()- .push((action.item_index, id, action.doc_bytes));+ if action.doc_id.is_none() {+ // Auto-id: brand-new doc, always "created"/201. Route the+ // whole group through ONE `index_batch_turbo_raw` call.+ let id = uuid::Uuid::new_v4().to_string();+ auto_id_batches+ .entry(action.target_index)+ .or_default()+ .push((action.item_index, id, action.doc_bytes));+ } else {+ // Explicit id: may overwrite → keep the per-doc path so+ // created/updated + 201/200 is resolved correctly.+ let id = action.doc_id.unwrap();+ index_batches+ .entry(action.target_index)+ .or_default()+ .push((action.item_index, id, action.doc_bytes));+ }} else {non_index_actions.push(action);}@@ -732,6 +753,80 @@ pub async fn process_bulk_with_opts(let group_ms = t_group.elapsed().as_millis() as u64;let t_exec = std::time::Instant::now();++ // ── Auto-id index actions: ONE turbo batch per target index ──────────+ //+ // `index_batch_turbo_raw` appends the whole group with a single WAL+ // batch + one parsed FTS-memtable insert per doc (the now-correct,+ // single-representation path). It returns one `IndexResponse` per+ // input doc IN ORDER, so we map them straight back onto the item+ // slots — all 201/"created" since auto-id docs can't overwrite.+ for (index_name, batch) in auto_id_batches {+ let idx = match engine.get_or_create_index(&index_name) {+ Ok(i) => i,+ Err(e) => {+ for (item_idx, id, _) in batch {+ items[item_idx] = Some(BulkItemResult {+ action: "index".into(),+ index: index_name.clone(),+ id,+ status: 500,+ result: None,+ error: Some(e.to_string()),+ get_source: None,+ });+ errors = true;+ }+ continue;+ }+ };++ // Keep item indices parallel to the docs we hand to turbo so we+ // can re-associate responses (turbo preserves input order).+ let item_indices: Vec<usize> = batch.iter().map(|(i, _, _)| *i).collect();+ let turbo_docs: Vec<(String, std::sync::Arc<[u8]>)> = batch+ .into_iter()+ .map(|(_, id, bytes)| (id, bytes))+ .collect();++ match idx.index_batch_turbo_raw(turbo_docs).await {+ Ok(responses) => {+ for (k, resp) in responses.into_iter().enumerate() {+ let item_idx = item_indices[k];+ items[item_idx] = Some(BulkItemResult {+ action: "index".into(),+ index: index_name.clone(),+ id: resp.id,+ status: 201,+ result: Some("created".into()),+ error: None,+ get_source: None,+ });+ }+ }+ Err(e) => {+ // Whole-batch failure (e.g. ResourceExhausted → 429):+ // mark every item in the batch with the same status.+ let status = match &e {+ EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. }) => 429,+ _ => 500,+ };+ for item_idx in item_indices {+ items[item_idx] = Some(BulkItemResult {+ action: "index".into(),+ index: index_name.clone(),+ id: String::new(),+ status,+ result: None,+ error: Some(e.to_string()),+ get_source: None,+ });+ }+ errors = true;+ }+ }+ }+// Execute turbo batches (index-only).for (index_name, batch) in index_batches {let idx = match engine.get_or_create_index(&index_name) {diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 3ea7b23aa..54b7df9ec 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -1244,27 +1244,27 @@ impl Index {let seq_nos = self.store.wal_append_batch(&wal_refs)?;- // Push each doc as a raw-bytes memtable entry. Route the whole- // batch by the first doc_id's hash so we hold ONE shard lock- // for the entire loop (same strategy as the Value-carrying- // turbo path).- // Two memtable inserts per doc:- // 1. Storage memtable (raw bytes) — for _source retrieval- // 2. FTS memtable (parsed fields) — for search + aggregations- let shard_idx = if docs.is_empty() {- 0- } else {- self.memtable.shard_for_dynamic(&docs[0].0)- };- // Storage memtable: raw bytes for _source.- self.memtable.with_shard_mut(shard_idx, |mem| {- for (i, (id, bytes)) in docs.iter().enumerate() {- mem.remove(id);- mem.insert_raw_bytes_with_seq(seq_nos[i], id.clone(), Arc::clone(bytes));- }- });-- // FTS memtable: parsed field values for search + aggs.+ // Insert each doc EXACTLY ONCE into the engine FTS memtable,+ // routed to the doc's OWN shard — bit-for-bit identical to the+ // per-doc `index_document` path.+ //+ // The earlier "ultra-turbo" shape pushed every doc TWICE: once+ // as a raw-bytes entry pinned to docs[0]'s shard (intended for+ // _source) and again via `insert` on the doc's own shard (for+ // search/aggs). That double-represented each doc in+ // `memtable.docs` — often on two different shards — so the+ // brute-force agg corpus (`all_docs_with_sources`) and every+ // terms/stats/cardinality/date_histogram aggregation counted+ // each UNFLUSHED doc twice (140-test regression).+ //+ // A single `insert` discharges BOTH responsibilities: it stores+ // the parsed `source` so `_source` GET (`get_doc_source_as_value`)+ // resolves it, AND it builds the inverted index + columnar+ // doc-values for search + aggregations. `wal_append_batch`+ // above already wrote the WAL frames and set the version_map+ // (so `_count`/`hits.total` via `version_map.live_count()` and+ // GET visibility are correct), exactly like `self.store.index`+ // does on the per-doc path.let mut responses = Vec::with_capacity(batch_len);for (i, (id, bytes)) in docs.iter().enumerate() {let source = serde_json::from_slice::<Value>(bytes)@@ -1273,11 +1273,15 @@ impl Index {// Dynamic mapping: add newly-seen fields to schema.self.evolve_schema_from_doc(&source).await;- // Index field values.- let schema_guard = self.schema.read().await;- let mem = &*self.memtable;- mem.insert(id.clone(), &source, &schema_guard.schema, seq_nos[i]);- drop(schema_guard);+ // Index field values into the FTS memtable once, on the+ // doc's own shard. `remove` first so a same-id overwrite+ // leaves exactly one live entry (mirrors index_document).+ {+ let schema_guard = self.schema.read().await;+ let mem = &*self.memtable;+ mem.remove(id);+ mem.insert(id.clone(), &source, &schema_guard.schema, seq_nos[i]);+ }let version = self.doc_count.fetch_add(1, Ordering::Relaxed) + 1;responses.push(IndexResponse {