AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(ingest): batch schema-evolve + skip bulk-response Value round-trip — c8 1M +15-19%
xerj-org83d3a0bfd093
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 9015b5085..8ded226a9 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -9401,6 +9401,40 @@ async fn process_bulk_body(}let resp = EsBulkResponse { took: took_ms, errors, items };++ // Fast path (hot bulk-ingest): the Value round-trip below exists ONLY to+ // patch `_seq_no`/`_primary_term` sentinels for indices that declare+ // `disable_sequence_numbers`. Scan the (small) settings map ONCE per+ // request; when no index declares the flag — the overwhelmingly common+ // case — serialize the typed response directly. The old shape paid+ // struct→Value (≈100 k node allocs for a 10 k-item batch), a per-item+ // DashMap get + full settings-Value clone, and then a SECOND+ // serialization of the Value to bytes, on every bulk response.+ let seq_disabled_anywhere = state.engine.index_settings.iter().any(|e| {+ let s = e.value();+ let as_bool = |val: &Value| {+ val.as_bool()+ .unwrap_or_else(|| val.as_str().map(|x| x == "true").unwrap_or(false))+ };+ s.pointer("/index/disable_sequence_numbers").map(as_bool).unwrap_or(false)+ || s.get("index")+ .and_then(|i| i.get("index.disable_sequence_numbers"))+ .map(as_bool)+ .unwrap_or(false)+ || s.get("index.disable_sequence_numbers").map(as_bool).unwrap_or(false)+ });+ if !seq_disabled_anywhere {+ let mut r = Json(&resp).into_response();+ if all_backpressure {+ *r.status_mut() = axum::http::StatusCode::TOO_MANY_REQUESTS;+ r.headers_mut().insert(+ axum::http::header::RETRY_AFTER,+ axum::http::HeaderValue::from_static("1"),+ );+ }+ return r;+ }+// Post-process to apply per-index `disable_sequence_numbers` sentinel// values (_seq_no=-2, _primary_term=0) in the response. Serialize to// Value first so we can mutate the per-item fields without wideningdiff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex 832c1c62a..7b8422b6e 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -1230,6 +1230,13 @@ pub async fn process_bulk_with_opts(let exec_ms = t_exec.elapsed().as_millis() as u64;let took_ms = started.elapsed().as_millis() as u64;+ // THROWAWAY prof (XERJ_PROF): whole-request bulk attribution.+ if std::env::var_os("XERJ_PROF").is_some() {+ eprintln!(+ "XERJ_PROF bulk total_ms={} lines_ms={} parse_ms={} group_ms={} exec_ms={} n_lines={}",+ took_ms, lines_ms, parse_ms, group_ms, exec_ms, lines.len()+ );+ }if took_ms >= 50 {tracing::debug!(total_ms = took_ms,diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 480a4baf0..d73d8ec24 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -1316,6 +1316,14 @@ impl Index {let index_start = std::time::Instant::now();let batch_len = docs.len();+ // THROWAWAY prof (XERJ_PROF): per-phase attribution of the turbo path.+ let prof = std::env::var_os("XERJ_PROF").is_some();+ let mut p_wal_us = 0u128;+ let mut p_parse_us = 0u128;+ let mut p_evolve_us = 0u128;+ let mut p_analyze_us = 0u128;+ let mut p_insert_us = 0u128;+ let p_t = std::time::Instant::now();// Build WAL refs directly from bytes. We pass `Arc<Value::Null>`// as the `source` parameter because `wal_append_batch` uses@@ -1328,6 +1336,9 @@ impl Index {.collect();let seq_nos = self.store.wal_append_batch(&wal_refs)?;+ if prof {+ p_wal_us = p_t.elapsed().as_micros();+ }// Insert each doc EXACTLY ONCE into the engine FTS memtable,// routed to the doc's OWN shard — bit-for-bit identical to the@@ -1383,6 +1394,7 @@ impl Index {// parse bursts never queue ahead of search/agg par_iters on the// global rayon pool — see the read-under-write collapse notes on// `ingest_pool()`.+ let p_t = std::time::Instant::now();let sources: Vec<Arc<Value>> = crate::ingest_pool().install(|| {docs.par_iter().map(|(_, bytes)| {@@ -1393,10 +1405,16 @@ impl Index {}).collect()});+ if prof {+ p_parse_us = p_t.elapsed().as_micros();+ }- // 2. Dynamic mapping: evolve schema serially, in doc order.- for source in &sources {- self.evolve_schema_from_doc(source.as_ref()).await;+ // 2. Dynamic mapping: evolve schema in doc order — batched (one+ // schema read-lock for the whole batch, not one per doc).+ let p_t = std::time::Instant::now();+ self.evolve_schema_from_docs(&sources).await;+ if prof {+ p_evolve_us = p_t.elapsed().as_micros();}// 3. Parallel shard-partitioned insert. Partition the batch by@@ -1421,6 +1439,7 @@ impl Index {let analyzer = mem.default_analyzer().expect("standard analyzer always present");+ let p_t = std::time::Instant::now();let analyzed: Vec<Vec<(String, Vec<xerj_fts::analyzer::Token>)>> =crate::ingest_pool().install(|| {sources@@ -1430,6 +1449,10 @@ impl Index {}).collect()});+ if prof {+ p_analyze_us = p_t.elapsed().as_micros();+ }+ let p_t = std::time::Instant::now();let n_shards = mem.shard_count().max(1);let mut buckets: Vec<Vec<usize>> = vec![Vec::new(); n_shards];@@ -1464,6 +1487,17 @@ impl Index {});});});+ if prof {+ p_insert_us = p_t.elapsed().as_micros();+ }+ }++ if prof {+ eprintln!(+ "XERJ_PROF turbo-batch docs={} wal_us={} parse_us={} evolve_us={} analyze_us={} insert_us={} total_us={}",+ batch_len, p_wal_us, p_parse_us, p_evolve_us, p_analyze_us, p_insert_us,+ index_start.elapsed().as_micros()+ );}// 4. One batch-level version stamp, assigned in request order.@@ -6246,6 +6280,60 @@ impl Index {// ── Helpers ───────────────────────────────────────────────────────────────+ /// Batch variant of [`evolve_schema_from_doc`]: one schema read-lock+ /// acquisition for the WHOLE batch instead of one per doc. The per-doc+ /// variant paid a tokio `RwLock::read().await` per document — measured+ /// ~22 ms per 10 k-doc bulk batch (≈ a quarter of the whole turbo exec+ /// phase) even though the schema stops evolving after the first docs.+ ///+ /// Semantics are identical to calling `evolve_schema_from_doc` serially+ /// in doc order: new fields are collected in FIRST-SEEN order (so the+ /// first doc that introduces a field decides its inferred type) and+ /// added under a single write lock, which re-checks `has_field` exactly+ /// like the per-doc slow path does.+ async fn evolve_schema_from_docs(&self, sources: &[Arc<Value>]) {+ let new_fields: Vec<(String, FieldConfig)> = {+ let schema = self.schema.read().await;+ if !matches!(schema.dynamic, xerj_common::schema::DynamicMapping::Dynamic) {+ return;+ }+ let mut out: Vec<(String, FieldConfig)> = Vec::new();+ for source in sources {+ let Some(obj) = source.as_object() else {+ continue;+ };+ for (key, val) in obj {+ if !schema.schema.has_field(key)+ && !out.iter().any(|(k, _)| k == key)+ {+ let ft = infer_field_type(val);+ out.push((key.clone(), FieldConfig::new(key.clone(), ft)));+ }+ }+ }+ out+ };++ if new_fields.is_empty() {+ return;+ }++ let mut schema = self.schema.write().await;+ if !matches!(schema.dynamic, xerj_common::schema::DynamicMapping::Dynamic) {+ return;+ }+ let mut schema_changed = false;+ for (_, fc) in new_fields {+ if !schema.schema.has_field(&fc.name) {+ let _ = schema.schema.add_field(fc);+ schema_changed = true;+ }+ }+ if schema_changed {+ let _ = self.save_schema(&schema).await;+ }+ }+async fn evolve_schema_from_doc(&self, source: &Value) {let obj = match source.as_object() {Some(o) => o,