Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(ingest): batch schema-evolve + skip bulk-response Value round-trip — c8 1M +15-19%

xerj-org83d3a0bfd093
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 9015b5085..8ded226a9 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -9401,6 +9401,40 @@ async fn process_bulk_body(
}
let resp = EsBulkResponse { took: took_ms, errors, items };
+
+ // Fast path (hot bulk-ingest): the Value round-trip below exists ONLY to
+ // patch `_seq_no`/`_primary_term` sentinels for indices that declare
+ // `disable_sequence_numbers`. Scan the (small) settings map ONCE per
+ // request; when no index declares the flag — the overwhelmingly common
+ // case — serialize the typed response directly. The old shape paid
+ // struct→Value (≈100 k node allocs for a 10 k-item batch), a per-item
+ // DashMap get + full settings-Value clone, and then a SECOND
+ // serialization of the Value to bytes, on every bulk response.
+ let seq_disabled_anywhere = state.engine.index_settings.iter().any(|e| {
+ let s = e.value();
+ let as_bool = |val: &Value| {
+ val.as_bool()
+ .unwrap_or_else(|| val.as_str().map(|x| x == "true").unwrap_or(false))
+ };
+ s.pointer("/index/disable_sequence_numbers").map(as_bool).unwrap_or(false)
+ || s.get("index")
+ .and_then(|i| i.get("index.disable_sequence_numbers"))
+ .map(as_bool)
+ .unwrap_or(false)
+ || s.get("index.disable_sequence_numbers").map(as_bool).unwrap_or(false)
+ });
+ if !seq_disabled_anywhere {
+ let mut r = Json(&resp).into_response();
+ if all_backpressure {
+ *r.status_mut() = axum::http::StatusCode::TOO_MANY_REQUESTS;
+ r.headers_mut().insert(
+ axum::http::header::RETRY_AFTER,
+ axum::http::HeaderValue::from_static("1"),
+ );
+ }
+ return r;
+ }
+
// Post-process to apply per-index `disable_sequence_numbers` sentinel
// values (_seq_no=-2, _primary_term=0) in the response. Serialize to
// Value first so we can mutate the per-item fields without widening
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index 832c1c62a..7b8422b6e 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -1230,6 +1230,13 @@ pub async fn process_bulk_with_opts(
let exec_ms = t_exec.elapsed().as_millis() as u64;
let took_ms = started.elapsed().as_millis() as u64;
+ // THROWAWAY prof (XERJ_PROF): whole-request bulk attribution.
+ if std::env::var_os("XERJ_PROF").is_some() {
+ eprintln!(
+ "XERJ_PROF bulk total_ms={} lines_ms={} parse_ms={} group_ms={} exec_ms={} n_lines={}",
+ took_ms, lines_ms, parse_ms, group_ms, exec_ms, lines.len()
+ );
+ }
if took_ms >= 50 {
tracing::debug!(
total_ms = took_ms,
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 480a4baf0..d73d8ec24 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -1316,6 +1316,14 @@ impl Index {
let index_start = std::time::Instant::now();
let batch_len = docs.len();
+ // THROWAWAY prof (XERJ_PROF): per-phase attribution of the turbo path.
+ let prof = std::env::var_os("XERJ_PROF").is_some();
+ let mut p_wal_us = 0u128;
+ let mut p_parse_us = 0u128;
+ let mut p_evolve_us = 0u128;
+ let mut p_analyze_us = 0u128;
+ let mut p_insert_us = 0u128;
+ let p_t = std::time::Instant::now();
// Build WAL refs directly from bytes. We pass `Arc<Value::Null>`
// as the `source` parameter because `wal_append_batch` uses
@@ -1328,6 +1336,9 @@ impl Index {
.collect();
let seq_nos = self.store.wal_append_batch(&wal_refs)?;
+ if prof {
+ p_wal_us = p_t.elapsed().as_micros();
+ }
// Insert each doc EXACTLY ONCE into the engine FTS memtable,
// routed to the doc's OWN shard — bit-for-bit identical to the
@@ -1383,6 +1394,7 @@ impl Index {
// parse bursts never queue ahead of search/agg par_iters on the
// global rayon pool — see the read-under-write collapse notes on
// `ingest_pool()`.
+ let p_t = std::time::Instant::now();
let sources: Vec<Arc<Value>> = crate::ingest_pool().install(|| {
docs.par_iter()
.map(|(_, bytes)| {
@@ -1393,10 +1405,16 @@ impl Index {
})
.collect()
});
+ if prof {
+ p_parse_us = p_t.elapsed().as_micros();
+ }
- // 2. Dynamic mapping: evolve schema serially, in doc order.
- for source in &sources {
- self.evolve_schema_from_doc(source.as_ref()).await;
+ // 2. Dynamic mapping: evolve schema in doc order — batched (one
+ // schema read-lock for the whole batch, not one per doc).
+ let p_t = std::time::Instant::now();
+ self.evolve_schema_from_docs(&sources).await;
+ if prof {
+ p_evolve_us = p_t.elapsed().as_micros();
}
// 3. Parallel shard-partitioned insert. Partition the batch by
@@ -1421,6 +1439,7 @@ impl Index {
let analyzer = mem
.default_analyzer()
.expect("standard analyzer always present");
+ let p_t = std::time::Instant::now();
let analyzed: Vec<Vec<(String, Vec<xerj_fts::analyzer::Token>)>> =
crate::ingest_pool().install(|| {
sources
@@ -1430,6 +1449,10 @@ impl Index {
})
.collect()
});
+ if prof {
+ p_analyze_us = p_t.elapsed().as_micros();
+ }
+ let p_t = std::time::Instant::now();
let n_shards = mem.shard_count().max(1);
let mut buckets: Vec<Vec<usize>> = vec![Vec::new(); n_shards];
@@ -1464,6 +1487,17 @@ impl Index {
});
});
});
+ if prof {
+ p_insert_us = p_t.elapsed().as_micros();
+ }
+ }
+
+ if prof {
+ eprintln!(
+ "XERJ_PROF turbo-batch docs={} wal_us={} parse_us={} evolve_us={} analyze_us={} insert_us={} total_us={}",
+ batch_len, p_wal_us, p_parse_us, p_evolve_us, p_analyze_us, p_insert_us,
+ index_start.elapsed().as_micros()
+ );
}
// 4. One batch-level version stamp, assigned in request order.
@@ -6246,6 +6280,60 @@ impl Index {
// ── Helpers ───────────────────────────────────────────────────────────────
+ /// Batch variant of [`evolve_schema_from_doc`]: one schema read-lock
+ /// acquisition for the WHOLE batch instead of one per doc. The per-doc
+ /// variant paid a tokio `RwLock::read().await` per document — measured
+ /// ~22 ms per 10 k-doc bulk batch (≈ a quarter of the whole turbo exec
+ /// phase) even though the schema stops evolving after the first docs.
+ ///
+ /// Semantics are identical to calling `evolve_schema_from_doc` serially
+ /// in doc order: new fields are collected in FIRST-SEEN order (so the
+ /// first doc that introduces a field decides its inferred type) and
+ /// added under a single write lock, which re-checks `has_field` exactly
+ /// like the per-doc slow path does.
+ async fn evolve_schema_from_docs(&self, sources: &[Arc<Value>]) {
+ let new_fields: Vec<(String, FieldConfig)> = {
+ let schema = self.schema.read().await;
+ if !matches!(schema.dynamic, xerj_common::schema::DynamicMapping::Dynamic) {
+ return;
+ }
+ let mut out: Vec<(String, FieldConfig)> = Vec::new();
+ for source in sources {
+ let Some(obj) = source.as_object() else {
+ continue;
+ };
+ for (key, val) in obj {
+ if !schema.schema.has_field(key)
+ && !out.iter().any(|(k, _)| k == key)
+ {
+ let ft = infer_field_type(val);
+ out.push((key.clone(), FieldConfig::new(key.clone(), ft)));
+ }
+ }
+ }
+ out
+ };
+
+ if new_fields.is_empty() {
+ return;
+ }
+
+ let mut schema = self.schema.write().await;
+ if !matches!(schema.dynamic, xerj_common::schema::DynamicMapping::Dynamic) {
+ return;
+ }
+ let mut schema_changed = false;
+ for (_, fc) in new_fields {
+ if !schema.schema.has_field(&fc.name) {
+ let _ = schema.schema.add_field(fc);
+ schema_changed = true;
+ }
+ }
+ if schema_changed {
+ let _ = self.save_schema(&schema).await;
+ }
+ }
+
async fn evolve_schema_from_doc(&self, source: &Value) {
let obj = match source.as_object() {
Some(o) => o,