AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(ingest): validate raw documents before WAL publication
bugerb2891887b765
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex ef94050c5..e45f41c80 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -6144,6 +6144,7 @@ dependencies = ["fst","lz4_flex","memmap2",+ "rayon","roaring","rustc-hash","serde",diff --git a/engine/crates/xerj-api/src/error.rs b/engine/crates/xerj-api/src/error.rsindex 9a87b94bc..cd02ff37f 100644--- a/engine/crates/xerj-api/src/error.rs+++ b/engine/crates/xerj-api/src/error.rs@@ -294,6 +294,7 @@ fn xerj_error_type(e: &XerjError) -> String {XerjError::StorageError { .. } | XerjError::WalError { .. } => "store_exception",// ── Serialization ─────────────────────────────────────────────────XerjError::SerializationError { .. } => "json_parse_exception",+ XerjError::InvalidDocumentJson { .. } => "json_parse_exception",// ── Configuration ─────────────────────────────────────────────────XerjError::ConfigError { .. } => "action_request_validation_exception",// ── Auth ──────────────────────────────────────────────────────────@@ -373,3 +374,24 @@ pub fn native_error(};(http_status, axum::Json(body))}++#[cfg(test)]+mod raw_document_error_tests {+ use super::*;++ #[test]+ fn invalid_document_json_renders_registered_actionable_400() {+ let reason = "invalid raw JSON for document [row-17] at one-based batch position 17: \+ trailing characters at line 1 column 9. Fix: replace that document body \+ with exactly one complete UTF-8 JSON value, then retry the batch. Related \+ help: `xerj index --help` and https://xerj.org/llms.txt";+ let (status, body) = ApiError::new(XerjError::invalid_document_json(reason)).into_parts();+ assert_eq!(status, StatusCode::BAD_REQUEST);+ assert_eq!(body.error.error_type, "json_parse_exception");+ assert_eq!(body.error.root_cause[0].error_type, "json_parse_exception");+ assert!(body.error.reason.contains("document [row-17]"));+ assert!(body.error.reason.contains("batch position 17"));+ assert!(body.error.reason.contains("Fix:"));+ assert!(body.error.reason.contains("Related help:"));+ }+}diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 88d05ab68..4acd461e8 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -12373,6 +12373,84 @@ async fn process_bulk_body(}}+#[cfg(test)]+mod malformed_bulk_route_tests {+ use super::*;+ use axum::{+ body::{to_bytes, Body},+ http::{Request, StatusCode},+ };+ use tower::ServiceExt;+ use xerj_common::{+ config::{Config, WalSync},+ metrics::Metrics,+ };+ use xerj_engine::Engine;++ fn test_state() -> AppState {+ let dir = tempfile::tempdir().expect("tempdir").keep();+ let mut config = Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = WalSync::Async;+ let metrics = Metrics::new().expect("metrics");+ let engine = Engine::new(config.clone()).expect("engine");+ AppState::new(config, engine, metrics)+ }++ #[tokio::test]+ async fn malformed_bulk_source_returns_actionable_public_item_error() {+ let body = concat!(+ "{\"index\":{\"_index\":\"route-errors\",\"_id\":\"valid-doc\"}}\n",+ "{\"title\":\"valid\"}\n",+ "{\"index\":{\"_index\":\"route-errors\",\"_id\":\"broken-doc\"}}\n",+ "{\"title\":\"unterminated}\n",+ );+ let response = crate::router::build_es_compat_router(test_state())+ .oneshot(+ Request::post("/_bulk")+ .header("content-type", "application/x-ndjson")+ .body(Body::from(body))+ .expect("request"),+ )+ .await+ .expect("route response");++ // ES bulk reports per-item parse failures inside a successful HTTP+ // response; callers must inspect `errors` and each item's status.+ assert_eq!(response.status(), StatusCode::OK);+ let bytes = to_bytes(response.into_body(), usize::MAX)+ .await+ .expect("response bytes");+ let response: Value = serde_json::from_slice(&bytes).expect("JSON response");+ assert_eq!(response["errors"], true);+ assert_eq!(response["items"][0]["index"]["status"], 201);++ let failed = &response["items"][1]["index"];+ assert_eq!(failed["_id"], "broken-doc");+ assert_eq!(failed["status"], 400);+ assert_eq!(+ failed["error"]["type"], "document_parsing_exception",+ "unexpected response: {response}"+ );+ let reason = failed["error"]["reason"]+ .as_str()+ .expect("error reason string");+ for expected in [+ "document ID \"broken-doc\"",+ "bulk item position 2",+ "parser error:",+ "parser location: line 1, column ",+ "Fix: replace this source line with exactly one complete UTF-8 JSON object.",+ "Related help: xerj index --help and https://xerj.org/llms.txt",+ ] {+ assert!(+ reason.contains(expected),+ "missing {expected:?} in public reason: {reason}"+ );+ }+ }+}+// ─────────────────────────────────────────────────────────────────────────────// Schema conversion helpers// ─────────────────────────────────────────────────────────────────────────────@@ -28928,16 +29006,14 @@ mod ml_datafeed_tests {19.0, 21.0, 20.0, 20.0, 22.0, 18.0, 21.0, 20.0, 19.0, 21.0, 20.0, 22.0, 96.0, 20.0,21.0, 19.0,];- let mut batch: Vec<(String, Value, std::sync::Arc<[u8]>)> = Vec::new();+ let mut batch: Vec<(String, Value)> = Vec::new();for (i, v) in cpu.iter().enumerate() {let ts = base + chrono::Duration::minutes(i as i64);let doc = json!({"@timestamp": ts.to_rfc3339_opts(chrono::SecondsFormat::Millis, true),"cpu": v,});- let bytes: std::sync::Arc<[u8]> =- std::sync::Arc::from(serde_json::to_vec(&doc).unwrap());- batch.push((format!("m{i:02}"), doc, bytes));+ batch.push((format!("m{i:02}"), doc));}idx.index_batch_turbo(batch, false, false).await.unwrap();@@ -29103,12 +29179,11 @@ mod reindex_keyset_tests {// Index N docs with distinct _ids into `src` via the batched turbo// path (one WAL append) so the test isn't dominated by per-doc fsyncs.let src = state.engine.get_or_create_index("src").expect("src index");- let mut batch: Vec<(String, Value, std::sync::Arc<[u8]>)> = Vec::with_capacity(n);+ let mut batch: Vec<(String, Value)> = Vec::with_capacity(n);for i in 0..n {let id = format!("doc-{i:06}");let v = json!({ "n": i });- let bytes: std::sync::Arc<[u8]> = std::sync::Arc::from(serde_json::to_vec(&v).unwrap());- batch.push((id, v, bytes));+ batch.push((id, v));}src.index_batch_turbo(batch, true, false).awaitdiff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rsindex c1603b237..ef64e91eb 100644--- a/engine/crates/xerj-api/src/native.rs+++ b/engine/crates/xerj-api/src/native.rs@@ -1773,19 +1773,14 @@ pub async fn turbo_ingest(let parallel = cfg.indexing.turbo_parallel;let fast_analyzer = cfg.indexing.turbo_fast_analyzer;- // Build `(id, source, source_bytes)` triples, auto-generating IDs- // where not supplied. The native API accepts already-parsed- // `Value` objects so we serialize once here (`to_vec`) to produce- // the WAL bytes — still one serialize total versus two in the- // pre-v14 path (one parse + one re-serialize).- let pairs: Vec<(String, Value, std::sync::Arc<[u8]>)> = docs+ // The engine accepts one parsed authority and serializes it exactly once+ // at the storage boundary; there is no second caller-provided byte+ // representation that could disagree with live indexing.+ let pairs: Vec<(String, Value)> = docs.into_iter().map(|doc| {let id = extract_or_gen_id(&doc);- let bytes: std::sync::Arc<[u8]> = serde_json::to_vec(&doc)- .map(std::sync::Arc::from)- .unwrap_or_else(|_| std::sync::Arc::from(&[][..]));- (id, doc, bytes)+ (id, doc)}).collect();diff --git a/engine/crates/xerj-common/src/error.rs b/engine/crates/xerj-common/src/error.rsindex b12554055..86d57fb6e 100644--- a/engine/crates/xerj-common/src/error.rs+++ b/engine/crates/xerj-common/src/error.rs@@ -55,6 +55,10 @@ pub enum XerjError {#[error("serialization error: {reason}")]SerializationError { reason: String },+ /// A user-supplied document body is not one complete JSON value.+ #[error("{reason}")]+ InvalidDocumentJson { reason: String },+// ── Configuration ──────────────────────────────────────────────────────/// The configuration file is invalid.#[error("config error: {reason}")]@@ -178,6 +182,12 @@ impl XerjError {}}+ pub fn invalid_document_json(reason: impl Into<String>) -> Self {+ Self::InvalidDocumentJson {+ reason: reason.into(),+ }+ }+pub fn config(reason: impl Into<String>) -> Self {Self::ConfigError {reason: reason.into(),@@ -252,6 +262,7 @@ impl XerjError {Self::IndexAlreadyExists { .. } | Self::VersionConflict { .. } => 409,Self::InvalidMapping { .. }| Self::InvalidQuery { .. }+ | Self::InvalidDocumentJson { .. }| Self::ConfigError { .. }| Self::ResultWindowTooLarge { .. } => 400,Self::IndexBlocked { .. } => 403,diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex bc6d7fdda..822b9f209 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -40,6 +40,23 @@ pub struct BulkItemResult {pub get_source: Option<Value>,}+fn malformed_document_reason(+ document_id: Option<&str>,+ item_index: usize,+ error: &serde_json::Error,+) -> String {+ let document_id = document_id.unwrap_or("<auto-generated>");+ format!(+ "invalid document JSON for document ID {document_id:?} at bulk item position {}: \+ parser error: {error}; parser location: line {}, column {}. \+ Fix: replace this source line with exactly one complete UTF-8 JSON object. \+ Related help: xerj index --help and https://xerj.org/llms.txt",+ item_index + 1,+ error.line(),+ error.column(),+ )+}+fn engine_error_http_status(error: &EngineError) -> u16 {match error {EngineError::Common(xerj_common::XerjError::IndexBlocked { block_type, .. })@@ -575,11 +592,13 @@ pub async fn process_bulk_with_opts(BulkItemResult {action: action_type.to_string(),index: target_index,- id: doc_id.unwrap_or_default(),+ id: doc_id.clone().unwrap_or_default(),status: 400,result: None,- error: Some(format!(- "failed to parse: invalid document JSON: {e}"+ error: Some(malformed_document_reason(+ doc_id.as_deref(),+ *item_index,+ &e,)),get_source: None,},@@ -630,10 +649,14 @@ pub async fn process_bulk_with_opts(BulkItemResult {action: action_type.to_string(),index: target_index,- id: doc_id.unwrap_or_default(),+ id: doc_id.clone().unwrap_or_default(),status: 400,result: None,- error: Some(format!("invalid document JSON: {e}")),+ error: Some(malformed_document_reason(+ doc_id.as_deref(),+ *item_index,+ &e,+ )),get_source: None,},*item_index,diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 0233b97b7..b1e39e01e 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -2368,7 +2368,7 @@ impl Index {/// failures are collected and returned rather than aborting the batch.pub async fn index_batch_turbo(&self,- docs: Vec<(String, Value, Arc<[u8]>)>,+ docs: Vec<(String, Value)>,parallel: bool,_fast_analyzer: bool,) -> Result<Vec<IndexResponse>> {@@ -2454,8 +2454,8 @@ impl Index {// Build the pipeline with a batch size equal to the full input so it// never auto-flushes mid-way through.let mut pipeline = TurboIngestPipeline::new(batch_len + 1, parallel);- for (id, source, source_bytes) in docs {- pipeline.push(id, source, source_bytes);+ for (id, source) in docs {+ pipeline.push(id, source, Arc::<[u8]>::from([]));}// Flush collects all IngestResult values (with pre-computed tokens).let results = pipeline.flush();@@ -2478,7 +2478,10 @@ impl Index {id: r.id,tokens: r.tokens,source,- source_bytes: r.source_bytes,+ // `copy_to` changed the parsed document. Force WAL+ // serialization from that value so live indexing and+ // crash replay cannot observe different sources.+ source_bytes: Arc::<[u8]>::from([]),}}).collect()@@ -2507,15 +2510,9 @@ impl Index {// segments where 150 k were expected, thrashing the flush// path and triggering back-pressure 429s. Rolled back.let t3 = std::time::Instant::now();- let wal_refs: Vec<(String, Arc<Value>, Arc<[u8]>)> = processed+ let wal_refs: Vec<(String, Arc<Value>)> = processed.iter()- .map(|r| {- (- r.id.clone(),- Arc::clone(&r.source),- Arc::clone(&r.source_bytes),- )- })+ .map(|r| (r.id.clone(), Arc::clone(&r.source))).collect();let wal_t = std::time::Instant::now();@@ -2557,6 +2554,11 @@ impl Index {// most one chunk, not the whole batch. `remove()` still precedes// each `insert_pretokenized_with_seq`, so overwrite semantics are// preserved regardless of the chunk boundary.+ let copy_schema = if has_copy_to {+ Some(self.schema.read().await)+ } else {+ None+ };{let mut base = 0usize;while base < batch_len {@@ -2565,12 +2567,25 @@ impl Index {for i in base..end {let ingest = &processed[i];mem.remove(&ingest.id);- mem.insert_pretokenized_with_seq(- seq_nos[i],- ingest.id.clone(),- Arc::clone(&ingest.source),- &ingest.tokens,- );+ if let Some(schema) = copy_schema.as_ref() {+ // copy_to created fields after the turbo pipeline+ // emitted its (currently empty) token list. Analyze+ // the transformed source so the target is searchable+ // immediately, not only after flush/merge.+ mem.insert(+ ingest.id.clone(),+ &ingest.source,+ &schema.schema,+ seq_nos[i],+ );+ } else {+ mem.insert_pretokenized_with_seq(+ seq_nos[i],+ ingest.id.clone(),+ Arc::clone(&ingest.source),+ &ingest.tokens,+ );+ }let version = self.doc_count.fetch_add(1, Ordering::Relaxed) + 1;@@ -2585,6 +2600,7 @@ impl Index {base = end;}}+ drop(copy_schema);let t4_dur = t4.elapsed();if batch_len >= 1000 {@@ -2686,17 +2702,10 @@ impl Index {/// M5.11 — ULTRA-turbo raw-bytes ingest.///- /// Takes already-serialized NDJSON document lines and pushes them- /// through the WAL + memtable without ever running- /// `serde_json::from_str` on the document body. The JSON parse is- /// deferred all the way to drain-for-flush time, where it runs on- /// a background task without contending with HTTP workers.- ///- /// Trade-off: schema auto-evolution and vector HNSW indexing do- /// NOT fire on this path — the parsed `Value` they need isn't- /// built. Callers must pre-register the index mapping and avoid- /// vector fields on this endpoint. Nginx-style log ingest is the- /// canonical use case.+ /// Takes already-serialized NDJSON document lines, validates and parses+ /// the complete batch once, then reuses those values for WAL, schema, and+ /// FTS work. Validation precedes every write-side action: malformed input+ /// cannot reserve sequence numbers, become replayable, or trigger a flush.pub async fn index_batch_turbo_raw(&self,docs: Vec<(String, Arc<[u8]>)>,@@ -2704,6 +2713,7 @@ impl Index {if docs.is_empty() {return Ok(Vec::new());}+ let batch_len = docs.len();if self.is_write_blocked().await {return Err(EngineError::Common(xerj_common::XerjError::index_blocked(self.name.as_str(),@@ -2713,7 +2723,16 @@ impl Index {// Process-wide admission: disk flood-stage block (item 3) + parent// memory circuit breaker (item 1). Fires before any per-index work.self.governor_write_gate()?;-+ let syntax_t = std::time::Instant::now();+ let validated = crate::ingest_pool()+ .install(|| xerj_storage::IndexStore::validate_raw_batch(docs))+ .map_err(|error| match error {+ xerj_storage::StorageError::RawBatchValidation { .. } => EngineError::Common(+ xerj_common::XerjError::invalid_document_json(error.to_string()),+ ),+ error => EngineError::Storage(error),+ })?;+ let syntax_elapsed = syntax_t.elapsed();let hard_block = self.flush_byte_threshold.saturating_mul(3);let soft_block = self.flush_byte_threshold.saturating_mul(2);{@@ -2748,8 +2767,17 @@ impl Index {}}+ let dom_t = std::time::Instant::now();+ let validated = crate::ingest_pool()+ .install(|| xerj_storage::IndexStore::parse_validated_raw_batch(validated))+ .map_err(|error| match error {+ xerj_storage::StorageError::RawBatchValidation { .. } => EngineError::Common(+ xerj_common::XerjError::invalid_document_json(error.to_string()),+ ),+ error => EngineError::Storage(error),+ })?;+ let raw_parse_elapsed = syntax_elapsed.saturating_add(dom_t.elapsed());let index_start = std::time::Instant::now();- let batch_len = docs.len();// THROWAWAY prof (XERJ_PROF): per-phase attribution of the turbo path.let prof = std::env::var_os("XERJ_PROF").is_some();let mut p_wal_us = 0u128;@@ -2759,18 +2787,43 @@ impl Index {let mut p_insert_us = 0u128;let p_t = std::time::Instant::now();- // Build WAL refs directly from bytes. We pass `Arc<Value::Null>`- // as the `source` parameter because `wal_append_batch` uses- // `source_bytes` when it's non-empty and never touches the- // Value tree on the fast path.- let null_val: Arc<Value> = Arc::new(Value::Null);- let wal_refs: Vec<(String, Arc<Value>, Arc<[u8]>)> = docs- .iter()- .map(|(id, bytes)| (id.clone(), Arc::clone(&null_val), Arc::clone(bytes)))- .collect();+ // The complete request was parsed before admission/back-pressure.+ // Record that preflight in the optional phase profiler.+ use rayon::prelude::*;+ if prof {+ p_parse_us = raw_parse_elapsed.as_micros();+ }+ let has_copy_to = {+ let schema = self.schema.read().await;+ schema_has_copy_to(&schema.schema)+ };let wal_t = std::time::Instant::now();- let seq_nos = self.store.wal_append_batch(&wal_refs)?;+ let (docs, sources, seq_nos) = if has_copy_to {+ let (docs, parsed) = validated.into_parts();+ let parsed = parsed.expect("parse_validated_raw_batch retains parsed values");+ let schema = self.schema.read().await;+ let sources: Vec<Arc<Value>> = parsed+ .iter()+ .map(|source| Arc::new(apply_copy_to(source, &schema.schema)))+ .collect();+ drop(schema);+ let wal_docs: Vec<(String, Arc<Value>)> = docs+ .iter()+ .zip(&sources)+ .map(|((id, _), source)| (id.clone(), Arc::clone(source)))+ .collect();+ let seq_nos = self.store.wal_append_batch(&wal_docs)?;+ (docs, sources, seq_nos)+ } else {+ let seq_nos = self.store.wal_append_batch_raw(&validated)?;+ let (docs, parsed) = validated.into_parts();+ (+ docs,+ parsed.expect("parse_validated_raw_batch retains parsed values"),+ seq_nos,+ )+ };// RC4 W4 item 2: WAL durability-write latency.if let Some(m) = crate::engine_metrics() {m.observe_wal_write(wal_t.elapsed().as_secs_f64());@@ -2778,7 +2831,6 @@ impl Index {if prof {p_wal_us = p_t.elapsed().as_micros();}-// Insert each doc EXACTLY ONCE into the engine FTS memtable,// routed to the doc's OWN shard — bit-for-bit identical to the// per-doc `index_document` path.@@ -2822,32 +2874,6 @@ impl Index {// with its own `seq_nos[i]`; (b) `version` = prior doc_count +// position + 1, assigned in doc order via one batch-level// `fetch_add`; (c) response order matches request order.- use rayon::prelude::*;-- // 1. Parse in parallel — each doc exactly once. Arc-wrapped so- // the memtable entry can share the allocation (no deep clone at- // insert time — pre-fix `insert` did `Arc::new(source.clone())`- // per doc under the shard write lock).- //- // Runs on the dedicated ingest pool (`crate::ingest_pool`) so bulk- // parse bursts never queue ahead of search/agg par_iters on the- // global rayon pool — see the read-under-write collapse notes on- // `ingest_pool()`.- let p_t = std::time::Instant::now();- let sources: Vec<Arc<Value>> = crate::ingest_pool().install(|| {- docs.par_iter()- .map(|(_, bytes)| {- Arc::new(- serde_json::from_slice::<Value>(bytes)- .unwrap_or_else(|_| Value::Object(serde_json::Map::new())),- )- })- .collect()- });- if prof {- p_parse_us = p_t.elapsed().as_micros();- }-// 2. Dynamic mapping: evolve schema in doc order — batched (one// schema read-lock for the whole batch, not one per doc).let p_t = std::time::Instant::now();@@ -2995,6 +3021,16 @@ impl Index {if docs.is_empty() {return Ok(0);}+ let batch_len = docs.len();+ self.governor_write_gate()?;+ let validated = crate::ingest_pool()+ .install(|| xerj_storage::IndexStore::validate_raw_batch(docs))+ .map_err(|error| match error {+ xerj_storage::StorageError::RawBatchValidation { .. } => EngineError::Common(+ xerj_common::XerjError::invalid_document_json(error.to_string()),+ ),+ error => EngineError::Storage(error),+ })?;let hard_block = self.flush_byte_threshold.saturating_mul(3);let soft_block = self.flush_byte_threshold.saturating_mul(2);@@ -3045,77 +3081,133 @@ impl Index {}let index_start = std::time::Instant::now();- let batch_len = docs.len();-- // Fast-path WAL append: no Arc<Value> wrapper, no per-batch Vec- // allocation. `wal_append_batch_raw` borrows docs directly.let wal_t = std::time::Instant::now();- let seq_nos = self.store.wal_append_batch_raw(&docs)?;+ // Preserve copy_to semantics on the synchronous CLI path too. The+ // common no-copy_to case keeps the sealed raw-byte WAL fast path.+ // For the rare mapped case, materialize once, transform the sole+ // source authority, and retain transformed bytes for the raw+ // memtable so live GET, flush, and replay all observe the same value.+ let has_copy_to = {+ let schema = self.schema.try_read().map_err(|_| {+ EngineError::Common(xerj_common::XerjError::resource_exhausted(+ "schema is being updated during synchronous ingest; retry the batch",+ ))+ })?;+ schema_has_copy_to(&schema.schema)+ };+ let (docs, seq_nos, copy_schema) = if has_copy_to {+ let validated = crate::ingest_pool()+ .install(|| xerj_storage::IndexStore::parse_validated_raw_batch(validated))+ .map_err(EngineError::Storage)?;+ let (raw_docs, parsed) = validated.into_parts();+ let parsed = parsed.expect("parse_validated_raw_batch retains parsed values");+ let schema = self.schema.try_read().map_err(|_| {+ EngineError::Common(xerj_common::XerjError::resource_exhausted(+ "schema is being updated during synchronous ingest; retry the batch",+ ))+ })?;+ let copy_schema = schema.schema.clone();+ let transformed: Vec<Arc<Value>> = parsed+ .iter()+ .map(|source| Arc::new(apply_copy_to(source, ©_schema)))+ .collect();+ drop(schema);+ let wal_docs: Vec<(String, Arc<Value>)> = raw_docs+ .iter()+ .zip(&transformed)+ .map(|((id, _), source)| (id.clone(), Arc::clone(source)))+ .collect();+ let seq_nos = self.store.wal_append_batch(&wal_docs)?;+ let docs = raw_docs+ .into_iter()+ .zip(transformed)+ .map(|((id, _), source)| {+ serde_json::to_vec(source.as_ref())+ .map(|bytes| (id, Arc::<[u8]>::from(bytes), Some(source)))+ .map_err(xerj_storage::StorageError::from)+ })+ .collect::<std::result::Result<Vec<_>, _>>()?;+ (docs, seq_nos, Some(copy_schema))+ } else {+ let seq_nos = self.store.wal_append_batch_raw(&validated)?;+ let (docs, parsed) = validated.into_parts();+ debug_assert!(parsed.is_none());+ (+ docs.into_iter()+ .map(|(id, bytes)| (id, bytes, None))+ .collect(),+ seq_nos,+ None,+ )+ };// RC4 W4 item 2: WAL durability-write latency.if let Some(m) = crate::engine_metrics() {m.observe_wal_write(wal_t.elapsed().as_secs_f64());}- // Use the instance method so routing matches the actual configured- // shard count. The previous `Self::shard_for` was hardcoded to a- // 16-shard mask and panicked on machines configured for fewer.- let shard_idx = self.memtable.shard_for_dynamic(&docs[0].0);- // Consume `docs` by value — move each (String, Arc<[u8]>) into- // the memtable without cloning. The CLI bulk path generates- // unique doc_ids so `insert_raw_bytes_fresh` is safe (no prior- // entry with this id); we skip the `remove()` HashMap miss- // lookup that the generic `insert_raw_bytes_with_seq` preceded.- //- // READ-UNDER-WRITE stall reduction: the insert loop is CHUNKED so- // the shard write lock is released between chunks. Previously a- // whole bulk batch (mixval: 10 000 docs) was routed to ONE shard and- // the write lock was held for the ENTIRE insert (~10-30 ms), so a- // concurrent search that touched that shard (`doc_ids_bounded`,- // `all_docs_with_sources_arc`, `terms_counts_columnar` all take- // `s.read()` on every shard) blocked for the full batch. Chunking- // bounds a reader's *lock* stall on a writing shard to ONE chunk- // (~0.5-1.5 ms). NOTE: measured in isolation this removes the- // shard-lock component of the read tail, but the mixed- // read-under-write p99 is dominated by CPU/scheduler contention with- // the full-speed ingest+flush+merge (already deprioritised via the- // nice-pool ladder), which this does not address — see commit body.- // The lock churn is trivial (batch_len/CHUNK acquisitions, e.g. 20- // for a 10 k batch) — nothing like the per-DOC locking that once- // caused the 4× ingest regression. Correctness is unchanged:- // seq_nos are pre-assigned, inserts stay in seq order, and a flush- // drain that slips between chunks is safe (docs are independent and- // `take_memtable_for_flush` re-sorts by seq_no).- {- let mut docs_iter = docs.into_iter();- let mut i = 0usize;- while i < batch_len {- let end = (i + MEMTABLE_INSERT_CHUNK).min(batch_len);- self.memtable.with_shard_mut(shard_idx, |mem| {- for &seq_no in &seq_nos[i..end] {- let (id, bytes) = docs_iter- .next()- .expect("docs_iter yields exactly batch_len items");- mem.insert_raw_bytes_fresh(seq_no, id, bytes);+ // Partition by each document's own shard. The old sync path routed the+ // entire batch to docs[0]'s shard, while GET/search route by each ID;+ // every sibling ID hashing elsewhere was therefore invisible until+ // flush. Preserve request order within each shard so duplicate IDs are+ // deterministic last-write-wins.+ let n_shards = self.memtable.shard_count().max(1);+ let mut buckets: Vec<Vec<(u64, String, Arc<[u8]>, Option<Arc<Value>>)>> =+ (0..n_shards).map(|_| Vec::new()).collect();+ for (seq_no, (id, bytes, source)) in seq_nos.iter().copied().zip(docs) {+ let shard = self.memtable.shard_for_dynamic(&id);+ buckets[shard].push((seq_no, id, bytes, source));+ }+ let touched_shards: Vec<usize> = crate::ingest_pool().install(|| {+ use rayon::prelude::*;+ buckets+ .into_par_iter()+ .enumerate()+ .filter_map(|(shard, bucket)| {+ if bucket.is_empty() {+ return None;}- });- i = end;- }- }+ for chunk in bucket.chunks(MEMTABLE_INSERT_CHUNK) {+ self.memtable.with_shard_mut(shard, |mem| {+ for (seq_no, id, bytes, source) in chunk {+ // `insert_raw_bytes_fresh` requires absence.+ // Remove unconditionally so an overwrite from+ // an earlier batch and a duplicate within this+ // batch have identical last-write-wins+ // semantics.+ mem.remove(id);+ if let (Some(source), Some(schema)) =+ (source.as_ref(), copy_schema.as_ref())+ {+ mem.insert(id.clone(), source, schema, *seq_no);+ } else {+ mem.insert_raw_bytes_fresh(+ *seq_no,+ id.clone(),+ Arc::clone(bytes),+ );+ }+ }+ });+ }+ Some(shard)+ })+ .collect()+ });// Single batch-level atomic instead of one per doc. At 1.7 M/s// × 10 k batch = 170 batches/s this cuts ~17 M atomic ops/s of// cache-line bouncing on `doc_count`.self.doc_count.fetch_add(batch_len as u64, Ordering::Relaxed);- // Cheap per-shard threshold check — one read-lock, one shard.- let n_shards = self.memtable.shard_count().max(1);- let per_shard_doc_t =- staggered_per_shard_threshold(self.flush_doc_threshold, shard_idx, n_shards);- let per_shard_byte_t =- staggered_per_shard_threshold(self.flush_byte_threshold, shard_idx, n_shards);- let (sd_docs, sd_bytes) = self.memtable.shard_load(shard_idx);- if sd_docs >= per_shard_doc_t || sd_bytes >= per_shard_byte_t {- self.try_spawn_sync_flush(shard_idx);+ for shard in touched_shards {+ let per_shard_doc_t =+ staggered_per_shard_threshold(self.flush_doc_threshold, shard, n_shards);+ let per_shard_byte_t =+ staggered_per_shard_threshold(self.flush_byte_threshold, shard, n_shards);+ let (sd_docs, sd_bytes) = self.memtable.shard_load(shard);+ if sd_docs >= per_shard_doc_t || sd_bytes >= per_shard_byte_t {+ self.try_spawn_sync_flush(shard);+ }}let elapsed_ms = index_start.elapsed().as_millis() as u64;@@ -28134,4 +28226,552 @@ mod write_publication_integration_tests {"successful delete invalidates cached responses exactly once");}++ fn raw_test_docs(payloads: &[(&str, &[u8])]) -> Vec<(String, Arc<[u8]>)> {+ payloads+ .iter()+ .map(|(id, bytes)| ((*id).to_owned(), Arc::<[u8]>::from(*bytes)))+ .collect()+ }++ fn engine_wal_bytes(idx: &Index) -> Vec<(PathBuf, Vec<u8>)> {+ let mut files: Vec<(PathBuf, Vec<u8>)> = std::fs::read_dir(idx.store.wal_dir())+ .unwrap()+ .map(|entry| entry.unwrap().path())+ .filter(|path| path.is_file())+ .map(|path| {+ (+ path.file_name().unwrap().into(),+ std::fs::read(&path).unwrap(),+ )+ })+ .collect();+ files.sort_by(|a, b| a.0.cmp(&b.0));+ files+ }++ #[tokio::test]+ async fn sync_raw_invalid_json_changes_no_engine_or_durable_state() {+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ engine+ .create_index("sync-raw-invalid", Schema::empty())+ .unwrap();+ let idx = engine.get_index("sync-raw-invalid").unwrap();+ idx.index_document(Some("seed".to_owned()), json!({"existing": true}))+ .await+ .unwrap();++ let invalid: &[&[u8]] = &[+ br#"{"truncated":"#,+ br#"{"valid":true} trailing"#,+ &[0xff, 0xfe],+ b" \n\t",+ ];+ for (case, bad) in invalid.iter().enumerate() {+ for bad_at in 0..3 {+ let mut docs = raw_test_docs(&[+ ("first", br#"{"field":1}"#),+ ("middle", br#"{"field":2}"#),+ ("last", br#"{"field":3}"#),+ ]);+ docs[bad_at].1 = Arc::<[u8]>::from(*bad);+ let ids: Vec<_> = docs.iter().map(|(id, _)| id.clone()).collect();+ let before_seq = idx.store.current_seq_no();+ let before_wal = engine_wal_bytes(&idx);+ let before_vm = idx.store.version_map.len();+ let before_mem = idx.memtable.size_bytes();+ let before_count = idx.doc_count.load(Ordering::Acquire);+ let before_dataset = idx.dataset_version.load(Ordering::Acquire);+ let before_metric = idx.metric_index_count.load(Ordering::Acquire);+ let before_schema = idx.schema.read().await.schema.clone();++ let error = idx.index_batch_sync_raw(docs).unwrap_err();+ let rendered = error.to_string();+ assert!(+ rendered.contains(&format!("document [{}]", ids[bad_at]))+ && rendered.contains(&format!("batch position {}", bad_at + 1))+ && rendered.contains("Fix:")+ && rendered.contains("Related help:"),+ "case {case}, position {bad_at}: {error}"+ );+ assert_eq!(idx.store.current_seq_no(), before_seq);+ assert_eq!(engine_wal_bytes(&idx), before_wal);+ assert_eq!(idx.store.version_map.len(), before_vm);+ assert_eq!(idx.memtable.size_bytes(), before_mem);+ assert_eq!(idx.doc_count.load(Ordering::Acquire), before_count);+ assert_eq!(idx.dataset_version.load(Ordering::Acquire), before_dataset);+ assert_eq!(+ idx.metric_index_count.load(Ordering::Acquire),+ before_metric+ );+ assert_eq!(idx.schema.read().await.schema, before_schema);+ for id in ids {+ assert!(idx.get_document(&id).await.unwrap().is_none());+ }+ }+ }+ let too_deep = format!("{}0{}", "[".repeat(256), "]".repeat(256)).into_bytes();+ let before_seq = idx.store.current_seq_no();+ let error = idx+ .index_batch_sync_raw(vec![("too-deep".to_owned(), Arc::<[u8]>::from(too_deep))])+ .unwrap_err();+ assert!(error.to_string().contains("nesting exceeds"), "{error}");+ assert_eq!(idx.store.current_seq_no(), before_seq);++ drop(idx);+ drop(engine);+ let reopened = crate::Engine::new({+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config+ })+ .unwrap();+ let idx = reopened.get_index("sync-raw-invalid").unwrap();+ assert_eq!(+ idx.get_document("seed").await.unwrap().unwrap()["existing"],+ true+ );+ for id in ["first", "middle", "last"] {+ assert!(idx.get_document(id).await.unwrap().is_none());+ }+ }++ #[tokio::test]+ async fn async_raw_invalid_json_rejects_whole_batch_before_wal() {+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ engine+ .create_index("async-raw-invalid", Schema::empty())+ .unwrap();+ let idx = engine.get_index("async-raw-invalid").unwrap();+ idx.index_document(Some("seed".to_owned()), json!({"existing": true}))+ .await+ .unwrap();+ let invalid: &[&[u8]] = &[+ br#"{"truncated":"#,+ br#"{"valid":true} trailing"#,+ &[0xff, 0xfe],+ b" \n\t",+ ];+ for (case, bad) in invalid.iter().enumerate() {+ for bad_at in 0..3 {+ let mut docs = raw_test_docs(&[+ ("good-first", br#"{"field":1}"#),+ ("middle", br#"{"field":2}"#),+ ("good-last", br#"{"field":3}"#),+ ]);+ docs[bad_at].1 = Arc::<[u8]>::from(*bad);+ let before_seq = idx.store.current_seq_no();+ let before_wal = engine_wal_bytes(&idx);+ let before_schema = idx.schema.read().await.schema.clone();+ let before_vm = idx.store.version_map.len();+ let before_mem = idx.memtable.size_bytes();+ let before_count = idx.doc_count.load(Ordering::Acquire);+ let before_dataset = idx.dataset_version.load(Ordering::Acquire);+ let before_metric = idx.metric_index_count.load(Ordering::Acquire);++ let error = idx.index_batch_turbo_raw(docs).await.unwrap_err();+ let rendered = error.to_string();+ assert!(+ rendered.contains(&format!("batch position {}", bad_at + 1))+ && rendered.contains("Fix:")+ && rendered.contains("Related help:"),+ "case {case}, position {bad_at}: {error}"+ );+ assert_eq!(idx.store.current_seq_no(), before_seq);+ assert_eq!(engine_wal_bytes(&idx), before_wal);+ assert_eq!(idx.store.version_map.len(), before_vm);+ assert_eq!(idx.memtable.size_bytes(), before_mem);+ assert_eq!(idx.doc_count.load(Ordering::Acquire), before_count);+ assert_eq!(idx.dataset_version.load(Ordering::Acquire), before_dataset);+ assert_eq!(+ idx.metric_index_count.load(Ordering::Acquire),+ before_metric+ );+ assert_eq!(idx.schema.read().await.schema, before_schema);+ assert_eq!(+ idx.get_document("seed").await.unwrap().unwrap()["existing"],+ true+ );+ }+ }+ }++ #[tokio::test]+ async fn raw_ingest_apis_accept_complete_scalar_object_and_array_json() {+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ engine.create_index("raw-valid", Schema::empty()).unwrap();+ let idx = engine.get_index("raw-valid").unwrap();++ assert_eq!(+ idx.index_batch_sync_raw(raw_test_docs(&[+ ("sync-scalar", b"42"),+ ("sync-object", br#"{"answer":42}"#),+ ("sync-array", b"[1,2,3]"),+ ]))+ .unwrap(),+ 3+ );+ assert_eq!(+ idx.index_batch_turbo_raw(raw_test_docs(&[+ ("async-scalar", b"true"),+ ("async-object", br#"{"answer":43}"#),+ ("async-array", b"[4,5,6]"),+ ]))+ .await+ .unwrap()+ .len(),+ 3+ );+ assert_eq!(idx.doc_count.load(Ordering::Acquire), 6);+ assert_eq!(+ idx.index_batch_sync_raw(raw_test_docs(&[+ ("duplicate", br#"{"revision":1}"#),+ ("duplicate", br#"{"revision":2}"#),+ ]))+ .unwrap(),+ 2+ );+ assert_eq!(+ idx.get_document("duplicate").await.unwrap().unwrap(),+ json!({"revision": 2})+ );+ let expected = [+ ("sync-scalar", json!(42)),+ ("sync-object", json!({"answer": 42})),+ ("sync-array", json!([1, 2, 3])),+ ("async-scalar", json!(true)),+ ("async-object", json!({"answer": 43})),+ ("async-array", json!([4, 5, 6])),+ ("duplicate", json!({"revision": 2})),+ ];+ for (id, value) in &expected {+ assert_eq!(idx.get_document(id).await.unwrap().as_ref(), Some(value));+ }++ idx.flush().await.unwrap();+ for (id, value) in &expected {+ assert_eq!(idx.get_document(id).await.unwrap().as_ref(), Some(value));+ }+ drop(idx);+ drop(engine);++ let reopened = crate::Engine::new({+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config+ })+ .unwrap();+ let idx = reopened.get_index("raw-valid").unwrap();+ for (id, value) in &expected {+ assert_eq!(idx.get_document(id).await.unwrap().as_ref(), Some(value));+ }+ }++ #[tokio::test]+ async fn turbo_copy_to_wal_replays_the_transformed_source() {+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ let mut schema = Schema::empty();+ let mut title = FieldConfig::new("title", FieldType::Text);+ title.options.null_value = Some(Value::String("__copy_to__:all_text".to_owned()));+ schema.add_field(title).unwrap();+ schema+ .add_field(FieldConfig::new("all_text", FieldType::Text))+ .unwrap();+ engine.create_index("turbo-copy-to", schema).unwrap();+ let idx = engine.get_index("turbo-copy-to").unwrap();+ let source = json!({"title": "durable copied text"});++ idx.index_batch_turbo(vec![("doc".to_owned(), source)], false, false)+ .await+ .unwrap();+ let live = idx.get_document("doc").await.unwrap().unwrap();+ assert_eq!(live["all_text"], "durable copied text");+ let request = SearchRequest {+ query: QueryNode::Match {+ field: "all_text".to_owned(),+ query: "durable copied".to_owned(),+ operator: Default::default(),+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ },+ ..SearchRequest::default()+ };+ // The raw sync path intentionally defers lexical analysis to flush,+ // but the live memtable must already contain exactly one revision.+ assert_eq!(idx.memtable.doc_count(), 1);+ idx.flush().await.unwrap();+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);+ drop(idx);+ drop(engine);++ let reopened = crate::Engine::new({+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config+ })+ .unwrap();+ let replayed = reopened+ .get_index("turbo-copy-to")+ .unwrap()+ .get_document("doc")+ .await+ .unwrap()+ .unwrap();+ assert_eq!(replayed, live, "live and WAL replay sources diverged");+ assert_eq!(+ reopened+ .get_index("turbo-copy-to")+ .unwrap()+ .search(&request)+ .await+ .unwrap()+ .total+ .value,+ 1+ );+ }++ #[tokio::test]+ async fn raw_turbo_copy_to_is_searchable_and_durable_after_flush_restart() {+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ let mut schema = Schema::empty();+ let mut title = FieldConfig::new("title", FieldType::Text);+ title.options.null_value = Some(Value::String("__copy_to__:all_text".to_owned()));+ schema.add_field(title).unwrap();+ schema+ .add_field(FieldConfig::new("all_text", FieldType::Text))+ .unwrap();+ engine.create_index("raw-copy-to", schema).unwrap();+ let idx = engine.get_index("raw-copy-to").unwrap();+ idx.index_batch_turbo_raw(raw_test_docs(&[(+ "doc",+ br#"{"title":"raw durable copied text"}"#,+ )]))+ .await+ .unwrap();+ let expected = idx.get_document("doc").await.unwrap().unwrap();+ assert_eq!(expected["title"], "raw durable copied text");+ assert_eq!(expected["all_text"], "raw durable copied text");+ let request = SearchRequest {+ query: QueryNode::Match {+ field: "all_text".to_owned(),+ query: "raw durable".to_owned(),+ operator: Default::default(),+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ },+ ..SearchRequest::default()+ };+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);+ idx.flush().await.unwrap();+ assert_eq!(idx.get_document("doc").await.unwrap().unwrap(), expected);+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);+ drop(idx);+ drop(engine);++ let reopened = crate::Engine::new({+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config+ })+ .unwrap();+ let idx = reopened.get_index("raw-copy-to").unwrap();+ assert_eq!(idx.get_document("doc").await.unwrap().unwrap(), expected);+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);+ }++ #[tokio::test]+ async fn sync_raw_copy_to_is_searchable_and_durable_after_flush_restart() {+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ let mut schema = Schema::empty();+ let mut title = FieldConfig::new("title", FieldType::Text);+ title.options.null_value = Some(Value::String("__copy_to__:all_text".to_owned()));+ schema.add_field(title).unwrap();+ schema+ .add_field(FieldConfig::new("all_text", FieldType::Text))+ .unwrap();+ engine.create_index("sync-raw-copy-to", schema).unwrap();+ let idx = engine.get_index("sync-raw-copy-to").unwrap();+ idx.index_batch_sync_raw(raw_test_docs(&[(+ "doc",+ br#"{"title":"sync durable copied text"}"#,+ )]))+ .unwrap();+ let expected = idx.get_document("doc").await.unwrap().unwrap();+ assert_eq!(expected["title"], "sync durable copied text");+ assert_eq!(expected["all_text"], "sync durable copied text");+ let request = SearchRequest {+ query: QueryNode::Match {+ field: "all_text".to_owned(),+ query: "sync durable".to_owned(),+ operator: Default::default(),+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ },+ ..SearchRequest::default()+ };+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);+ idx.flush().await.unwrap();+ assert_eq!(idx.get_document("doc").await.unwrap().unwrap(), expected);+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);+ drop(idx);+ drop(engine);++ let reopened = crate::Engine::new({+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config+ })+ .unwrap();+ let idx = reopened.get_index("sync-raw-copy-to").unwrap();+ assert_eq!(idx.get_document("doc").await.unwrap().unwrap(), expected);+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);+ }++ #[tokio::test]+ async fn sync_raw_cross_batch_overwrite_is_single_and_durable() {+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ engine+ .create_index("sync-raw-overwrite", Schema::empty())+ .unwrap();+ let idx = engine.get_index("sync-raw-overwrite").unwrap();+ idx.index_batch_sync_raw(raw_test_docs(&[(+ "same-id",+ br#"{"revision":1,"tag":"obsolete"}"#,+ )]))+ .unwrap();+ idx.flush().await.unwrap();+ idx.index_batch_sync_raw(raw_test_docs(&[(+ "same-id",+ br#"{"revision":2,"tag":"current"}"#,+ )]))+ .unwrap();+ let expected = json!({"revision": 2, "tag": "current"});+ assert_eq!(+ idx.get_document("same-id").await.unwrap().as_ref(),+ Some(&expected)+ );+ let request = SearchRequest {+ query: QueryNode::Match {+ field: "tag".to_owned(),+ query: "current".to_owned(),+ operator: Default::default(),+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ },+ ..SearchRequest::default()+ };+ let obsolete_request = SearchRequest {+ query: QueryNode::Match {+ field: "tag".to_owned(),+ query: "obsolete".to_owned(),+ operator: Default::default(),+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ },+ ..SearchRequest::default()+ };+ let aggregate_request = SearchRequest {+ query: QueryNode::MatchAll,+ size: 0,+ aggs: Some(json!({+ "tags": {"terms": {"field": "tag"}},+ "revision_cardinality": {"cardinality": {"field": "revision"}},+ "max_revision": {"max": {"field": "revision"}}+ })),+ ..SearchRequest::default()+ };+ assert_eq!(+ idx.memtable.doc_count(),+ 1,+ "cross-call overwrite must leave one live memtable entry"+ );+ // Ordinary sync raw ingest defers lexical analysis until flush. The+ // disk-old/memory-new overlap must nevertheless hide stale postings;+ // exact source, logical count, and aggregations use revision 2.+ assert_eq!(idx.search(&obsolete_request).await.unwrap().total.value, 0);+ let aggregate = idx.search(&aggregate_request).await.unwrap();+ assert_eq!(aggregate.total.value, 1);+ let aggs = aggregate.aggs.unwrap();+ assert_eq!(aggs["revision_cardinality"]["value"], 1);+ assert_eq!(aggs["max_revision"]["value"], 2.0);+ assert_eq!(aggs["tags"]["buckets"].as_array().unwrap().len(), 1);+ assert_eq!(aggs["tags"]["buckets"][0]["key"], "current");+ drop(idx);+ drop(engine);++ // Revision 2 exists only in WAL/memory here. Restart must replay it+ // over the durable revision-1 segment without exposing stale terms or+ // double-counting the logical document.+ let reopened = crate::Engine::new({+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config+ })+ .unwrap();+ let idx = reopened.get_index("sync-raw-overwrite").unwrap();+ assert_eq!(+ idx.get_document("same-id").await.unwrap().as_ref(),+ Some(&expected)+ );+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);+ assert_eq!(idx.search(&obsolete_request).await.unwrap().total.value, 0);+ let aggregate = idx.search(&aggregate_request).await.unwrap();+ assert_eq!(aggregate.total.value, 1);+ assert_eq!(aggregate.aggs.unwrap()["max_revision"]["value"], 2.0);+ idx.flush().await.unwrap();+ drop(idx);+ drop(reopened);++ // The same invariants must survive publication of revision 2 and a+ // second restart.+ let reopened_after_second_flush = crate::Engine::new({+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config+ })+ .unwrap();+ let idx = reopened_after_second_flush+ .get_index("sync-raw-overwrite")+ .unwrap();+ assert_eq!(+ idx.get_document("same-id").await.unwrap().as_ref(),+ Some(&expected)+ );+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);+ assert_eq!(idx.search(&obsolete_request).await.unwrap().total.value, 0);+ let aggregate = idx.search(&aggregate_request).await.unwrap();+ assert_eq!(aggregate.total.value, 1);+ let aggs = aggregate.aggs.unwrap();+ assert_eq!(aggs["revision_cardinality"]["value"], 1);+ assert_eq!(aggs["max_revision"]["value"], 2.0);+ assert_eq!(aggs["tags"]["buckets"].as_array().unwrap().len(), 1);+ assert_eq!(aggs["tags"]["buckets"][0]["key"], "current");+ }}diff --git a/engine/crates/xerj-engine/tests/perf_benchmark.rs b/engine/crates/xerj-engine/tests/perf_benchmark.rsindex 68ec30935..6476fe47a 100644--- a/engine/crates/xerj-engine/tests/perf_benchmark.rs+++ b/engine/crates/xerj-engine/tests/perf_benchmark.rs@@ -622,12 +622,11 @@ async fn perf_turbo_indexing_throughput() {let idx_turbo = engine_turbo.get_index("turbo_bench").unwrap();let turbo_start = Instant::now();- let mut batch: Vec<(String, serde_json::Value, std::sync::Arc<[u8]>)> =- Vec::with_capacity(TURBO_BATCH);+ let mut batch: Vec<(String, serde_json::Value)> = Vec::with_capacity(TURBO_BATCH);for i in 0..N {- let empty_bytes: std::sync::Arc<[u8]> = std::sync::Arc::from(&[][..]);- batch.push((format!("doc{i}"), make_doc(i), empty_bytes));+ let source = make_doc(i);+ batch.push((format!("doc{i}"), source));if batch.len() >= TURBO_BATCH {let b = std::mem::replace(&mut batch, Vec::with_capacity(TURBO_BATCH));diff --git a/engine/crates/xerj-storage/Cargo.toml b/engine/crates/xerj-storage/Cargo.tomlindex b35d8f0ae..4acb526ec 100644--- a/engine/crates/xerj-storage/Cargo.toml+++ b/engine/crates/xerj-storage/Cargo.toml@@ -15,6 +15,7 @@ tokio = { workspace = true }# Serializationserde = { workspace = true }serde_json = { workspace = true }+rayon = { workspace = true }# Binary I/Obytes = { workspace = true }diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rsindex 63c1f04c8..5c32e345a 100644--- a/engine/crates/xerj-storage/src/index_store.rs+++ b/engine/crates/xerj-storage/src/index_store.rs@@ -310,6 +310,68 @@ pub struct IndexStoreConfig {pub num_wal_shards: usize,}+/// A raw JSON batch that has been completely validated before publication.+///+/// Fields are private so callers cannot bypass validation and hand malformed+/// bytes to the WAL writer. The parsed form is retained only when requested+/// by an engine path that will reuse it for schema/FTS work.+pub type RawJsonDoc = (String, std::sync::Arc<[u8]>);+pub type ParsedRawSources = Vec<std::sync::Arc<serde_json::Value>>;++#[derive(Debug)]+pub struct ValidatedRawBatch {+ docs: Vec<RawJsonDoc>,+ parsed: Option<ParsedRawSources>,+}++impl ValidatedRawBatch {+ pub fn docs(&self) -> &[RawJsonDoc] {+ &self.docs+ }++ pub fn parsed(&self) -> Option<&[std::sync::Arc<serde_json::Value>]> {+ self.parsed.as_deref()+ }++ pub fn into_parts(self) -> (Vec<RawJsonDoc>, Option<ParsedRawSources>) {+ (self.docs, self.parsed)+ }+}++const MAX_RAW_JSON_NESTING: usize = 128;++fn validate_raw_json_nesting(bytes: &[u8]) -> std::result::Result<(), String> {+ let mut depth = 0usize;+ let mut in_string = false;+ let mut escaped = false;+ for &byte in bytes {+ if in_string {+ if escaped {+ escaped = false;+ } else if byte == b'\\' {+ escaped = true;+ } else if byte == b'"' {+ in_string = false;+ }+ continue;+ }+ match byte {+ b'"' => in_string = true,+ b'{' | b'[' => {+ depth += 1;+ if depth > MAX_RAW_JSON_NESTING {+ return Err(format!(+ "JSON nesting exceeds the supported limit of {MAX_RAW_JSON_NESTING}"+ ));+ }+ }+ b'}' | b']' => depth = depth.saturating_sub(1),+ _ => {}+ }+ }+ Ok(())+}+impl Default for IndexStoreConfig {fn default() -> Self {Self {@@ -2889,10 +2951,72 @@ impl IndexStore {////// All on-disk framing is byte-identical to `wal_append_batch`; the/// two entries interleave freely in the WAL.- pub fn wal_append_batch_raw(- &self,- docs: &[(String, std::sync::Arc<[u8]>)],- ) -> Result<Vec<SeqNo>> {+ /// Validate raw JSON without materializing a DOM. The returned sealed+ /// value is the only input accepted by [`Self::wal_append_batch_raw`].+ pub fn validate_raw_batch(docs: Vec<RawJsonDoc>) -> Result<ValidatedRawBatch> {+ use rayon::prelude::*;+ docs.par_iter()+ .enumerate()+ .map(|(position, (doc_id, source_bytes))| {+ validate_raw_json_nesting(source_bytes).map_err(|reason| {+ StorageError::RawBatchValidation {+ doc_id: doc_id.clone(),+ position: position + 1,+ reason,+ }+ })?;+ serde_json::from_slice::<serde::de::IgnoredAny>(source_bytes)+ .map(|_| ())+ .map_err(|error| StorageError::RawBatchValidation {+ doc_id: doc_id.clone(),+ position: position + 1,+ reason: error.to_string(),+ })+ })+ .collect::<Result<()>>()?;+ Ok(ValidatedRawBatch { docs, parsed: None })+ }++ /// Validate and retain parsed values for callers that need both WAL bytes+ /// and a JSON DOM. This keeps the async raw ingest path to one parse.+ pub fn parse_raw_batch(docs: Vec<RawJsonDoc>) -> Result<ValidatedRawBatch> {+ use rayon::prelude::*;+ let parsed = docs+ .par_iter()+ .enumerate()+ .map(|(position, (doc_id, source_bytes))| {+ validate_raw_json_nesting(source_bytes).map_err(|reason| {+ StorageError::RawBatchValidation {+ doc_id: doc_id.clone(),+ position: position + 1,+ reason,+ }+ })?;+ serde_json::from_slice(source_bytes)+ .map(std::sync::Arc::new)+ .map_err(|error| StorageError::RawBatchValidation {+ doc_id: doc_id.clone(),+ position: position + 1,+ reason: error.to_string(),+ })+ })+ .collect::<Result<Vec<_>>>()?;+ Ok(ValidatedRawBatch {+ docs,+ parsed: Some(parsed),+ })+ }++ /// Materialize parsed values from a batch whose complete JSON syntax was+ /// already validated. This consumes the sealed proof and retains the+ /// original immutable bytes alongside the DOM.+ pub fn parse_validated_raw_batch(batch: ValidatedRawBatch) -> Result<ValidatedRawBatch> {+ debug_assert!(batch.parsed.is_none());+ Self::parse_raw_batch(batch.docs)+ }++ pub fn wal_append_batch_raw(&self, batch: &ValidatedRawBatch) -> Result<Vec<SeqNo>> {+ let docs = batch.docs();if docs.is_empty() {return Ok(Vec::new());}@@ -3010,17 +3134,15 @@ impl IndexStore {Ok(seq_nos)}- // The `docs` slice element is a WAL-batch tuple (doc_id, source JSON,- // pre-serialized bytes); the shape is part of the public batch API so we- // keep it inline rather than refactor the signature.- #[allow(clippy::type_complexity)]+ /// Append parsed documents to the WAL.+ ///+ /// The parsed `Value` is the sole authority and is serialized here.+ /// Accepting a second caller-provided byte representation made it possible+ /// for live indexing and replay to observe different documents; verifying+ /// that representation required a second full DOM parse on the hot path.pub fn wal_append_batch(&self,- docs: &[(- String,- std::sync::Arc<serde_json::Value>,- std::sync::Arc<[u8]>,- )],+ docs: &[(String, std::sync::Arc<serde_json::Value>)],) -> Result<Vec<SeqNo>> {if docs.is_empty() {return Ok(Vec::new());@@ -3062,25 +3184,13 @@ impl IndexStore {let mut seq_nos: Vec<SeqNo> = Vec::with_capacity(docs.len());// Estimate total frame size: per-doc overhead ~80 bytes + source- let est_total: usize = docs- .iter()- .map(|(id, _, sb)| id.len() + sb.len() + 100)- .sum();+ let est_total: usize = docs.iter().map(|(id, _source)| id.len() + 600).sum();let mut frames: Vec<u8> = Vec::with_capacity(est_total);- for (i, (doc_id, source, source_bytes)) in docs.iter().enumerate() {+ for (i, (doc_id, source)) in docs.iter().enumerate() {let seq_no = start_seq + i as u64;seq_nos.push(seq_no);- let bytes_to_write: std::borrow::Cow<[u8]> = if !source_bytes.is_empty() {- std::borrow::Cow::Borrowed(source_bytes.as_ref())- } else {- match serde_json::to_vec(source.as_ref()) {- Ok(v) => std::borrow::Cow::Owned(v),- Err(_) => std::borrow::Cow::Owned(b"null".to_vec()),- }- };-// Build JSON envelope directlylet payload_start = frames.len();// Reserve space for entry_len (4 bytes) + seq_no (8) + op (1)@@ -3107,7 +3217,7 @@ impl IndexStore {frames.extend_from_slice(doc_id.as_bytes());}frames.extend_from_slice(br#"","source":"#);- frames.extend_from_slice(&bytes_to_write);+ serde_json::to_writer(&mut frames, source.as_ref())?;frames.extend_from_slice(b"}}");let payload_end = frames.len();@@ -3198,7 +3308,7 @@ impl IndexStore {// lookups before flush resolve to `IN_MEMORY_SEGMENT_ID`.// This is the only per-doc side effect this method has// outside the WAL itself.- for (i, (doc_id, _source, _bytes)) in docs.iter().enumerate() {+ for (i, (doc_id, _source)) in docs.iter().enumerate() {let seq_no = seq_nos[i];self.version_map.set(doc_id, seq_no, IN_MEMORY_SEGMENT_ID, false);@@ -3225,6 +3335,120 @@ mod tests {.unwrap()}+ fn wal_bytes(store: &IndexStore) -> Vec<(PathBuf, Vec<u8>)> {+ let mut files: Vec<(PathBuf, Vec<u8>)> = Vec::new();+ for entry in std::fs::read_dir(store.wal_dir()).unwrap() {+ let path = entry.unwrap().path();+ if path.is_file() {+ files.push((+ path.file_name().unwrap().into(),+ std::fs::read(&path).unwrap(),+ ));+ }+ }+ files.sort_by(|a, b| a.0.cmp(&b.0));+ files+ }++ #[test]+ fn raw_wal_batch_rejects_entire_invalid_batch_without_side_effects() {+ let invalid_payloads: &[&[u8]] = &[+ br#"{"truncated":"#,+ br#"{"valid":true} trailing"#,+ &[0xff, 0xfe, 0xfd],+ ];++ for (case, invalid) in invalid_payloads.iter().enumerate() {+ for bad_at in 0..3 {+ let dir = tempfile::tempdir().unwrap();+ let store = open_test_store(dir.path());+ let before_seq = store.current_seq_no();+ let before_wal = wal_bytes(&store);+ let mut docs = vec![+ (+ "first".to_owned(),+ Arc::<[u8]>::from(br#"{"v":1}"#.as_slice()),+ ),+ (+ "middle".to_owned(),+ Arc::<[u8]>::from(br#"{"v":2}"#.as_slice()),+ ),+ (+ "last".to_owned(),+ Arc::<[u8]>::from(br#"{"v":3}"#.as_slice()),+ ),+ ];+ docs[bad_at].1 = Arc::<[u8]>::from(*invalid);++ let error = IndexStore::validate_raw_batch(docs.clone()).unwrap_err();+ assert!(+ matches!(+ error,+ StorageError::RawBatchValidation {+ ref doc_id,+ position,+ ..+ } if doc_id == &docs[bad_at].0 && position == bad_at + 1+ ),+ "case {case}, position {bad_at}: {error:?}"+ );+ let rendered = error.to_string();+ assert!(rendered.contains("Fix:"));+ assert!(rendered.contains("Related help:"));+ assert_eq!(store.current_seq_no(), before_seq);+ assert_eq!(wal_bytes(&store), before_wal);+ for (id, _) in &docs {+ assert!(store.version_map.get(id).is_none(), "{id} became visible");+ }++ drop(store);+ let reopened = open_test_store(dir.path());+ assert_eq!(reopened.current_seq_no(), before_seq);+ for (id, _) in &docs {+ assert!(+ reopened.version_map.get(id).is_none(),+ "{id} appeared after replay"+ );+ }+ }+ }+ }++ #[test]+ fn raw_wal_batch_accepts_complete_scalar_object_and_array_values() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_test_store(dir.path());+ let docs = vec![+ ("scalar".to_owned(), Arc::<[u8]>::from(b"42".as_slice())),+ (+ "object".to_owned(),+ Arc::<[u8]>::from(br#"{"answer":42}"#.as_slice()),+ ),+ (+ "array".to_owned(),+ Arc::<[u8]>::from(br#"[1,2,3]"#.as_slice()),+ ),+ ];+ let start_seq = store.current_seq_no();+ let validated = IndexStore::validate_raw_batch(docs.clone()).unwrap();+ assert_eq!(+ store.wal_append_batch_raw(&validated).unwrap(),+ vec![start_seq, start_seq + 1, start_seq + 2]+ );+ for (id, _) in &docs {+ assert!(store.version_map.get(id).is_some(), "{id} not visible");+ }+ drop(store);++ let reopened = open_test_store(dir.path());+ for (id, _) in &docs {+ assert!(+ reopened.version_map.get(id).is_some(),+ "{id} missing after replay"+ );+ }+ }+/// RC4 W1 #8 regression — bulk-during-flush + kill -9 must lose ZERO/// acked writes.///diff --git a/engine/crates/xerj-storage/src/lib.rs b/engine/crates/xerj-storage/src/lib.rsindex dab33f7e7..51d766ff5 100644--- a/engine/crates/xerj-storage/src/lib.rs+++ b/engine/crates/xerj-storage/src/lib.rs@@ -38,7 +38,7 @@ pub use backend::{FileMetadata, LocalFsBackend, S3Backend, StorageBackend};pub use cache::SegmentCache;pub use index_store::{DrainedMemtable, FsckReport, FsckSectionReport, FsckSegmentReport, IndexSnapshot, IndexStore,- IndexStoreConfig, StorageMode,+ IndexStoreConfig, RawJsonDoc, StorageMode, ValidatedRawBatch,};// `MergeExecutor` is intentionally NOT re-exported here: it is a// `#[doc(hidden)]`, storage-crate-only test helper with real footguns@@ -98,6 +98,17 @@ pub enum StorageError {#[error("Serialization error: {0}")]Serialization(#[from] serde_json::Error),+ #[error(+ "invalid raw JSON for document [{doc_id}] at one-based batch position {position}: \+ {reason}. Fix: replace that document body with exactly one complete UTF-8 JSON value, \+ then retry the batch. Related help: `xerj index --help` and https://xerj.org/llms.txt"+ )]+ RawBatchValidation {+ doc_id: String,+ position: usize,+ reason: String,+ },+#[error("Merge aborted: {0}")]MergeAborted(String),