Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(ingest): validate raw documents before WAL publication

bugerb2891887b765
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index ef94050c5..e45f41c80 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -6144,6 +6144,7 @@ dependencies = [
"fst",
"lz4_flex",
"memmap2",
+ "rayon",
"roaring",
"rustc-hash",
"serde",
diff --git a/engine/crates/xerj-api/src/error.rs b/engine/crates/xerj-api/src/error.rs
index 9a87b94bc..cd02ff37f 100644
--- a/engine/crates/xerj-api/src/error.rs
+++ b/engine/crates/xerj-api/src/error.rs
@@ -294,6 +294,7 @@ fn xerj_error_type(e: &XerjError) -> String {
XerjError::StorageError { .. } | XerjError::WalError { .. } => "store_exception",
// ── Serialization ─────────────────────────────────────────────────
XerjError::SerializationError { .. } => "json_parse_exception",
+ XerjError::InvalidDocumentJson { .. } => "json_parse_exception",
// ── Configuration ─────────────────────────────────────────────────
XerjError::ConfigError { .. } => "action_request_validation_exception",
// ── Auth ──────────────────────────────────────────────────────────
@@ -373,3 +374,24 @@ pub fn native_error(
};
(http_status, axum::Json(body))
}
+
+#[cfg(test)]
+mod raw_document_error_tests {
+ use super::*;
+
+ #[test]
+ fn invalid_document_json_renders_registered_actionable_400() {
+ let reason = "invalid raw JSON for document [row-17] at one-based batch position 17: \
+ trailing characters at line 1 column 9. Fix: replace that document body \
+ with exactly one complete UTF-8 JSON value, then retry the batch. Related \
+ help: `xerj index --help` and https://xerj.org/llms.txt";
+ let (status, body) = ApiError::new(XerjError::invalid_document_json(reason)).into_parts();
+ assert_eq!(status, StatusCode::BAD_REQUEST);
+ assert_eq!(body.error.error_type, "json_parse_exception");
+ assert_eq!(body.error.root_cause[0].error_type, "json_parse_exception");
+ assert!(body.error.reason.contains("document [row-17]"));
+ assert!(body.error.reason.contains("batch position 17"));
+ assert!(body.error.reason.contains("Fix:"));
+ assert!(body.error.reason.contains("Related help:"));
+ }
+}
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 88d05ab68..4acd461e8 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -12373,6 +12373,84 @@ async fn process_bulk_body(
}
}
+#[cfg(test)]
+mod malformed_bulk_route_tests {
+ use super::*;
+ use axum::{
+ body::{to_bytes, Body},
+ http::{Request, StatusCode},
+ };
+ use tower::ServiceExt;
+ use xerj_common::{
+ config::{Config, WalSync},
+ metrics::Metrics,
+ };
+ use xerj_engine::Engine;
+
+ fn test_state() -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = WalSync::Async;
+ let metrics = Metrics::new().expect("metrics");
+ let engine = Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ #[tokio::test]
+ async fn malformed_bulk_source_returns_actionable_public_item_error() {
+ let body = concat!(
+ "{\"index\":{\"_index\":\"route-errors\",\"_id\":\"valid-doc\"}}\n",
+ "{\"title\":\"valid\"}\n",
+ "{\"index\":{\"_index\":\"route-errors\",\"_id\":\"broken-doc\"}}\n",
+ "{\"title\":\"unterminated}\n",
+ );
+ let response = crate::router::build_es_compat_router(test_state())
+ .oneshot(
+ Request::post("/_bulk")
+ .header("content-type", "application/x-ndjson")
+ .body(Body::from(body))
+ .expect("request"),
+ )
+ .await
+ .expect("route response");
+
+ // ES bulk reports per-item parse failures inside a successful HTTP
+ // response; callers must inspect `errors` and each item's status.
+ assert_eq!(response.status(), StatusCode::OK);
+ let bytes = to_bytes(response.into_body(), usize::MAX)
+ .await
+ .expect("response bytes");
+ let response: Value = serde_json::from_slice(&bytes).expect("JSON response");
+ assert_eq!(response["errors"], true);
+ assert_eq!(response["items"][0]["index"]["status"], 201);
+
+ let failed = &response["items"][1]["index"];
+ assert_eq!(failed["_id"], "broken-doc");
+ assert_eq!(failed["status"], 400);
+ assert_eq!(
+ failed["error"]["type"], "document_parsing_exception",
+ "unexpected response: {response}"
+ );
+ let reason = failed["error"]["reason"]
+ .as_str()
+ .expect("error reason string");
+ for expected in [
+ "document ID \"broken-doc\"",
+ "bulk item position 2",
+ "parser error:",
+ "parser location: line 1, column ",
+ "Fix: replace this source line with exactly one complete UTF-8 JSON object.",
+ "Related help: xerj index --help and https://xerj.org/llms.txt",
+ ] {
+ assert!(
+ reason.contains(expected),
+ "missing {expected:?} in public reason: {reason}"
+ );
+ }
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Schema conversion helpers
// ─────────────────────────────────────────────────────────────────────────────
@@ -28928,16 +29006,14 @@ mod ml_datafeed_tests {
19.0, 21.0, 20.0, 20.0, 22.0, 18.0, 21.0, 20.0, 19.0, 21.0, 20.0, 22.0, 96.0, 20.0,
21.0, 19.0,
];
- let mut batch: Vec<(String, Value, std::sync::Arc<[u8]>)> = Vec::new();
+ let mut batch: Vec<(String, Value)> = Vec::new();
for (i, v) in cpu.iter().enumerate() {
let ts = base + chrono::Duration::minutes(i as i64);
let doc = json!({
"@timestamp": ts.to_rfc3339_opts(chrono::SecondsFormat::Millis, true),
"cpu": v,
});
- let bytes: std::sync::Arc<[u8]> =
- std::sync::Arc::from(serde_json::to_vec(&doc).unwrap());
- batch.push((format!("m{i:02}"), doc, bytes));
+ batch.push((format!("m{i:02}"), doc));
}
idx.index_batch_turbo(batch, false, false).await.unwrap();
@@ -29103,12 +29179,11 @@ mod reindex_keyset_tests {
// Index N docs with distinct _ids into `src` via the batched turbo
// path (one WAL append) so the test isn't dominated by per-doc fsyncs.
let src = state.engine.get_or_create_index("src").expect("src index");
- let mut batch: Vec<(String, Value, std::sync::Arc<[u8]>)> = Vec::with_capacity(n);
+ let mut batch: Vec<(String, Value)> = Vec::with_capacity(n);
for i in 0..n {
let id = format!("doc-{i:06}");
let v = json!({ "n": i });
- let bytes: std::sync::Arc<[u8]> = std::sync::Arc::from(serde_json::to_vec(&v).unwrap());
- batch.push((id, v, bytes));
+ batch.push((id, v));
}
src.index_batch_turbo(batch, true, false)
.await
diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rs
index c1603b237..ef64e91eb 100644
--- a/engine/crates/xerj-api/src/native.rs
+++ b/engine/crates/xerj-api/src/native.rs
@@ -1773,19 +1773,14 @@ pub async fn turbo_ingest(
let parallel = cfg.indexing.turbo_parallel;
let fast_analyzer = cfg.indexing.turbo_fast_analyzer;
- // Build `(id, source, source_bytes)` triples, auto-generating IDs
- // where not supplied. The native API accepts already-parsed
- // `Value` objects so we serialize once here (`to_vec`) to produce
- // the WAL bytes — still one serialize total versus two in the
- // pre-v14 path (one parse + one re-serialize).
- let pairs: Vec<(String, Value, std::sync::Arc<[u8]>)> = docs
+ // The engine accepts one parsed authority and serializes it exactly once
+ // at the storage boundary; there is no second caller-provided byte
+ // representation that could disagree with live indexing.
+ let pairs: Vec<(String, Value)> = docs
.into_iter()
.map(|doc| {
let id = extract_or_gen_id(&doc);
- let bytes: std::sync::Arc<[u8]> = serde_json::to_vec(&doc)
- .map(std::sync::Arc::from)
- .unwrap_or_else(|_| std::sync::Arc::from(&[][..]));
- (id, doc, bytes)
+ (id, doc)
})
.collect();
diff --git a/engine/crates/xerj-common/src/error.rs b/engine/crates/xerj-common/src/error.rs
index b12554055..86d57fb6e 100644
--- a/engine/crates/xerj-common/src/error.rs
+++ b/engine/crates/xerj-common/src/error.rs
@@ -55,6 +55,10 @@ pub enum XerjError {
#[error("serialization error: {reason}")]
SerializationError { reason: String },
+ /// A user-supplied document body is not one complete JSON value.
+ #[error("{reason}")]
+ InvalidDocumentJson { reason: String },
+
// ── Configuration ──────────────────────────────────────────────────────
/// The configuration file is invalid.
#[error("config error: {reason}")]
@@ -178,6 +182,12 @@ impl XerjError {
}
}
+ pub fn invalid_document_json(reason: impl Into<String>) -> Self {
+ Self::InvalidDocumentJson {
+ reason: reason.into(),
+ }
+ }
+
pub fn config(reason: impl Into<String>) -> Self {
Self::ConfigError {
reason: reason.into(),
@@ -252,6 +262,7 @@ impl XerjError {
Self::IndexAlreadyExists { .. } | Self::VersionConflict { .. } => 409,
Self::InvalidMapping { .. }
| Self::InvalidQuery { .. }
+ | Self::InvalidDocumentJson { .. }
| Self::ConfigError { .. }
| Self::ResultWindowTooLarge { .. } => 400,
Self::IndexBlocked { .. } => 403,
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index bc6d7fdda..822b9f209 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -40,6 +40,23 @@ pub struct BulkItemResult {
pub get_source: Option<Value>,
}
+fn malformed_document_reason(
+ document_id: Option<&str>,
+ item_index: usize,
+ error: &serde_json::Error,
+) -> String {
+ let document_id = document_id.unwrap_or("<auto-generated>");
+ format!(
+ "invalid document JSON for document ID {document_id:?} at bulk item position {}: \
+ parser error: {error}; parser location: line {}, column {}. \
+ Fix: replace this source line with exactly one complete UTF-8 JSON object. \
+ Related help: xerj index --help and https://xerj.org/llms.txt",
+ item_index + 1,
+ error.line(),
+ error.column(),
+ )
+}
+
fn engine_error_http_status(error: &EngineError) -> u16 {
match error {
EngineError::Common(xerj_common::XerjError::IndexBlocked { block_type, .. })
@@ -575,11 +592,13 @@ pub async fn process_bulk_with_opts(
BulkItemResult {
action: action_type.to_string(),
index: target_index,
- id: doc_id.unwrap_or_default(),
+ id: doc_id.clone().unwrap_or_default(),
status: 400,
result: None,
- error: Some(format!(
- "failed to parse: invalid document JSON: {e}"
+ error: Some(malformed_document_reason(
+ doc_id.as_deref(),
+ *item_index,
+ &e,
)),
get_source: None,
},
@@ -630,10 +649,14 @@ pub async fn process_bulk_with_opts(
BulkItemResult {
action: action_type.to_string(),
index: target_index,
- id: doc_id.unwrap_or_default(),
+ id: doc_id.clone().unwrap_or_default(),
status: 400,
result: None,
- error: Some(format!("invalid document JSON: {e}")),
+ error: Some(malformed_document_reason(
+ doc_id.as_deref(),
+ *item_index,
+ &e,
+ )),
get_source: None,
},
*item_index,
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 0233b97b7..b1e39e01e 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -2368,7 +2368,7 @@ impl Index {
/// failures are collected and returned rather than aborting the batch.
pub async fn index_batch_turbo(
&self,
- docs: Vec<(String, Value, Arc<[u8]>)>,
+ docs: Vec<(String, Value)>,
parallel: bool,
_fast_analyzer: bool,
) -> Result<Vec<IndexResponse>> {
@@ -2454,8 +2454,8 @@ impl Index {
// Build the pipeline with a batch size equal to the full input so it
// never auto-flushes mid-way through.
let mut pipeline = TurboIngestPipeline::new(batch_len + 1, parallel);
- for (id, source, source_bytes) in docs {
- pipeline.push(id, source, source_bytes);
+ for (id, source) in docs {
+ pipeline.push(id, source, Arc::<[u8]>::from([]));
}
// Flush collects all IngestResult values (with pre-computed tokens).
let results = pipeline.flush();
@@ -2478,7 +2478,10 @@ impl Index {
id: r.id,
tokens: r.tokens,
source,
- source_bytes: r.source_bytes,
+ // `copy_to` changed the parsed document. Force WAL
+ // serialization from that value so live indexing and
+ // crash replay cannot observe different sources.
+ source_bytes: Arc::<[u8]>::from([]),
}
})
.collect()
@@ -2507,15 +2510,9 @@ impl Index {
// segments where 150 k were expected, thrashing the flush
// path and triggering back-pressure 429s. Rolled back.
let t3 = std::time::Instant::now();
- let wal_refs: Vec<(String, Arc<Value>, Arc<[u8]>)> = processed
+ let wal_refs: Vec<(String, Arc<Value>)> = processed
.iter()
- .map(|r| {
- (
- r.id.clone(),
- Arc::clone(&r.source),
- Arc::clone(&r.source_bytes),
- )
- })
+ .map(|r| (r.id.clone(), Arc::clone(&r.source)))
.collect();
let wal_t = std::time::Instant::now();
@@ -2557,6 +2554,11 @@ impl Index {
// most one chunk, not the whole batch. `remove()` still precedes
// each `insert_pretokenized_with_seq`, so overwrite semantics are
// preserved regardless of the chunk boundary.
+ let copy_schema = if has_copy_to {
+ Some(self.schema.read().await)
+ } else {
+ None
+ };
{
let mut base = 0usize;
while base < batch_len {
@@ -2565,12 +2567,25 @@ impl Index {
for i in base..end {
let ingest = &processed[i];
mem.remove(&ingest.id);
- mem.insert_pretokenized_with_seq(
- seq_nos[i],
- ingest.id.clone(),
- Arc::clone(&ingest.source),
- &ingest.tokens,
- );
+ if let Some(schema) = copy_schema.as_ref() {
+ // copy_to created fields after the turbo pipeline
+ // emitted its (currently empty) token list. Analyze
+ // the transformed source so the target is searchable
+ // immediately, not only after flush/merge.
+ mem.insert(
+ ingest.id.clone(),
+ &ingest.source,
+ &schema.schema,
+ seq_nos[i],
+ );
+ } else {
+ mem.insert_pretokenized_with_seq(
+ seq_nos[i],
+ ingest.id.clone(),
+ Arc::clone(&ingest.source),
+ &ingest.tokens,
+ );
+ }
let version = self.doc_count.fetch_add(1, Ordering::Relaxed) + 1;
@@ -2585,6 +2600,7 @@ impl Index {
base = end;
}
}
+ drop(copy_schema);
let t4_dur = t4.elapsed();
if batch_len >= 1000 {
@@ -2686,17 +2702,10 @@ impl Index {
/// M5.11 — ULTRA-turbo raw-bytes ingest.
///
- /// Takes already-serialized NDJSON document lines and pushes them
- /// through the WAL + memtable without ever running
- /// `serde_json::from_str` on the document body. The JSON parse is
- /// deferred all the way to drain-for-flush time, where it runs on
- /// a background task without contending with HTTP workers.
- ///
- /// Trade-off: schema auto-evolution and vector HNSW indexing do
- /// NOT fire on this path — the parsed `Value` they need isn't
- /// built. Callers must pre-register the index mapping and avoid
- /// vector fields on this endpoint. Nginx-style log ingest is the
- /// canonical use case.
+ /// Takes already-serialized NDJSON document lines, validates and parses
+ /// the complete batch once, then reuses those values for WAL, schema, and
+ /// FTS work. Validation precedes every write-side action: malformed input
+ /// cannot reserve sequence numbers, become replayable, or trigger a flush.
pub async fn index_batch_turbo_raw(
&self,
docs: Vec<(String, Arc<[u8]>)>,
@@ -2704,6 +2713,7 @@ impl Index {
if docs.is_empty() {
return Ok(Vec::new());
}
+ let batch_len = docs.len();
if self.is_write_blocked().await {
return Err(EngineError::Common(xerj_common::XerjError::index_blocked(
self.name.as_str(),
@@ -2713,7 +2723,16 @@ impl Index {
// Process-wide admission: disk flood-stage block (item 3) + parent
// memory circuit breaker (item 1). Fires before any per-index work.
self.governor_write_gate()?;
-
+ let syntax_t = std::time::Instant::now();
+ let validated = crate::ingest_pool()
+ .install(|| xerj_storage::IndexStore::validate_raw_batch(docs))
+ .map_err(|error| match error {
+ xerj_storage::StorageError::RawBatchValidation { .. } => EngineError::Common(
+ xerj_common::XerjError::invalid_document_json(error.to_string()),
+ ),
+ error => EngineError::Storage(error),
+ })?;
+ let syntax_elapsed = syntax_t.elapsed();
let hard_block = self.flush_byte_threshold.saturating_mul(3);
let soft_block = self.flush_byte_threshold.saturating_mul(2);
{
@@ -2748,8 +2767,17 @@ impl Index {
}
}
+ let dom_t = std::time::Instant::now();
+ let validated = crate::ingest_pool()
+ .install(|| xerj_storage::IndexStore::parse_validated_raw_batch(validated))
+ .map_err(|error| match error {
+ xerj_storage::StorageError::RawBatchValidation { .. } => EngineError::Common(
+ xerj_common::XerjError::invalid_document_json(error.to_string()),
+ ),
+ error => EngineError::Storage(error),
+ })?;
+ let raw_parse_elapsed = syntax_elapsed.saturating_add(dom_t.elapsed());
let index_start = std::time::Instant::now();
- let batch_len = docs.len();
// THROWAWAY prof (XERJ_PROF): per-phase attribution of the turbo path.
let prof = std::env::var_os("XERJ_PROF").is_some();
let mut p_wal_us = 0u128;
@@ -2759,18 +2787,43 @@ impl Index {
let mut p_insert_us = 0u128;
let p_t = std::time::Instant::now();
- // Build WAL refs directly from bytes. We pass `Arc<Value::Null>`
- // as the `source` parameter because `wal_append_batch` uses
- // `source_bytes` when it's non-empty and never touches the
- // Value tree on the fast path.
- let null_val: Arc<Value> = Arc::new(Value::Null);
- let wal_refs: Vec<(String, Arc<Value>, Arc<[u8]>)> = docs
- .iter()
- .map(|(id, bytes)| (id.clone(), Arc::clone(&null_val), Arc::clone(bytes)))
- .collect();
+ // The complete request was parsed before admission/back-pressure.
+ // Record that preflight in the optional phase profiler.
+ use rayon::prelude::*;
+ if prof {
+ p_parse_us = raw_parse_elapsed.as_micros();
+ }
+ let has_copy_to = {
+ let schema = self.schema.read().await;
+ schema_has_copy_to(&schema.schema)
+ };
let wal_t = std::time::Instant::now();
- let seq_nos = self.store.wal_append_batch(&wal_refs)?;
+ let (docs, sources, seq_nos) = if has_copy_to {
+ let (docs, parsed) = validated.into_parts();
+ let parsed = parsed.expect("parse_validated_raw_batch retains parsed values");
+ let schema = self.schema.read().await;
+ let sources: Vec<Arc<Value>> = parsed
+ .iter()
+ .map(|source| Arc::new(apply_copy_to(source, &schema.schema)))
+ .collect();
+ drop(schema);
+ let wal_docs: Vec<(String, Arc<Value>)> = docs
+ .iter()
+ .zip(&sources)
+ .map(|((id, _), source)| (id.clone(), Arc::clone(source)))
+ .collect();
+ let seq_nos = self.store.wal_append_batch(&wal_docs)?;
+ (docs, sources, seq_nos)
+ } else {
+ let seq_nos = self.store.wal_append_batch_raw(&validated)?;
+ let (docs, parsed) = validated.into_parts();
+ (
+ docs,
+ parsed.expect("parse_validated_raw_batch retains parsed values"),
+ seq_nos,
+ )
+ };
// RC4 W4 item 2: WAL durability-write latency.
if let Some(m) = crate::engine_metrics() {
m.observe_wal_write(wal_t.elapsed().as_secs_f64());
@@ -2778,7 +2831,6 @@ impl Index {
if prof {
p_wal_us = p_t.elapsed().as_micros();
}
-
// Insert each doc EXACTLY ONCE into the engine FTS memtable,
// routed to the doc's OWN shard — bit-for-bit identical to the
// per-doc `index_document` path.
@@ -2822,32 +2874,6 @@ impl Index {
// with its own `seq_nos[i]`; (b) `version` = prior doc_count +
// position + 1, assigned in doc order via one batch-level
// `fetch_add`; (c) response order matches request order.
- use rayon::prelude::*;
-
- // 1. Parse in parallel — each doc exactly once. Arc-wrapped so
- // the memtable entry can share the allocation (no deep clone at
- // insert time — pre-fix `insert` did `Arc::new(source.clone())`
- // per doc under the shard write lock).
- //
- // Runs on the dedicated ingest pool (`crate::ingest_pool`) so bulk
- // parse bursts never queue ahead of search/agg par_iters on the
- // global rayon pool — see the read-under-write collapse notes on
- // `ingest_pool()`.
- let p_t = std::time::Instant::now();
- let sources: Vec<Arc<Value>> = crate::ingest_pool().install(|| {
- docs.par_iter()
- .map(|(_, bytes)| {
- Arc::new(
- serde_json::from_slice::<Value>(bytes)
- .unwrap_or_else(|_| Value::Object(serde_json::Map::new())),
- )
- })
- .collect()
- });
- if prof {
- p_parse_us = p_t.elapsed().as_micros();
- }
-
// 2. Dynamic mapping: evolve schema in doc order — batched (one
// schema read-lock for the whole batch, not one per doc).
let p_t = std::time::Instant::now();
@@ -2995,6 +3021,16 @@ impl Index {
if docs.is_empty() {
return Ok(0);
}
+ let batch_len = docs.len();
+ self.governor_write_gate()?;
+ let validated = crate::ingest_pool()
+ .install(|| xerj_storage::IndexStore::validate_raw_batch(docs))
+ .map_err(|error| match error {
+ xerj_storage::StorageError::RawBatchValidation { .. } => EngineError::Common(
+ xerj_common::XerjError::invalid_document_json(error.to_string()),
+ ),
+ error => EngineError::Storage(error),
+ })?;
let hard_block = self.flush_byte_threshold.saturating_mul(3);
let soft_block = self.flush_byte_threshold.saturating_mul(2);
@@ -3045,77 +3081,133 @@ impl Index {
}
let index_start = std::time::Instant::now();
- let batch_len = docs.len();
-
- // Fast-path WAL append: no Arc<Value> wrapper, no per-batch Vec
- // allocation. `wal_append_batch_raw` borrows docs directly.
let wal_t = std::time::Instant::now();
- let seq_nos = self.store.wal_append_batch_raw(&docs)?;
+ // Preserve copy_to semantics on the synchronous CLI path too. The
+ // common no-copy_to case keeps the sealed raw-byte WAL fast path.
+ // For the rare mapped case, materialize once, transform the sole
+ // source authority, and retain transformed bytes for the raw
+ // memtable so live GET, flush, and replay all observe the same value.
+ let has_copy_to = {
+ let schema = self.schema.try_read().map_err(|_| {
+ EngineError::Common(xerj_common::XerjError::resource_exhausted(
+ "schema is being updated during synchronous ingest; retry the batch",
+ ))
+ })?;
+ schema_has_copy_to(&schema.schema)
+ };
+ let (docs, seq_nos, copy_schema) = if has_copy_to {
+ let validated = crate::ingest_pool()
+ .install(|| xerj_storage::IndexStore::parse_validated_raw_batch(validated))
+ .map_err(EngineError::Storage)?;
+ let (raw_docs, parsed) = validated.into_parts();
+ let parsed = parsed.expect("parse_validated_raw_batch retains parsed values");
+ let schema = self.schema.try_read().map_err(|_| {
+ EngineError::Common(xerj_common::XerjError::resource_exhausted(
+ "schema is being updated during synchronous ingest; retry the batch",
+ ))
+ })?;
+ let copy_schema = schema.schema.clone();
+ let transformed: Vec<Arc<Value>> = parsed
+ .iter()
+ .map(|source| Arc::new(apply_copy_to(source, &copy_schema)))
+ .collect();
+ drop(schema);
+ let wal_docs: Vec<(String, Arc<Value>)> = raw_docs
+ .iter()
+ .zip(&transformed)
+ .map(|((id, _), source)| (id.clone(), Arc::clone(source)))
+ .collect();
+ let seq_nos = self.store.wal_append_batch(&wal_docs)?;
+ let docs = raw_docs
+ .into_iter()
+ .zip(transformed)
+ .map(|((id, _), source)| {
+ serde_json::to_vec(source.as_ref())
+ .map(|bytes| (id, Arc::<[u8]>::from(bytes), Some(source)))
+ .map_err(xerj_storage::StorageError::from)
+ })
+ .collect::<std::result::Result<Vec<_>, _>>()?;
+ (docs, seq_nos, Some(copy_schema))
+ } else {
+ let seq_nos = self.store.wal_append_batch_raw(&validated)?;
+ let (docs, parsed) = validated.into_parts();
+ debug_assert!(parsed.is_none());
+ (
+ docs.into_iter()
+ .map(|(id, bytes)| (id, bytes, None))
+ .collect(),
+ seq_nos,
+ None,
+ )
+ };
// RC4 W4 item 2: WAL durability-write latency.
if let Some(m) = crate::engine_metrics() {
m.observe_wal_write(wal_t.elapsed().as_secs_f64());
}
- // Use the instance method so routing matches the actual configured
- // shard count. The previous `Self::shard_for` was hardcoded to a
- // 16-shard mask and panicked on machines configured for fewer.
- let shard_idx = self.memtable.shard_for_dynamic(&docs[0].0);
- // Consume `docs` by value — move each (String, Arc<[u8]>) into
- // the memtable without cloning. The CLI bulk path generates
- // unique doc_ids so `insert_raw_bytes_fresh` is safe (no prior
- // entry with this id); we skip the `remove()` HashMap miss
- // lookup that the generic `insert_raw_bytes_with_seq` preceded.
- //
- // READ-UNDER-WRITE stall reduction: the insert loop is CHUNKED so
- // the shard write lock is released between chunks. Previously a
- // whole bulk batch (mixval: 10 000 docs) was routed to ONE shard and
- // the write lock was held for the ENTIRE insert (~10-30 ms), so a
- // concurrent search that touched that shard (`doc_ids_bounded`,
- // `all_docs_with_sources_arc`, `terms_counts_columnar` all take
- // `s.read()` on every shard) blocked for the full batch. Chunking
- // bounds a reader's *lock* stall on a writing shard to ONE chunk
- // (~0.5-1.5 ms). NOTE: measured in isolation this removes the
- // shard-lock component of the read tail, but the mixed
- // read-under-write p99 is dominated by CPU/scheduler contention with
- // the full-speed ingest+flush+merge (already deprioritised via the
- // nice-pool ladder), which this does not address — see commit body.
- // The lock churn is trivial (batch_len/CHUNK acquisitions, e.g. 20
- // for a 10 k batch) — nothing like the per-DOC locking that once
- // caused the 4× ingest regression. Correctness is unchanged:
- // seq_nos are pre-assigned, inserts stay in seq order, and a flush
- // drain that slips between chunks is safe (docs are independent and
- // `take_memtable_for_flush` re-sorts by seq_no).
- {
- let mut docs_iter = docs.into_iter();
- let mut i = 0usize;
- while i < batch_len {
- let end = (i + MEMTABLE_INSERT_CHUNK).min(batch_len);
- self.memtable.with_shard_mut(shard_idx, |mem| {
- for &seq_no in &seq_nos[i..end] {
- let (id, bytes) = docs_iter
- .next()
- .expect("docs_iter yields exactly batch_len items");
- mem.insert_raw_bytes_fresh(seq_no, id, bytes);
+ // Partition by each document's own shard. The old sync path routed the
+ // entire batch to docs[0]'s shard, while GET/search route by each ID;
+ // every sibling ID hashing elsewhere was therefore invisible until
+ // flush. Preserve request order within each shard so duplicate IDs are
+ // deterministic last-write-wins.
+ let n_shards = self.memtable.shard_count().max(1);
+ let mut buckets: Vec<Vec<(u64, String, Arc<[u8]>, Option<Arc<Value>>)>> =
+ (0..n_shards).map(|_| Vec::new()).collect();
+ for (seq_no, (id, bytes, source)) in seq_nos.iter().copied().zip(docs) {
+ let shard = self.memtable.shard_for_dynamic(&id);
+ buckets[shard].push((seq_no, id, bytes, source));
+ }
+ let touched_shards: Vec<usize> = crate::ingest_pool().install(|| {
+ use rayon::prelude::*;
+ buckets
+ .into_par_iter()
+ .enumerate()
+ .filter_map(|(shard, bucket)| {
+ if bucket.is_empty() {
+ return None;
}
- });
- i = end;
- }
- }
+ for chunk in bucket.chunks(MEMTABLE_INSERT_CHUNK) {
+ self.memtable.with_shard_mut(shard, |mem| {
+ for (seq_no, id, bytes, source) in chunk {
+ // `insert_raw_bytes_fresh` requires absence.
+ // Remove unconditionally so an overwrite from
+ // an earlier batch and a duplicate within this
+ // batch have identical last-write-wins
+ // semantics.
+ mem.remove(id);
+ if let (Some(source), Some(schema)) =
+ (source.as_ref(), copy_schema.as_ref())
+ {
+ mem.insert(id.clone(), source, schema, *seq_no);
+ } else {
+ mem.insert_raw_bytes_fresh(
+ *seq_no,
+ id.clone(),
+ Arc::clone(bytes),
+ );
+ }
+ }
+ });
+ }
+ Some(shard)
+ })
+ .collect()
+ });
// Single batch-level atomic instead of one per doc. At 1.7 M/s
// × 10 k batch = 170 batches/s this cuts ~17 M atomic ops/s of
// cache-line bouncing on `doc_count`.
self.doc_count
.fetch_add(batch_len as u64, Ordering::Relaxed);
- // Cheap per-shard threshold check — one read-lock, one shard.
- let n_shards = self.memtable.shard_count().max(1);
- let per_shard_doc_t =
- staggered_per_shard_threshold(self.flush_doc_threshold, shard_idx, n_shards);
- let per_shard_byte_t =
- staggered_per_shard_threshold(self.flush_byte_threshold, shard_idx, n_shards);
- let (sd_docs, sd_bytes) = self.memtable.shard_load(shard_idx);
- if sd_docs >= per_shard_doc_t || sd_bytes >= per_shard_byte_t {
- self.try_spawn_sync_flush(shard_idx);
+ for shard in touched_shards {
+ let per_shard_doc_t =
+ staggered_per_shard_threshold(self.flush_doc_threshold, shard, n_shards);
+ let per_shard_byte_t =
+ staggered_per_shard_threshold(self.flush_byte_threshold, shard, n_shards);
+ let (sd_docs, sd_bytes) = self.memtable.shard_load(shard);
+ if sd_docs >= per_shard_doc_t || sd_bytes >= per_shard_byte_t {
+ self.try_spawn_sync_flush(shard);
+ }
}
let elapsed_ms = index_start.elapsed().as_millis() as u64;
@@ -28134,4 +28226,552 @@ mod write_publication_integration_tests {
"successful delete invalidates cached responses exactly once"
);
}
+
+ fn raw_test_docs(payloads: &[(&str, &[u8])]) -> Vec<(String, Arc<[u8]>)> {
+ payloads
+ .iter()
+ .map(|(id, bytes)| ((*id).to_owned(), Arc::<[u8]>::from(*bytes)))
+ .collect()
+ }
+
+ fn engine_wal_bytes(idx: &Index) -> Vec<(PathBuf, Vec<u8>)> {
+ let mut files: Vec<(PathBuf, Vec<u8>)> = std::fs::read_dir(idx.store.wal_dir())
+ .unwrap()
+ .map(|entry| entry.unwrap().path())
+ .filter(|path| path.is_file())
+ .map(|path| {
+ (
+ path.file_name().unwrap().into(),
+ std::fs::read(&path).unwrap(),
+ )
+ })
+ .collect();
+ files.sort_by(|a, b| a.0.cmp(&b.0));
+ files
+ }
+
+ #[tokio::test]
+ async fn sync_raw_invalid_json_changes_no_engine_or_durable_state() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ engine
+ .create_index("sync-raw-invalid", Schema::empty())
+ .unwrap();
+ let idx = engine.get_index("sync-raw-invalid").unwrap();
+ idx.index_document(Some("seed".to_owned()), json!({"existing": true}))
+ .await
+ .unwrap();
+
+ let invalid: &[&[u8]] = &[
+ br#"{"truncated":"#,
+ br#"{"valid":true} trailing"#,
+ &[0xff, 0xfe],
+ b" \n\t",
+ ];
+ for (case, bad) in invalid.iter().enumerate() {
+ for bad_at in 0..3 {
+ let mut docs = raw_test_docs(&[
+ ("first", br#"{"field":1}"#),
+ ("middle", br#"{"field":2}"#),
+ ("last", br#"{"field":3}"#),
+ ]);
+ docs[bad_at].1 = Arc::<[u8]>::from(*bad);
+ let ids: Vec<_> = docs.iter().map(|(id, _)| id.clone()).collect();
+ let before_seq = idx.store.current_seq_no();
+ let before_wal = engine_wal_bytes(&idx);
+ let before_vm = idx.store.version_map.len();
+ let before_mem = idx.memtable.size_bytes();
+ let before_count = idx.doc_count.load(Ordering::Acquire);
+ let before_dataset = idx.dataset_version.load(Ordering::Acquire);
+ let before_metric = idx.metric_index_count.load(Ordering::Acquire);
+ let before_schema = idx.schema.read().await.schema.clone();
+
+ let error = idx.index_batch_sync_raw(docs).unwrap_err();
+ let rendered = error.to_string();
+ assert!(
+ rendered.contains(&format!("document [{}]", ids[bad_at]))
+ && rendered.contains(&format!("batch position {}", bad_at + 1))
+ && rendered.contains("Fix:")
+ && rendered.contains("Related help:"),
+ "case {case}, position {bad_at}: {error}"
+ );
+ assert_eq!(idx.store.current_seq_no(), before_seq);
+ assert_eq!(engine_wal_bytes(&idx), before_wal);
+ assert_eq!(idx.store.version_map.len(), before_vm);
+ assert_eq!(idx.memtable.size_bytes(), before_mem);
+ assert_eq!(idx.doc_count.load(Ordering::Acquire), before_count);
+ assert_eq!(idx.dataset_version.load(Ordering::Acquire), before_dataset);
+ assert_eq!(
+ idx.metric_index_count.load(Ordering::Acquire),
+ before_metric
+ );
+ assert_eq!(idx.schema.read().await.schema, before_schema);
+ for id in ids {
+ assert!(idx.get_document(&id).await.unwrap().is_none());
+ }
+ }
+ }
+ let too_deep = format!("{}0{}", "[".repeat(256), "]".repeat(256)).into_bytes();
+ let before_seq = idx.store.current_seq_no();
+ let error = idx
+ .index_batch_sync_raw(vec![("too-deep".to_owned(), Arc::<[u8]>::from(too_deep))])
+ .unwrap_err();
+ assert!(error.to_string().contains("nesting exceeds"), "{error}");
+ assert_eq!(idx.store.current_seq_no(), before_seq);
+
+ drop(idx);
+ drop(engine);
+ let reopened = crate::Engine::new({
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config
+ })
+ .unwrap();
+ let idx = reopened.get_index("sync-raw-invalid").unwrap();
+ assert_eq!(
+ idx.get_document("seed").await.unwrap().unwrap()["existing"],
+ true
+ );
+ for id in ["first", "middle", "last"] {
+ assert!(idx.get_document(id).await.unwrap().is_none());
+ }
+ }
+
+ #[tokio::test]
+ async fn async_raw_invalid_json_rejects_whole_batch_before_wal() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ engine
+ .create_index("async-raw-invalid", Schema::empty())
+ .unwrap();
+ let idx = engine.get_index("async-raw-invalid").unwrap();
+ idx.index_document(Some("seed".to_owned()), json!({"existing": true}))
+ .await
+ .unwrap();
+ let invalid: &[&[u8]] = &[
+ br#"{"truncated":"#,
+ br#"{"valid":true} trailing"#,
+ &[0xff, 0xfe],
+ b" \n\t",
+ ];
+ for (case, bad) in invalid.iter().enumerate() {
+ for bad_at in 0..3 {
+ let mut docs = raw_test_docs(&[
+ ("good-first", br#"{"field":1}"#),
+ ("middle", br#"{"field":2}"#),
+ ("good-last", br#"{"field":3}"#),
+ ]);
+ docs[bad_at].1 = Arc::<[u8]>::from(*bad);
+ let before_seq = idx.store.current_seq_no();
+ let before_wal = engine_wal_bytes(&idx);
+ let before_schema = idx.schema.read().await.schema.clone();
+ let before_vm = idx.store.version_map.len();
+ let before_mem = idx.memtable.size_bytes();
+ let before_count = idx.doc_count.load(Ordering::Acquire);
+ let before_dataset = idx.dataset_version.load(Ordering::Acquire);
+ let before_metric = idx.metric_index_count.load(Ordering::Acquire);
+
+ let error = idx.index_batch_turbo_raw(docs).await.unwrap_err();
+ let rendered = error.to_string();
+ assert!(
+ rendered.contains(&format!("batch position {}", bad_at + 1))
+ && rendered.contains("Fix:")
+ && rendered.contains("Related help:"),
+ "case {case}, position {bad_at}: {error}"
+ );
+ assert_eq!(idx.store.current_seq_no(), before_seq);
+ assert_eq!(engine_wal_bytes(&idx), before_wal);
+ assert_eq!(idx.store.version_map.len(), before_vm);
+ assert_eq!(idx.memtable.size_bytes(), before_mem);
+ assert_eq!(idx.doc_count.load(Ordering::Acquire), before_count);
+ assert_eq!(idx.dataset_version.load(Ordering::Acquire), before_dataset);
+ assert_eq!(
+ idx.metric_index_count.load(Ordering::Acquire),
+ before_metric
+ );
+ assert_eq!(idx.schema.read().await.schema, before_schema);
+ assert_eq!(
+ idx.get_document("seed").await.unwrap().unwrap()["existing"],
+ true
+ );
+ }
+ }
+ }
+
+ #[tokio::test]
+ async fn raw_ingest_apis_accept_complete_scalar_object_and_array_json() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ engine.create_index("raw-valid", Schema::empty()).unwrap();
+ let idx = engine.get_index("raw-valid").unwrap();
+
+ assert_eq!(
+ idx.index_batch_sync_raw(raw_test_docs(&[
+ ("sync-scalar", b"42"),
+ ("sync-object", br#"{"answer":42}"#),
+ ("sync-array", b"[1,2,3]"),
+ ]))
+ .unwrap(),
+ 3
+ );
+ assert_eq!(
+ idx.index_batch_turbo_raw(raw_test_docs(&[
+ ("async-scalar", b"true"),
+ ("async-object", br#"{"answer":43}"#),
+ ("async-array", b"[4,5,6]"),
+ ]))
+ .await
+ .unwrap()
+ .len(),
+ 3
+ );
+ assert_eq!(idx.doc_count.load(Ordering::Acquire), 6);
+ assert_eq!(
+ idx.index_batch_sync_raw(raw_test_docs(&[
+ ("duplicate", br#"{"revision":1}"#),
+ ("duplicate", br#"{"revision":2}"#),
+ ]))
+ .unwrap(),
+ 2
+ );
+ assert_eq!(
+ idx.get_document("duplicate").await.unwrap().unwrap(),
+ json!({"revision": 2})
+ );
+ let expected = [
+ ("sync-scalar", json!(42)),
+ ("sync-object", json!({"answer": 42})),
+ ("sync-array", json!([1, 2, 3])),
+ ("async-scalar", json!(true)),
+ ("async-object", json!({"answer": 43})),
+ ("async-array", json!([4, 5, 6])),
+ ("duplicate", json!({"revision": 2})),
+ ];
+ for (id, value) in &expected {
+ assert_eq!(idx.get_document(id).await.unwrap().as_ref(), Some(value));
+ }
+
+ idx.flush().await.unwrap();
+ for (id, value) in &expected {
+ assert_eq!(idx.get_document(id).await.unwrap().as_ref(), Some(value));
+ }
+ drop(idx);
+ drop(engine);
+
+ let reopened = crate::Engine::new({
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config
+ })
+ .unwrap();
+ let idx = reopened.get_index("raw-valid").unwrap();
+ for (id, value) in &expected {
+ assert_eq!(idx.get_document(id).await.unwrap().as_ref(), Some(value));
+ }
+ }
+
+ #[tokio::test]
+ async fn turbo_copy_to_wal_replays_the_transformed_source() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ let mut schema = Schema::empty();
+ let mut title = FieldConfig::new("title", FieldType::Text);
+ title.options.null_value = Some(Value::String("__copy_to__:all_text".to_owned()));
+ schema.add_field(title).unwrap();
+ schema
+ .add_field(FieldConfig::new("all_text", FieldType::Text))
+ .unwrap();
+ engine.create_index("turbo-copy-to", schema).unwrap();
+ let idx = engine.get_index("turbo-copy-to").unwrap();
+ let source = json!({"title": "durable copied text"});
+
+ idx.index_batch_turbo(vec![("doc".to_owned(), source)], false, false)
+ .await
+ .unwrap();
+ let live = idx.get_document("doc").await.unwrap().unwrap();
+ assert_eq!(live["all_text"], "durable copied text");
+ let request = SearchRequest {
+ query: QueryNode::Match {
+ field: "all_text".to_owned(),
+ query: "durable copied".to_owned(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ },
+ ..SearchRequest::default()
+ };
+ // The raw sync path intentionally defers lexical analysis to flush,
+ // but the live memtable must already contain exactly one revision.
+ assert_eq!(idx.memtable.doc_count(), 1);
+ idx.flush().await.unwrap();
+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);
+ drop(idx);
+ drop(engine);
+
+ let reopened = crate::Engine::new({
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config
+ })
+ .unwrap();
+ let replayed = reopened
+ .get_index("turbo-copy-to")
+ .unwrap()
+ .get_document("doc")
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(replayed, live, "live and WAL replay sources diverged");
+ assert_eq!(
+ reopened
+ .get_index("turbo-copy-to")
+ .unwrap()
+ .search(&request)
+ .await
+ .unwrap()
+ .total
+ .value,
+ 1
+ );
+ }
+
+ #[tokio::test]
+ async fn raw_turbo_copy_to_is_searchable_and_durable_after_flush_restart() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ let mut schema = Schema::empty();
+ let mut title = FieldConfig::new("title", FieldType::Text);
+ title.options.null_value = Some(Value::String("__copy_to__:all_text".to_owned()));
+ schema.add_field(title).unwrap();
+ schema
+ .add_field(FieldConfig::new("all_text", FieldType::Text))
+ .unwrap();
+ engine.create_index("raw-copy-to", schema).unwrap();
+ let idx = engine.get_index("raw-copy-to").unwrap();
+ idx.index_batch_turbo_raw(raw_test_docs(&[(
+ "doc",
+ br#"{"title":"raw durable copied text"}"#,
+ )]))
+ .await
+ .unwrap();
+ let expected = idx.get_document("doc").await.unwrap().unwrap();
+ assert_eq!(expected["title"], "raw durable copied text");
+ assert_eq!(expected["all_text"], "raw durable copied text");
+ let request = SearchRequest {
+ query: QueryNode::Match {
+ field: "all_text".to_owned(),
+ query: "raw durable".to_owned(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ },
+ ..SearchRequest::default()
+ };
+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);
+ idx.flush().await.unwrap();
+ assert_eq!(idx.get_document("doc").await.unwrap().unwrap(), expected);
+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);
+ drop(idx);
+ drop(engine);
+
+ let reopened = crate::Engine::new({
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config
+ })
+ .unwrap();
+ let idx = reopened.get_index("raw-copy-to").unwrap();
+ assert_eq!(idx.get_document("doc").await.unwrap().unwrap(), expected);
+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);
+ }
+
+ #[tokio::test]
+ async fn sync_raw_copy_to_is_searchable_and_durable_after_flush_restart() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ let mut schema = Schema::empty();
+ let mut title = FieldConfig::new("title", FieldType::Text);
+ title.options.null_value = Some(Value::String("__copy_to__:all_text".to_owned()));
+ schema.add_field(title).unwrap();
+ schema
+ .add_field(FieldConfig::new("all_text", FieldType::Text))
+ .unwrap();
+ engine.create_index("sync-raw-copy-to", schema).unwrap();
+ let idx = engine.get_index("sync-raw-copy-to").unwrap();
+ idx.index_batch_sync_raw(raw_test_docs(&[(
+ "doc",
+ br#"{"title":"sync durable copied text"}"#,
+ )]))
+ .unwrap();
+ let expected = idx.get_document("doc").await.unwrap().unwrap();
+ assert_eq!(expected["title"], "sync durable copied text");
+ assert_eq!(expected["all_text"], "sync durable copied text");
+ let request = SearchRequest {
+ query: QueryNode::Match {
+ field: "all_text".to_owned(),
+ query: "sync durable".to_owned(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ },
+ ..SearchRequest::default()
+ };
+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);
+ idx.flush().await.unwrap();
+ assert_eq!(idx.get_document("doc").await.unwrap().unwrap(), expected);
+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);
+ drop(idx);
+ drop(engine);
+
+ let reopened = crate::Engine::new({
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config
+ })
+ .unwrap();
+ let idx = reopened.get_index("sync-raw-copy-to").unwrap();
+ assert_eq!(idx.get_document("doc").await.unwrap().unwrap(), expected);
+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);
+ }
+
+ #[tokio::test]
+ async fn sync_raw_cross_batch_overwrite_is_single_and_durable() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ engine
+ .create_index("sync-raw-overwrite", Schema::empty())
+ .unwrap();
+ let idx = engine.get_index("sync-raw-overwrite").unwrap();
+ idx.index_batch_sync_raw(raw_test_docs(&[(
+ "same-id",
+ br#"{"revision":1,"tag":"obsolete"}"#,
+ )]))
+ .unwrap();
+ idx.flush().await.unwrap();
+ idx.index_batch_sync_raw(raw_test_docs(&[(
+ "same-id",
+ br#"{"revision":2,"tag":"current"}"#,
+ )]))
+ .unwrap();
+ let expected = json!({"revision": 2, "tag": "current"});
+ assert_eq!(
+ idx.get_document("same-id").await.unwrap().as_ref(),
+ Some(&expected)
+ );
+ let request = SearchRequest {
+ query: QueryNode::Match {
+ field: "tag".to_owned(),
+ query: "current".to_owned(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ },
+ ..SearchRequest::default()
+ };
+ let obsolete_request = SearchRequest {
+ query: QueryNode::Match {
+ field: "tag".to_owned(),
+ query: "obsolete".to_owned(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ },
+ ..SearchRequest::default()
+ };
+ let aggregate_request = SearchRequest {
+ query: QueryNode::MatchAll,
+ size: 0,
+ aggs: Some(json!({
+ "tags": {"terms": {"field": "tag"}},
+ "revision_cardinality": {"cardinality": {"field": "revision"}},
+ "max_revision": {"max": {"field": "revision"}}
+ })),
+ ..SearchRequest::default()
+ };
+ assert_eq!(
+ idx.memtable.doc_count(),
+ 1,
+ "cross-call overwrite must leave one live memtable entry"
+ );
+ // Ordinary sync raw ingest defers lexical analysis until flush. The
+ // disk-old/memory-new overlap must nevertheless hide stale postings;
+ // exact source, logical count, and aggregations use revision 2.
+ assert_eq!(idx.search(&obsolete_request).await.unwrap().total.value, 0);
+ let aggregate = idx.search(&aggregate_request).await.unwrap();
+ assert_eq!(aggregate.total.value, 1);
+ let aggs = aggregate.aggs.unwrap();
+ assert_eq!(aggs["revision_cardinality"]["value"], 1);
+ assert_eq!(aggs["max_revision"]["value"], 2.0);
+ assert_eq!(aggs["tags"]["buckets"].as_array().unwrap().len(), 1);
+ assert_eq!(aggs["tags"]["buckets"][0]["key"], "current");
+ drop(idx);
+ drop(engine);
+
+ // Revision 2 exists only in WAL/memory here. Restart must replay it
+ // over the durable revision-1 segment without exposing stale terms or
+ // double-counting the logical document.
+ let reopened = crate::Engine::new({
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config
+ })
+ .unwrap();
+ let idx = reopened.get_index("sync-raw-overwrite").unwrap();
+ assert_eq!(
+ idx.get_document("same-id").await.unwrap().as_ref(),
+ Some(&expected)
+ );
+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);
+ assert_eq!(idx.search(&obsolete_request).await.unwrap().total.value, 0);
+ let aggregate = idx.search(&aggregate_request).await.unwrap();
+ assert_eq!(aggregate.total.value, 1);
+ assert_eq!(aggregate.aggs.unwrap()["max_revision"]["value"], 2.0);
+ idx.flush().await.unwrap();
+ drop(idx);
+ drop(reopened);
+
+ // The same invariants must survive publication of revision 2 and a
+ // second restart.
+ let reopened_after_second_flush = crate::Engine::new({
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config
+ })
+ .unwrap();
+ let idx = reopened_after_second_flush
+ .get_index("sync-raw-overwrite")
+ .unwrap();
+ assert_eq!(
+ idx.get_document("same-id").await.unwrap().as_ref(),
+ Some(&expected)
+ );
+ assert_eq!(idx.search(&request).await.unwrap().total.value, 1);
+ assert_eq!(idx.search(&obsolete_request).await.unwrap().total.value, 0);
+ let aggregate = idx.search(&aggregate_request).await.unwrap();
+ assert_eq!(aggregate.total.value, 1);
+ let aggs = aggregate.aggs.unwrap();
+ assert_eq!(aggs["revision_cardinality"]["value"], 1);
+ assert_eq!(aggs["max_revision"]["value"], 2.0);
+ assert_eq!(aggs["tags"]["buckets"].as_array().unwrap().len(), 1);
+ assert_eq!(aggs["tags"]["buckets"][0]["key"], "current");
+ }
}
diff --git a/engine/crates/xerj-engine/tests/perf_benchmark.rs b/engine/crates/xerj-engine/tests/perf_benchmark.rs
index 68ec30935..6476fe47a 100644
--- a/engine/crates/xerj-engine/tests/perf_benchmark.rs
+++ b/engine/crates/xerj-engine/tests/perf_benchmark.rs
@@ -622,12 +622,11 @@ async fn perf_turbo_indexing_throughput() {
let idx_turbo = engine_turbo.get_index("turbo_bench").unwrap();
let turbo_start = Instant::now();
- let mut batch: Vec<(String, serde_json::Value, std::sync::Arc<[u8]>)> =
- Vec::with_capacity(TURBO_BATCH);
+ let mut batch: Vec<(String, serde_json::Value)> = Vec::with_capacity(TURBO_BATCH);
for i in 0..N {
- let empty_bytes: std::sync::Arc<[u8]> = std::sync::Arc::from(&[][..]);
- batch.push((format!("doc{i}"), make_doc(i), empty_bytes));
+ let source = make_doc(i);
+ batch.push((format!("doc{i}"), source));
if batch.len() >= TURBO_BATCH {
let b = std::mem::replace(&mut batch, Vec::with_capacity(TURBO_BATCH));
diff --git a/engine/crates/xerj-storage/Cargo.toml b/engine/crates/xerj-storage/Cargo.toml
index b35d8f0ae..4acb526ec 100644
--- a/engine/crates/xerj-storage/Cargo.toml
+++ b/engine/crates/xerj-storage/Cargo.toml
@@ -15,6 +15,7 @@ tokio = { workspace = true }
# Serialization
serde = { workspace = true }
serde_json = { workspace = true }
+rayon = { workspace = true }
# Binary I/O
bytes = { workspace = true }
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 63c1f04c8..5c32e345a 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -310,6 +310,68 @@ pub struct IndexStoreConfig {
pub num_wal_shards: usize,
}
+/// A raw JSON batch that has been completely validated before publication.
+///
+/// Fields are private so callers cannot bypass validation and hand malformed
+/// bytes to the WAL writer. The parsed form is retained only when requested
+/// by an engine path that will reuse it for schema/FTS work.
+pub type RawJsonDoc = (String, std::sync::Arc<[u8]>);
+pub type ParsedRawSources = Vec<std::sync::Arc<serde_json::Value>>;
+
+#[derive(Debug)]
+pub struct ValidatedRawBatch {
+ docs: Vec<RawJsonDoc>,
+ parsed: Option<ParsedRawSources>,
+}
+
+impl ValidatedRawBatch {
+ pub fn docs(&self) -> &[RawJsonDoc] {
+ &self.docs
+ }
+
+ pub fn parsed(&self) -> Option<&[std::sync::Arc<serde_json::Value>]> {
+ self.parsed.as_deref()
+ }
+
+ pub fn into_parts(self) -> (Vec<RawJsonDoc>, Option<ParsedRawSources>) {
+ (self.docs, self.parsed)
+ }
+}
+
+const MAX_RAW_JSON_NESTING: usize = 128;
+
+fn validate_raw_json_nesting(bytes: &[u8]) -> std::result::Result<(), String> {
+ let mut depth = 0usize;
+ let mut in_string = false;
+ let mut escaped = false;
+ for &byte in bytes {
+ if in_string {
+ if escaped {
+ escaped = false;
+ } else if byte == b'\\' {
+ escaped = true;
+ } else if byte == b'"' {
+ in_string = false;
+ }
+ continue;
+ }
+ match byte {
+ b'"' => in_string = true,
+ b'{' | b'[' => {
+ depth += 1;
+ if depth > MAX_RAW_JSON_NESTING {
+ return Err(format!(
+ "JSON nesting exceeds the supported limit of {MAX_RAW_JSON_NESTING}"
+ ));
+ }
+ }
+ b'}' | b']' => depth = depth.saturating_sub(1),
+ _ => {}
+ }
+ }
+ Ok(())
+}
+
impl Default for IndexStoreConfig {
fn default() -> Self {
Self {
@@ -2889,10 +2951,72 @@ impl IndexStore {
///
/// All on-disk framing is byte-identical to `wal_append_batch`; the
/// two entries interleave freely in the WAL.
- pub fn wal_append_batch_raw(
- &self,
- docs: &[(String, std::sync::Arc<[u8]>)],
- ) -> Result<Vec<SeqNo>> {
+ /// Validate raw JSON without materializing a DOM. The returned sealed
+ /// value is the only input accepted by [`Self::wal_append_batch_raw`].
+ pub fn validate_raw_batch(docs: Vec<RawJsonDoc>) -> Result<ValidatedRawBatch> {
+ use rayon::prelude::*;
+ docs.par_iter()
+ .enumerate()
+ .map(|(position, (doc_id, source_bytes))| {
+ validate_raw_json_nesting(source_bytes).map_err(|reason| {
+ StorageError::RawBatchValidation {
+ doc_id: doc_id.clone(),
+ position: position + 1,
+ reason,
+ }
+ })?;
+ serde_json::from_slice::<serde::de::IgnoredAny>(source_bytes)
+ .map(|_| ())
+ .map_err(|error| StorageError::RawBatchValidation {
+ doc_id: doc_id.clone(),
+ position: position + 1,
+ reason: error.to_string(),
+ })
+ })
+ .collect::<Result<()>>()?;
+ Ok(ValidatedRawBatch { docs, parsed: None })
+ }
+
+ /// Validate and retain parsed values for callers that need both WAL bytes
+ /// and a JSON DOM. This keeps the async raw ingest path to one parse.
+ pub fn parse_raw_batch(docs: Vec<RawJsonDoc>) -> Result<ValidatedRawBatch> {
+ use rayon::prelude::*;
+ let parsed = docs
+ .par_iter()
+ .enumerate()
+ .map(|(position, (doc_id, source_bytes))| {
+ validate_raw_json_nesting(source_bytes).map_err(|reason| {
+ StorageError::RawBatchValidation {
+ doc_id: doc_id.clone(),
+ position: position + 1,
+ reason,
+ }
+ })?;
+ serde_json::from_slice(source_bytes)
+ .map(std::sync::Arc::new)
+ .map_err(|error| StorageError::RawBatchValidation {
+ doc_id: doc_id.clone(),
+ position: position + 1,
+ reason: error.to_string(),
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+ Ok(ValidatedRawBatch {
+ docs,
+ parsed: Some(parsed),
+ })
+ }
+
+ /// Materialize parsed values from a batch whose complete JSON syntax was
+ /// already validated. This consumes the sealed proof and retains the
+ /// original immutable bytes alongside the DOM.
+ pub fn parse_validated_raw_batch(batch: ValidatedRawBatch) -> Result<ValidatedRawBatch> {
+ debug_assert!(batch.parsed.is_none());
+ Self::parse_raw_batch(batch.docs)
+ }
+
+ pub fn wal_append_batch_raw(&self, batch: &ValidatedRawBatch) -> Result<Vec<SeqNo>> {
+ let docs = batch.docs();
if docs.is_empty() {
return Ok(Vec::new());
}
@@ -3010,17 +3134,15 @@ impl IndexStore {
Ok(seq_nos)
}
- // The `docs` slice element is a WAL-batch tuple (doc_id, source JSON,
- // pre-serialized bytes); the shape is part of the public batch API so we
- // keep it inline rather than refactor the signature.
- #[allow(clippy::type_complexity)]
+ /// Append parsed documents to the WAL.
+ ///
+ /// The parsed `Value` is the sole authority and is serialized here.
+ /// Accepting a second caller-provided byte representation made it possible
+ /// for live indexing and replay to observe different documents; verifying
+ /// that representation required a second full DOM parse on the hot path.
pub fn wal_append_batch(
&self,
- docs: &[(
- String,
- std::sync::Arc<serde_json::Value>,
- std::sync::Arc<[u8]>,
- )],
+ docs: &[(String, std::sync::Arc<serde_json::Value>)],
) -> Result<Vec<SeqNo>> {
if docs.is_empty() {
return Ok(Vec::new());
@@ -3062,25 +3184,13 @@ impl IndexStore {
let mut seq_nos: Vec<SeqNo> = Vec::with_capacity(docs.len());
// Estimate total frame size: per-doc overhead ~80 bytes + source
- let est_total: usize = docs
- .iter()
- .map(|(id, _, sb)| id.len() + sb.len() + 100)
- .sum();
+ let est_total: usize = docs.iter().map(|(id, _source)| id.len() + 600).sum();
let mut frames: Vec<u8> = Vec::with_capacity(est_total);
- for (i, (doc_id, source, source_bytes)) in docs.iter().enumerate() {
+ for (i, (doc_id, source)) in docs.iter().enumerate() {
let seq_no = start_seq + i as u64;
seq_nos.push(seq_no);
- let bytes_to_write: std::borrow::Cow<[u8]> = if !source_bytes.is_empty() {
- std::borrow::Cow::Borrowed(source_bytes.as_ref())
- } else {
- match serde_json::to_vec(source.as_ref()) {
- Ok(v) => std::borrow::Cow::Owned(v),
- Err(_) => std::borrow::Cow::Owned(b"null".to_vec()),
- }
- };
-
// Build JSON envelope directly
let payload_start = frames.len();
// Reserve space for entry_len (4 bytes) + seq_no (8) + op (1)
@@ -3107,7 +3217,7 @@ impl IndexStore {
frames.extend_from_slice(doc_id.as_bytes());
}
frames.extend_from_slice(br#"","source":"#);
- frames.extend_from_slice(&bytes_to_write);
+ serde_json::to_writer(&mut frames, source.as_ref())?;
frames.extend_from_slice(b"}}");
let payload_end = frames.len();
@@ -3198,7 +3308,7 @@ impl IndexStore {
// lookups before flush resolve to `IN_MEMORY_SEGMENT_ID`.
// This is the only per-doc side effect this method has
// outside the WAL itself.
- for (i, (doc_id, _source, _bytes)) in docs.iter().enumerate() {
+ for (i, (doc_id, _source)) in docs.iter().enumerate() {
let seq_no = seq_nos[i];
self.version_map
.set(doc_id, seq_no, IN_MEMORY_SEGMENT_ID, false);
@@ -3225,6 +3335,120 @@ mod tests {
.unwrap()
}
+ fn wal_bytes(store: &IndexStore) -> Vec<(PathBuf, Vec<u8>)> {
+ let mut files: Vec<(PathBuf, Vec<u8>)> = Vec::new();
+ for entry in std::fs::read_dir(store.wal_dir()).unwrap() {
+ let path = entry.unwrap().path();
+ if path.is_file() {
+ files.push((
+ path.file_name().unwrap().into(),
+ std::fs::read(&path).unwrap(),
+ ));
+ }
+ }
+ files.sort_by(|a, b| a.0.cmp(&b.0));
+ files
+ }
+
+ #[test]
+ fn raw_wal_batch_rejects_entire_invalid_batch_without_side_effects() {
+ let invalid_payloads: &[&[u8]] = &[
+ br#"{"truncated":"#,
+ br#"{"valid":true} trailing"#,
+ &[0xff, 0xfe, 0xfd],
+ ];
+
+ for (case, invalid) in invalid_payloads.iter().enumerate() {
+ for bad_at in 0..3 {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ let before_seq = store.current_seq_no();
+ let before_wal = wal_bytes(&store);
+ let mut docs = vec![
+ (
+ "first".to_owned(),
+ Arc::<[u8]>::from(br#"{"v":1}"#.as_slice()),
+ ),
+ (
+ "middle".to_owned(),
+ Arc::<[u8]>::from(br#"{"v":2}"#.as_slice()),
+ ),
+ (
+ "last".to_owned(),
+ Arc::<[u8]>::from(br#"{"v":3}"#.as_slice()),
+ ),
+ ];
+ docs[bad_at].1 = Arc::<[u8]>::from(*invalid);
+
+ let error = IndexStore::validate_raw_batch(docs.clone()).unwrap_err();
+ assert!(
+ matches!(
+ error,
+ StorageError::RawBatchValidation {
+ ref doc_id,
+ position,
+ ..
+ } if doc_id == &docs[bad_at].0 && position == bad_at + 1
+ ),
+ "case {case}, position {bad_at}: {error:?}"
+ );
+ let rendered = error.to_string();
+ assert!(rendered.contains("Fix:"));
+ assert!(rendered.contains("Related help:"));
+ assert_eq!(store.current_seq_no(), before_seq);
+ assert_eq!(wal_bytes(&store), before_wal);
+ for (id, _) in &docs {
+ assert!(store.version_map.get(id).is_none(), "{id} became visible");
+ }
+
+ drop(store);
+ let reopened = open_test_store(dir.path());
+ assert_eq!(reopened.current_seq_no(), before_seq);
+ for (id, _) in &docs {
+ assert!(
+ reopened.version_map.get(id).is_none(),
+ "{id} appeared after replay"
+ );
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn raw_wal_batch_accepts_complete_scalar_object_and_array_values() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ let docs = vec![
+ ("scalar".to_owned(), Arc::<[u8]>::from(b"42".as_slice())),
+ (
+ "object".to_owned(),
+ Arc::<[u8]>::from(br#"{"answer":42}"#.as_slice()),
+ ),
+ (
+ "array".to_owned(),
+ Arc::<[u8]>::from(br#"[1,2,3]"#.as_slice()),
+ ),
+ ];
+ let start_seq = store.current_seq_no();
+ let validated = IndexStore::validate_raw_batch(docs.clone()).unwrap();
+ assert_eq!(
+ store.wal_append_batch_raw(&validated).unwrap(),
+ vec![start_seq, start_seq + 1, start_seq + 2]
+ );
+ for (id, _) in &docs {
+ assert!(store.version_map.get(id).is_some(), "{id} not visible");
+ }
+ drop(store);
+
+ let reopened = open_test_store(dir.path());
+ for (id, _) in &docs {
+ assert!(
+ reopened.version_map.get(id).is_some(),
+ "{id} missing after replay"
+ );
+ }
+ }
+
/// RC4 W1 #8 regression — bulk-during-flush + kill -9 must lose ZERO
/// acked writes.
///
diff --git a/engine/crates/xerj-storage/src/lib.rs b/engine/crates/xerj-storage/src/lib.rs
index dab33f7e7..51d766ff5 100644
--- a/engine/crates/xerj-storage/src/lib.rs
+++ b/engine/crates/xerj-storage/src/lib.rs
@@ -38,7 +38,7 @@ pub use backend::{FileMetadata, LocalFsBackend, S3Backend, StorageBackend};
pub use cache::SegmentCache;
pub use index_store::{
DrainedMemtable, FsckReport, FsckSectionReport, FsckSegmentReport, IndexSnapshot, IndexStore,
- IndexStoreConfig, StorageMode,
+ IndexStoreConfig, RawJsonDoc, StorageMode, ValidatedRawBatch,
};
// `MergeExecutor` is intentionally NOT re-exported here: it is a
// `#[doc(hidden)]`, storage-crate-only test helper with real footguns
@@ -98,6 +98,17 @@ pub enum StorageError {
#[error("Serialization error: {0}")]
Serialization(#[from] serde_json::Error),
+ #[error(
+ "invalid raw JSON for document [{doc_id}] at one-based batch position {position}: \
+ {reason}. Fix: replace that document body with exactly one complete UTF-8 JSON value, \
+ then retry the batch. Related help: `xerj index --help` and https://xerj.org/llms.txt"
+ )]
+ RawBatchValidation {
+ doc_id: String,
+ position: usize,
+ reason: String,
+ },
+
#[error("Merge aborted: {0}")]
MergeAborted(String),