AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(bulk): validate index-action doc JSON on the turbo-raw path (per-item 400, ES-shaped)
xerj-org0a6b4e9d8be9
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex b738724f4..27496643b 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -10607,7 +10607,6 @@ async fn process_bulk_body(// Map common per-item error phrases to their ES exception// type names so clients can match on `error.type`.let error_type = if e.starts_with("if _id is specified")- || e.contains("invalid document JSON")|| e.contains("missing document body")|| e.contains("unknown action type")|| e.contains("no write index is defined")@@ -10618,7 +10617,13 @@ async fn process_bulk_body("version_conflict_engine_exception"} else if e.contains("index not found") || e.starts_with("no such index") {"index_not_found_exception"- } else if e.contains("dynamic template") || e.contains("failed to parse field") {+ } else if e.contains("dynamic template")+ || e.contains("failed to parse field")+ // Malformed doc-body JSON on a bulk item: ES 8.13+ // reports `document_parsing_exception` ("[1:1] failed+ // to parse: ..."), not illegal_argument.+ || e.contains("invalid document JSON")+ {"document_parsing_exception"} else {"engine_exception"diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex abe5df2ed..080a76aaf 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -537,7 +537,63 @@ pub async fn process_bulk_with_opts(// actions still need the parsed Value because they// take different code paths below — but `index` is// the dominant one on ingest benchmarks.+ //+ // The bytes are still VALIDATED here (syntax + top-level+ // object) via `IgnoredAny`, which walks the JSON without+ // building a Value tree — no per-key String allocations,+ // so the turbo win (skipping the tree build) is kept.+ // Without this, a malformed doc line was deferred to the+ // background parse whose `.unwrap_or({})` stored the doc+ // as EMPTY `{}` while the response said 201/errors:false+ // — silent data loss. ES rejects the item with a 400+ // `document_parsing_exception` instead; mirror that.if action_type == "index" {+ if let Err(e) =+ serde_json::from_slice::<serde::de::IgnoredAny>(dl.as_bytes())+ {+ return ParseOutcome::Err(+ BulkItemResult {+ action: action_type.to_string(),+ index: target_index,+ id: doc_id.unwrap_or_default(),+ status: 400,+ result: None,+ error: Some(format!(+ "failed to parse: invalid document JSON: {e}"+ )),+ get_source: None,+ },+ *item_index,+ );+ }+ // ES requires the doc body to be a JSON object;+ // a bare array / scalar is rejected per-item+ // (ES 8.13: `not_x_content_exception`; we report+ // the same condition as a 400 parse error).+ let is_object = dl+ .as_bytes()+ .iter()+ .find(|b| !b" \t\r\n".contains(b))+ .map(|b| *b == b'{')+ .unwrap_or(false);+ if !is_object {+ return ParseOutcome::Err(+ BulkItemResult {+ action: action_type.to_string(),+ index: target_index,+ id: doc_id.unwrap_or_default(),+ status: 400,+ result: None,+ error: Some(+ "failed to parse: invalid document JSON: \+ document body must be a JSON object"+ .to_string(),+ ),+ get_source: None,+ },+ *item_index,+ );+ }let bytes: std::sync::Arc<[u8]> = std::sync::Arc::from(dl.as_bytes());(None, bytes)} else {@@ -992,8 +1048,26 @@ pub async fn process_bulk_with_opts(// WAL write, storage memtable (_source), AND FTS memtable// (search + aggregations) in one call — no separate raw insert.for (item_idx, doc_id, doc_bytes) in batch {- let source = serde_json::from_slice::<Value>(&doc_bytes)- .unwrap_or(Value::Object(serde_json::Map::new()));+ // The parse stage already validated these bytes (syntax ++ // object-ness), so this parse cannot fail in practice — but+ // never fall back to indexing an EMPTY doc: a `{}` here would+ // be silent data loss reported as success. Fail the item loud.+ let source = match serde_json::from_slice::<Value>(&doc_bytes) {+ Ok(v) => v,+ Err(e) => {+ items[item_idx] = Some(BulkItemResult {+ action: "index".into(),+ index: index_name.clone(),+ id: doc_id,+ status: 400,+ result: None,+ error: Some(format!("failed to parse: invalid document JSON: {e}")),+ get_source: None,+ });+ errors = true;+ continue;+ }+ };tracing::debug!(doc_id = doc_id.as_str(),source = %source,diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rsindex 4a7cc384b..3c71ddc99 100644--- a/engine/crates/xerj-engine/tests/integration.rs+++ b/engine/crates/xerj-engine/tests/integration.rs@@ -4653,3 +4653,63 @@ async fn test_fused_memtable_total_matches_shortcut_recount() {);}}++// ═════════════════════════════════════════════════════════════════════════════+// RC4 Stream B regressions — silent wrong data+// ═════════════════════════════════════════════════════════════════════════════++/// Blocker 3: a malformed doc line under a bulk `index` action used to be+/// stored as EMPTY `{}` with `201 / errors:false` (the turbo-raw path+/// deferred the parse and `.unwrap_or({})`-ed the failure). ES rejects the+/// item with a per-item 400 `document_parsing_exception`; the engine must+/// reject it per-item and must not store anything.+#[tokio::test]+async fn test_bulk_index_malformed_doc_rejected_per_item() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ let body = concat!(+ "{\"index\":{\"_index\":\"b3\",\"_id\":\"bad\"}}\n",+ "{\"broken json here\n",+ "{\"index\":{\"_index\":\"b3\",\"_id\":\"good\"}}\n",+ "{\"v\":\"ok\"}\n",+ );+ let result = xerj_engine::bulk::process_bulk(&engine, None, body).await;+ assert!(result.errors, "bulk response must flag errors:true");+ assert_eq!(result.items.len(), 2);++ let bad = &result.items[0];+ assert_eq!(bad.status, 400, "malformed item must be a per-item 400");+ assert!(+ bad.error+ .as_deref()+ .unwrap_or("")+ .contains("invalid document JSON"),+ "error must say the document JSON is invalid, got: {:?}",+ bad.error+ );++ let good = &result.items[1];+ assert_eq!(good.status, 201, "valid sibling item must still index");++ // The malformed doc must NOT be stored (it used to land as `{}`).+ let idx = engine.get_index("b3").unwrap();+ let all = idx+ .search(&make_search(json!({"match_all": {}})))+ .await+ .unwrap();+ assert_eq!(all.total.value, 1, "only the valid doc may be stored");+ assert_eq!(all.hits[0].id, "good");++ // Valid JSON that is NOT an object is rejected too (ES errors on it).+ let body2 = "{\"index\":{\"_index\":\"b3\",\"_id\":\"arr\"}}\n[1,2,3]\n";+ let r2 = xerj_engine::bulk::process_bulk(&engine, None, body2).await;+ assert!(r2.errors);+ assert_eq!(r2.items[0].status, 400);+ let all2 = idx+ .search(&make_search(json!({"match_all": {}})))+ .await+ .unwrap();+ assert_eq!(all2.total.value, 1, "non-object body must not be stored");+}+