Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(bulk): validate index-action doc JSON on the turbo-raw path (per-item 400, ES-shaped)

xerj-org0a6b4e9d8be9
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index b738724f4..27496643b 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -10607,7 +10607,6 @@ async fn process_bulk_body(
// Map common per-item error phrases to their ES exception
// type names so clients can match on `error.type`.
let error_type = if e.starts_with("if _id is specified")
- || e.contains("invalid document JSON")
|| e.contains("missing document body")
|| e.contains("unknown action type")
|| e.contains("no write index is defined")
@@ -10618,7 +10617,13 @@ async fn process_bulk_body(
"version_conflict_engine_exception"
} else if e.contains("index not found") || e.starts_with("no such index") {
"index_not_found_exception"
- } else if e.contains("dynamic template") || e.contains("failed to parse field") {
+ } else if e.contains("dynamic template")
+ || e.contains("failed to parse field")
+ // Malformed doc-body JSON on a bulk item: ES 8.13
+ // reports `document_parsing_exception` ("[1:1] failed
+ // to parse: ..."), not illegal_argument.
+ || e.contains("invalid document JSON")
+ {
"document_parsing_exception"
} else {
"engine_exception"
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index abe5df2ed..080a76aaf 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -537,7 +537,63 @@ pub async fn process_bulk_with_opts(
// actions still need the parsed Value because they
// take different code paths below — but `index` is
// the dominant one on ingest benchmarks.
+ //
+ // The bytes are still VALIDATED here (syntax + top-level
+ // object) via `IgnoredAny`, which walks the JSON without
+ // building a Value tree — no per-key String allocations,
+ // so the turbo win (skipping the tree build) is kept.
+ // Without this, a malformed doc line was deferred to the
+ // background parse whose `.unwrap_or({})` stored the doc
+ // as EMPTY `{}` while the response said 201/errors:false
+ // — silent data loss. ES rejects the item with a 400
+ // `document_parsing_exception` instead; mirror that.
if action_type == "index" {
+ if let Err(e) =
+ serde_json::from_slice::<serde::de::IgnoredAny>(dl.as_bytes())
+ {
+ return ParseOutcome::Err(
+ BulkItemResult {
+ action: action_type.to_string(),
+ index: target_index,
+ id: doc_id.unwrap_or_default(),
+ status: 400,
+ result: None,
+ error: Some(format!(
+ "failed to parse: invalid document JSON: {e}"
+ )),
+ get_source: None,
+ },
+ *item_index,
+ );
+ }
+ // ES requires the doc body to be a JSON object;
+ // a bare array / scalar is rejected per-item
+ // (ES 8.13: `not_x_content_exception`; we report
+ // the same condition as a 400 parse error).
+ let is_object = dl
+ .as_bytes()
+ .iter()
+ .find(|b| !b" \t\r\n".contains(b))
+ .map(|b| *b == b'{')
+ .unwrap_or(false);
+ if !is_object {
+ return ParseOutcome::Err(
+ BulkItemResult {
+ action: action_type.to_string(),
+ index: target_index,
+ id: doc_id.unwrap_or_default(),
+ status: 400,
+ result: None,
+ error: Some(
+ "failed to parse: invalid document JSON: \
+ document body must be a JSON object"
+ .to_string(),
+ ),
+ get_source: None,
+ },
+ *item_index,
+ );
+ }
let bytes: std::sync::Arc<[u8]> = std::sync::Arc::from(dl.as_bytes());
(None, bytes)
} else {
@@ -992,8 +1048,26 @@ pub async fn process_bulk_with_opts(
// WAL write, storage memtable (_source), AND FTS memtable
// (search + aggregations) in one call — no separate raw insert.
for (item_idx, doc_id, doc_bytes) in batch {
- let source = serde_json::from_slice::<Value>(&doc_bytes)
- .unwrap_or(Value::Object(serde_json::Map::new()));
+ // The parse stage already validated these bytes (syntax +
+ // object-ness), so this parse cannot fail in practice — but
+ // never fall back to indexing an EMPTY doc: a `{}` here would
+ // be silent data loss reported as success. Fail the item loud.
+ let source = match serde_json::from_slice::<Value>(&doc_bytes) {
+ Ok(v) => v,
+ Err(e) => {
+ items[item_idx] = Some(BulkItemResult {
+ action: "index".into(),
+ index: index_name.clone(),
+ id: doc_id,
+ status: 400,
+ result: None,
+ error: Some(format!("failed to parse: invalid document JSON: {e}")),
+ get_source: None,
+ });
+ errors = true;
+ continue;
+ }
+ };
tracing::debug!(
doc_id = doc_id.as_str(),
source = %source,
diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rs
index 4a7cc384b..3c71ddc99 100644
--- a/engine/crates/xerj-engine/tests/integration.rs
+++ b/engine/crates/xerj-engine/tests/integration.rs
@@ -4653,3 +4653,63 @@ async fn test_fused_memtable_total_matches_shortcut_recount() {
);
}
}
+
+// ═════════════════════════════════════════════════════════════════════════════
+// RC4 Stream B regressions — silent wrong data
+// ═════════════════════════════════════════════════════════════════════════════
+
+/// Blocker 3: a malformed doc line under a bulk `index` action used to be
+/// stored as EMPTY `{}` with `201 / errors:false` (the turbo-raw path
+/// deferred the parse and `.unwrap_or({})`-ed the failure). ES rejects the
+/// item with a per-item 400 `document_parsing_exception`; the engine must
+/// reject it per-item and must not store anything.
+#[tokio::test]
+async fn test_bulk_index_malformed_doc_rejected_per_item() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ let body = concat!(
+ "{\"index\":{\"_index\":\"b3\",\"_id\":\"bad\"}}\n",
+ "{\"broken json here\n",
+ "{\"index\":{\"_index\":\"b3\",\"_id\":\"good\"}}\n",
+ "{\"v\":\"ok\"}\n",
+ );
+ let result = xerj_engine::bulk::process_bulk(&engine, None, body).await;
+ assert!(result.errors, "bulk response must flag errors:true");
+ assert_eq!(result.items.len(), 2);
+
+ let bad = &result.items[0];
+ assert_eq!(bad.status, 400, "malformed item must be a per-item 400");
+ assert!(
+ bad.error
+ .as_deref()
+ .unwrap_or("")
+ .contains("invalid document JSON"),
+ "error must say the document JSON is invalid, got: {:?}",
+ bad.error
+ );
+
+ let good = &result.items[1];
+ assert_eq!(good.status, 201, "valid sibling item must still index");
+
+ // The malformed doc must NOT be stored (it used to land as `{}`).
+ let idx = engine.get_index("b3").unwrap();
+ let all = idx
+ .search(&make_search(json!({"match_all": {}})))
+ .await
+ .unwrap();
+ assert_eq!(all.total.value, 1, "only the valid doc may be stored");
+ assert_eq!(all.hits[0].id, "good");
+
+ // Valid JSON that is NOT an object is rejected too (ES errors on it).
+ let body2 = "{\"index\":{\"_index\":\"b3\",\"_id\":\"arr\"}}\n[1,2,3]\n";
+ let r2 = xerj_engine::bulk::process_bulk(&engine, None, body2).await;
+ assert!(r2.errors);
+ assert_eq!(r2.items[0].status, 400);
+ let all2 = idx
+ .search(&make_search(json!({"match_all": {}})))
+ .await
+ .unwrap();
+ assert_eq!(all2.total.value, 1, "non-object body must not be stored");
+}
+