AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(bulk): stop truncating an action-line _id at an escaped quote (#954)
xerj-org972d81aea440
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex 55044ba0e..06e92a063 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -564,6 +564,16 @@ pub async fn process_bulk_with_opts(j};let inner = &bytes[p + 1..inner_end];+ // #954: a backslash inside the action object can only be+ // part of a string escape in valid JSON, and find_field+ // below scans for the next raw '"' without unescaping —+ // `{"_id":"a\"b"}` would yield `a\` and desync the rest of+ // the line, collapsing distinct ids. Any escaped content+ // (_id, _index, op_type alike) opts out of the fast path;+ // the serde_json fallback below parses it correctly.+ if inner.contains(&b'\\') {+ break 'fast (None, None, None);+ }let find_field = |needle: &[u8]| -> Option<String> {// Look for `"_key":"value"` (quoted) or `"_key":NUM`// (numeric, e.g. `_id:1`) inside `inner`. ES bulkdiff --git a/engine/crates/xerj-engine/tests/bulk_escaped_id.rs b/engine/crates/xerj-engine/tests/bulk_escaped_id.rsnew file mode 100644index 000000000..776d6d621--- /dev/null+++ b/engine/crates/xerj-engine/tests/bulk_escaped_id.rs@@ -0,0 +1,131 @@+//! Issue #954: `_bulk` truncated an action-line `_id` at an escaped quote.+//!+//! The fast path that extracts `_id`/`_index`/`op_type` from an action line+//! scanned for the next raw `"` without unescaping, so `{"_id":"a\"b"}` parsed+//! as id `a\` and the line's remainder desynced — two documents with distinct+//! ids collapsed onto one, and the second write either overwrote the first or+//! 409'd as a duplicate create. The fix bails out of the fast path whenever+//! the action object contains a backslash (any escape) and lets the serde_json+//! fallback parse the line correctly.++use serde_json::json;+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::bulk::process_bulk;+use xerj_engine::Engine;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++fn seed(engine: &Engine, name: &str) {+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("body", FieldType::Text));+ engine.create_index(name, schema).expect("create index");+}++/// Distinct ids that differ only inside an escaped segment must stay distinct:+/// all items index (no duplicate-create 409), and a GET-by-id finds each body+/// under its own id. FAIL-BEFORE: the fast path truncates both quote-escaped+/// ids to `a\`, so the two collapse and one body shadows the other; the+/// `\uXXXX` id is stored as its literal 6 escape characters (backslash+/// u 0 0 e 9) instead of the character it names.+#[tokio::test]+async fn escaped_quote_ids_stay_distinct() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine, "t");++ // Raw NDJSON. JSON-unescaped, the three ids are `a"b`, `a"c` and `é`+ // (the last via a \uXXXX escape, which the fast path used to store+ // literally).+ let body = concat!(+ r#"{"index":{"_index":"t","_id":"a\"b"}}"#,+ "\n",+ r#"{"body":"first"}"#,+ "\n",+ r#"{"index":{"_index":"t","_id":"a\"c"}}"#,+ "\n",+ r#"{"body":"second"}"#,+ "\n",+ r#"{"index":{"_index":"t","_id":"\u00e9"}}"#,+ "\n",+ r#"{"body":"third"}"#,+ "\n",+ );+ let result = process_bulk(&engine, None, body).await;+ assert!(!result.errors, "bulk must not error: {:?}", result.items);+ assert_eq!(result.items.len(), 3);+ // The response must echo the FULL unescaped id, never the truncation.+ assert_eq!(result.items[0].id, "a\"b");+ assert_eq!(result.items[1].id, "a\"c");+ assert_eq!(result.items[2].id, "é");++ let idx = engine.get_index("t").expect("get index");+ // Rust-side: "a\"b" is a"b and "a\"c" is a"c — the JSON-unescaped ids.+ for (id, body_text) in [("a\"b", "first"), ("a\"c", "second"), ("é", "third")] {+ let doc = idx+ .get_document(id)+ .await+ .expect("lookup")+ .unwrap_or_else(|| panic!("document {id:?} must exist"));+ assert_eq!(+ doc.get("body").and_then(|b| b.as_str()),+ Some(body_text),+ "id {id:?} must hold its own body"+ );+ }+}++/// The escape opt-out must not change behaviour for ids WITHOUT escapes+/// (the fast path stays exact for the canonical case), and an invalid escape+/// sequence must surface as a per-item 400, never as a silent truncation.+#[tokio::test]+async fn plain_ids_unchanged_and_bad_escape_is_a_per_item_error() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine, "t");++ let body = concat!(+ r#"{"index":{"_index":"t","_id":"plain"}}"#,+ "\n",+ r#"{"body":"ok"}"#,+ "\n",+ );+ let result = process_bulk(&engine, None, body).await;+ assert!(+ !result.errors,+ "plain ids must keep working: {:?}",+ result.items+ );+ let idx = engine.get_index("t").expect("get index");+ assert!(idx.get_document("plain").await.expect("lookup").is_some());++ // `\x` is not a valid JSON escape: the serde fallback must reject the+ // action line as a 400 item with the parse error, not truncate `_id`.+ let body = concat!(+ r#"{"index":{"_index":"t","_id":"bad\x22id"}}"#,+ "\n",+ r#"{"body":"never"}"#,+ "\n",+ );+ let result = process_bulk(&engine, None, body).await;+ assert_eq!(result.items.len(), 1, "one action, one item");+ assert_eq!(result.items[0].status, 400, "invalid escape must 400");+ assert!(result.items[0].error.is_some());+ let all = idx+ .search(+ &xerj_query::parse_request(&json!({+ "query": { "match": { "body": "never" } }, "size": 10+ }))+ .expect("parse"),+ )+ .await+ .expect("search");+ assert_eq!(all.total.value, 0, "the rejected item must not be indexed");+}