AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): report an embedding width that contradicts the mapping as a per-item 400 (#2191) (#2217)
xerj-orgd6eaaa358c72
diff --git a/engine/crates/xerj-api/src/binary_protocol.rs b/engine/crates/xerj-api/src/binary_protocol.rsindex a0da45ef6..15d0c8526 100644--- a/engine/crates/xerj-api/src/binary_protocol.rs+++ b/engine/crates/xerj-api/src/binary_protocol.rs@@ -502,6 +502,11 @@ mod tests {/// A mock OpenAI-compatible `/v1/embeddings` endpoint on an ephemeral port.async fn mock_embedding_backend() -> (String, BackendCalls) {+ mock_embedding_backend_with_width(DIMS).await+ }++ /// The same mock, answering with vectors of `width` dimensions.+ async fn mock_embedding_backend_with_width(width: usize) -> (String, BackendCalls) {let calls = Arc::new(AtomicUsize::new(0));let widest_call = Arc::new(AtomicUsize::new(0));let texts = Arc::new(AtomicUsize::new(0));@@ -525,7 +530,7 @@ mod tests {w.fetch_max(n, Ordering::SeqCst);let data: Vec<Value> = (0..n).map(|i| {- let embedding: Vec<f64> = (0..DIMS)+ let embedding: Vec<f64> = (0..width).map(|d| ((i + d) % 7) as f64 / 7.0 + 0.1).collect();json!({ "index": i, "embedding": embedding })@@ -555,6 +560,30 @@ mod tests {/// An index with one `semantic_text`-shaped field, served by the binary/// protocol on an ephemeral port. The temp dir must outlive the engine.async fn node(endpoint: String) -> (std::net::SocketAddr, tempfile::TempDir) {+ let (config, engine, dir) = semantic_engine(endpoint);+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let state = Arc::new(crate::state::AppState::new(config, engine, metrics));+ let listener = TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind binary protocol");+ let addr = listener.local_addr().expect("binary protocol addr");+ tokio::spawn(async move {+ while let Ok((stream, _)) = listener.accept().await {+ tokio::spawn(handle_connection(stream, Arc::clone(&state)));+ }+ });+ (addr, dir)+ }++ /// An engine with a `notes` index holding one `semantic_text`-shaped+ /// field of `DIMS` dimensions, embedded by the proxy at `endpoint`.+ fn semantic_engine(+ endpoint: String,+ ) -> (+ xerj_common::config::Config,+ xerj_engine::Engine,+ tempfile::TempDir,+ ) {let dir = tempfile::tempdir().expect("tempdir");let mut config = xerj_common::config::Config::default();config.server.data_dir = dir.path().to_string_lossy().into_owned();@@ -563,7 +592,6 @@ mod tests {config.embedding.default_endpoint = endpoint;config.embedding.default_model = "mock-embed".into();- let metrics = xerj_common::metrics::Metrics::new().expect("metrics");let engine = xerj_engine::Engine::new(config.clone()).expect("engine");let mut schema = Schema::empty();@@ -581,18 +609,7 @@ mod tests {companion.options.similarity = Some("cosine".into());schema.add_field(companion).expect("companion field");engine.create_index("notes", schema).expect("create index");-- let state = Arc::new(crate::state::AppState::new(config, engine, metrics));- let listener = TcpListener::bind("127.0.0.1:0")- .await- .expect("bind binary protocol");- let addr = listener.local_addr().expect("binary protocol addr");- tokio::spawn(async move {- while let Ok((stream, _)) = listener.accept().await {- tokio::spawn(handle_connection(stream, Arc::clone(&state)));- }- });- (addr, dir)+ (config, engine, dir)}/// One framed request/response round trip on a fresh connection.@@ -709,4 +726,27 @@ mod tests {let missing = round_trip(addr, op::GET, json!({ "index": "notes", "id": "bad" })).await;assert_eq!(missing["found"], json!(false), "get response: {missing}");}++ /// #2191: a backend whose vector width contradicts the mapping fails the+ /// document with a 400 mapping error, not a retryable 500 backend error —+ /// the width is fixed, so a client retry loop could never succeed.+ #[tokio::test]+ async fn an_embedding_width_mismatch_is_a_per_item_400() {+ let (endpoint, _backend) = mock_embedding_backend_with_width(DIMS / 2).await;+ let (_config, engine, _dir) = semantic_engine(endpoint);+ let idx = engine.get_index("notes").expect("index");++ let results = idx+ .index_documents_batched(vec![(Some("d0".into()), json!({ "body": "a passage" }))])+ .await;+ assert_eq!(results.len(), 1);+ let error = results[0].as_ref().expect_err("a width mismatch must fail");+ let xerj_engine::EngineError::Common(common) = error else {+ panic!("expected a common error, got {error:?}");+ };+ assert_eq!(common.http_status(), 400, "{common}");+ let message = common.to_string();+ assert!(message.contains("mapping requires"), "{message}");+ assert!(message.contains("dimensions"), "{message}");+ }}diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex ea75b4ed8..e8c3ca799 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -19681,31 +19681,7 @@ async fn process_bulk_body(status: item.status,get,error: item.error.map(|e| {- // Map common per-item error phrases to their ES exception- // type names so clients can match on `error.type`.- let error_type = if e.starts_with("if _id is specified")- || e.contains("missing document body")- || e.contains("unknown action type")- || e.contains("no write index is defined")- || e.starts_with("pipeline with id")- {- "illegal_argument_exception"- } else if e.starts_with("version conflict") {- "version_conflict_engine_exception"- } else if e.contains("index not found") || e.starts_with("no such index") {- "index_not_found_exception"- } else if e.contains("dynamic template")- || e.contains("failed to parse field")- // Malformed doc-body JSON on a bulk item: ES 8.13- // reports `document_parsing_exception` ("[1:1] failed- // to parse: ..."), not illegal_argument.- || e.contains("invalid document JSON")- {- "document_parsing_exception"- } else {- "engine_exception"- }- .to_string();+ let error_type = bulk_item_error_type(&e).to_string();BulkItemError {error_type,reason: e,@@ -19852,6 +19828,72 @@ async fn process_bulk_body(r}+/// Map a per-item `_bulk` error reason to its ES exception type name so+/// clients can match on `error.type`.+fn bulk_item_error_type(e: &str) -> &'static str {+ if e.starts_with("if _id is specified")+ || e.contains("missing document body")+ || e.contains("unknown action type")+ || e.contains("no write index is defined")+ || e.starts_with("pipeline with id")+ {+ "illegal_argument_exception"+ } else if e.starts_with("version conflict") {+ "version_conflict_engine_exception"+ } else if e.contains("index not found") || e.starts_with("no such index") {+ "index_not_found_exception"+ } else if e.contains("dynamic template")+ || e.contains("failed to parse field")+ // Malformed doc-body JSON on a bulk item: ES 8.13+ // reports `document_parsing_exception` ("[1:1] failed+ // to parse: ..."), not illegal_argument.+ || e.contains("invalid document JSON")+ {+ "document_parsing_exception"+ } else if e.contains("embedding backend returned") && e.contains("mapping requires") {+ // #2191: the embedding backend's vector width contradicts the+ // semantic_text mapping. Retrying cannot help, so it is reported as a+ // mapping error, not a retryable `engine_exception`.+ "mapper_parsing_exception"+ } else {+ "engine_exception"+ }+}++#[cfg(test)]+mod bulk_item_error_type_tests {+ use super::bulk_item_error_type;++ /// #2191: an embedding width that contradicts the mapping is a+ /// `mapper_parsing_exception`, while a genuine backend fault stays an+ /// `engine_exception`.+ #[test]+ fn an_embedding_width_mismatch_is_a_mapper_parsing_exception() {+ assert_eq!(+ bulk_item_error_type(+ "invalid mapping: field [body]: embedding backend returned width 4, \+ mapping requires 8; the embedding backend's vector width is fixed"+ ),+ "mapper_parsing_exception"+ );+ assert_eq!(+ bulk_item_error_type(+ "invalid mapping: field [body]: embedding backend returned 1 vectors with \+ widths {4} for 1 texts; mapping requires width 8"+ ),+ "mapper_parsing_exception"+ );+ assert_eq!(+ bulk_item_error_type("semantic_text batch embed failed: connection refused"),+ "engine_exception"+ );+ assert_eq!(+ bulk_item_error_type("failed to parse field [n]"),+ "document_parsing_exception"+ );+ }+}+#[cfg(test)]mod malformed_bulk_route_tests {use super::*;diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex ce9a75849..5b1353f4d 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -17642,6 +17642,11 @@ impl Index {let mut outputs: Vec<Option<Vec<Vec<f32>>>> = vec![None; jobs.len()];let mut failures = pre_failures;let mut admission_rejections = vec![false; sources.len()];+ // #2191: a backend that answers with the wrong vector WIDTH is a+ // mapping contradiction, not a transient fault — retrying the same+ // document can never succeed. Those failures surface as a per-item+ // 400 (`invalid mapping`) instead of a retryable 500.+ let mut width_mismatches = vec![false; sources.len()];let embedder = self.embedder.read().await;if embedder.is_active() {@@ -17730,6 +17735,7 @@ impl Index {.iter().find(|vector| vector.len() != jobs[job_idx].dims){+ width_mismatches[jobs[job_idx].source_idx] = true;failures[jobs[job_idx].source_idx] = Some(format!("field [{}]: embedding backend returned width {}, mapping requires {}",jobs[job_idx].field,@@ -17773,6 +17779,10 @@ impl Index {.iter().map(Vec::len).collect::<std::collections::BTreeSet<_>>();+ // Only a width error when every text got a+ // vector; a short answer is a backend fault.+ width_mismatches[jobs[job_idx].source_idx] =+ v.len() == jobs[job_idx].texts.len();failures[jobs[job_idx].source_idx] = Some(format!("field [{}]: embedding backend returned {} vectors with widths {:?} for {} texts; mapping requires width {}",jobs[job_idx].field,@@ -17783,6 +17793,7 @@ impl Index {));}Err(e) => {+ width_mismatches[jobs[job_idx].source_idx] = false;admission_rejections[jobs[job_idx].source_idx] =is_embedding_admission_rejection(&e);failures[jobs[job_idx].source_idx] =@@ -17873,6 +17884,9 @@ impl Index {"semantic_text batch embed rejected before tokenization: {error}")),)),+ Some(error) if width_mismatches[idx] => {+ Err(EngineError::Common(semantic_width_mismatch_error(&error)))+ }Some(error) => Err(EngineError::Common(embed_backend_error(anyhow::anyhow!(error),"semantic_text batch embed failed",@@ -37732,6 +37746,18 @@ fn build_highlight_fragments(fragments}+/// #2191: the embedding backend answered with vectors whose width contradicts+/// the `semantic_text` field's mapped `dimensions`. The backend's width is+/// fixed, so a retry cannot succeed: report it as a mapping error (HTTP 400,+/// `mapper_parsing_exception` on `_bulk`) rather than a retryable backend 500.+fn semantic_width_mismatch_error(detail: &str) -> xerj_common::XerjError {+ xerj_common::XerjError::invalid_mapping(format!(+ "{detail}; the embedding backend's vector width is fixed, so retrying cannot \+ succeed: set the semantic_text field's `dimensions` to the backend's width, \+ or point the field at a backend that returns the mapped width"+ ))+}+/// Build the effective [`xerj_ai::Embedder`] from `Config.embedding`.////// Backend selection follows `embedding.mode`: