AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(embedding): mode=proxy fails closed; batch_size reaches the proxy
xerj-orgccf6351e60d0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f57ed00eb..eb6bb7508 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- **`embedding.mode = "proxy"` with an unusable endpoint silently ran+ lexical** (issue [#1189](https://github.com/xerj-org/xerj/issues/1189)) —+ an empty `default_endpoint` or a typo'd URL (`htp://…`) produced a+ startup WARN and a lexical embedder; indexing and semantic search then+ "succeeded" while writing lexical feature-hash vectors into the index, and+ `/v1/embedding/identity` reported `backend: lexical` for a config that+ said `proxy`. An explicit `mode = "proxy"` now fails closed at startup+ with an actionable error in both the embedder and the identity path+ (including the reporter's scheme typo, which carries the underlying+ `http or https` message); the auto path (endpoint set, mode not pinned)+ keeps the documented warn-plus-lexical fallback. `xerj autoindex` now+ names the live embedding backend in its run output so a degraded backend+ is visible where the work happens.++- **`embedding.batch_size` was never read** (issue+ [#1190](https://github.com/xerj-org/xerj/issues/1190)) — the setting+ documented "Maximum documents per embedding API call (default: 64)" but+ no code read it: one `_bulk` of 20 documents left the proxy as a single+ request with 20 inputs, and an autoindex run was observed sending a+ 192-input request. Providers cap the request `input` array (OpenAI 2048,+ self-hosted gateways often far lower), so a large bulk could fail on+ request shape, not load. The proxy now splits every batch into+ concurrent requests of at most `batch_size` inputs (each with its own+ retry/backoff and concurrency permit; the semaphore now limits in-flight+ HTTP requests, which was its purpose) and reassembles the vectors in+ input order. Verified by a mock-endpoint test: 10 texts with+ `batch_size: 4` leave as three requests of 4/4/2 inputs and the returned+ vectors keep input order across chunk boundaries.+- **Long BM25 queries silently fell back to per-segment IDF/avgdl**(issue [#1186](https://github.com/xerj-org/xerj/issues/1186)) — theindex-wide statistics pre-pass (#188) budgeted `(fields + terms) ×diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 29cc0d6ac..8a76c39a4 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -7242,6 +7242,7 @@ dependencies = ["candle-core","candle-nn","candle-transformers",+ "futures-util","hf-hub","ort","reqwest",diff --git a/engine/crates/xerj-ai/Cargo.toml b/engine/crates/xerj-ai/Cargo.tomlindex 14cbb9348..60191a0d0 100644--- a/engine/crates/xerj-ai/Cargo.toml+++ b/engine/crates/xerj-ai/Cargo.toml@@ -15,6 +15,9 @@ tokio.workspace = truetracing.workspace = truethiserror.workspace = trueanyhow.workspace = true+# Only for try_join_all in the embedding proxy's batch chunking (#1190) —+# already in the tree via xerj-engine, no new crate for the workspace.+futures-util = "0.3"# Built-in neural embedder (candle + MiniLM), download-on-first-run.# Feature-gated so the default build stays dependency-light and thediff --git a/engine/crates/xerj-ai/src/embed.rs b/engine/crates/xerj-ai/src/embed.rsindex 12eaa2bd0..9da5ce92c 100644--- a/engine/crates/xerj-ai/src/embed.rs+++ b/engine/crates/xerj-ai/src/embed.rs@@ -39,6 +39,12 @@ pub struct EmbeddingProxyConfig {/// Maximum number of retries on transient failures.#[serde(default = "default_max_retries")]pub max_retries: u32,+ /// Maximum input texts per request (#1190). Providers cap+ /// `input` array length (OpenAI: 2048, many self-hosted ones far lower);+ /// one `_bulk` of N docs used to leave here as a single N-input request.+ /// Default 64 mirrors `embedding.batch_size`.+ #[serde(default = "default_batch_size")]+ pub batch_size: usize,}fn default_timeout_secs() -> u64 {@@ -50,6 +56,9 @@ fn default_max_concurrent() -> usize {fn default_max_retries() -> u32 {3}+fn default_batch_size() -> usize {+ 64+}impl EmbeddingProxyConfig {pub fn new(endpoint: impl Into<String>, model: impl Into<String>) -> Self {@@ -60,6 +69,7 @@ impl EmbeddingProxyConfig {timeout_secs: default_timeout_secs(),max_concurrent: default_max_concurrent(),max_retries: default_max_retries(),+ batch_size: default_batch_size(),}}@@ -178,14 +188,29 @@ impl EmbeddingProxy {return Ok(vec![]);}- let _permit = self- .semaphore- .acquire()- .await- .map_err(|e| XerjError::embedding(format!("semaphore: {e}")))?;+ // #1190: split into requests of at most `batch_size` inputs. The+ // semaphore permit moved from here to per-chunk: it limits in-flight+ // HTTP requests (its actual purpose), not logical batches.+ let chunk_len = self.config.batch_size.max(1);+ let chunks: Vec<&[String]> = if texts.len() <= chunk_len {+ vec![texts.as_slice()]+ } else {+ texts.chunks(chunk_len).collect()+ };- let result = self.send_with_retry(&texts, model).await?;- Ok(result)+ let mut futs = Vec::with_capacity(chunks.len());+ for chunk in &chunks {+ futs.push(async {+ let _permit = self+ .semaphore+ .acquire()+ .await+ .map_err(|e| XerjError::embedding(format!("semaphore: {e}")))?;+ self.send_with_retry(chunk, model).await+ });+ }+ let results = futures_util::future::try_join_all(futs).await?;+ Ok(results.into_iter().flatten().collect())}async fn send_with_retry(&self, texts: &[String], model: &str) -> Result<Vec<Vec<f32>>> {@@ -311,6 +336,7 @@ mod tests {assert_eq!(cfg.timeout_secs, 30);assert_eq!(cfg.max_concurrent, 4);assert_eq!(cfg.max_retries, 3);+ assert_eq!(cfg.batch_size, 64);assert!(cfg.api_key.is_none());}@@ -398,4 +424,76 @@ mod tests {);drop(handle); // listener already closed by loop exit}++ // #1190: one embed_batch of 10 texts with batch_size 4 must leave as+ // requests of at most 4 inputs, and the returned vectors stay in input+ // order across chunk boundaries.+ #[tokio::test]+ async fn batch_size_splits_requests_and_preserves_order() {+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let addr = listener.local_addr().unwrap();+ let handle = std::thread::spawn(move || {+ use std::io::{Read, Write};+ let mut input_sizes = vec![];+ for _ in 0..3 {+ let Ok((mut s, _)) = listener.accept() else { break };+ // Fixed single read, like permanent_4xx above: the client+ // holds the connection open awaiting the response, so+ // read_to_end would deadlock.+ let mut buf = [0u8; 8192];+ let n = s.read(&mut buf).unwrap();+ let body = String::from_utf8_lossy(&buf[..n]);+ let payload = body+ .split("\r\n\r\n")+ .nth(1)+ .unwrap_or_default()+ .to_string();+ let parsed: serde_json::Value = serde_json::from_str(&payload).unwrap();+ let n = parsed["input"].as_array().unwrap().len();+ input_sizes.push(n);+ // Embedding for input i is the vector [i, i, ...]: order is+ // observable in the output itself.+ let data: Vec<String> = parsed["input"]+ .as_array()+ .unwrap()+ .iter()+ .enumerate()+ .map(|(i, _)| {+ // The provider sees per-request indices; recover the+ // global index from the text itself ("t<global>").+ let text = parsed["input"][i].as_str().unwrap();+ let g: f32 = text[1..].parse().unwrap();+ format!(+ "{{\"object\":\"embedding\",\"index\":{i},\"embedding\":[{g},{g}]}}"+ )+ })+ .collect();+ let resp_body = format!(+ "{{\"object\":\"list\",\"data\":[{}]}}",+ data.join(",")+ );+ let resp = format!(+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",+ resp_body.len(),+ resp_body+ );+ let _ = s.write_all(resp.as_bytes());+ }+ input_sizes+ });++ let mut cfg = EmbeddingProxyConfig::new(format!("http://{addr}/v1/embeddings"), "m");+ cfg.batch_size = 4;+ let proxy = EmbeddingProxy::new(cfg).unwrap();++ let texts: Vec<String> = (0..10).map(|i| format!("t{i}")).collect();+ let out = proxy.embed_batch(texts).await.unwrap();++ assert_eq!(out.len(), 10);+ for (i, v) in out.iter().enumerate() {+ assert_eq!(v, &vec![i as f32, i as f32], "output {i} out of order");+ }+ let sizes = handle.join().unwrap();+ assert_eq!(sizes, vec![4, 4, 2], "requests must respect batch_size");+ }}diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 90566d8c1..2ae2879e6 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -6568,6 +6568,14 @@ fn run_index_report_inner(identity.resumable,identity.non_resumable_reason.as_deref(),)?;+ // #1189: name the live backend in the run output. A server started+ // with mode=proxy that had silently degraded to lexical was+ // indistinguishable here until the vectors came back wrong.+ pr.note(&format!(+ "semantic autoindex embedding backend: {} ({}resumable)",+ identity.backend,+ if identity.resumable { "" } else { "non-" }+ ));}// ── create indices with explicit mappings ────────────────────────────diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rsindex 704a02d7e..0c58f6e9e 100644--- a/engine/crates/xerj-common/src/config.rs+++ b/engine/crates/xerj-common/src/config.rs@@ -1354,9 +1354,13 @@ impl Default for LogsConfig {/// to lexical./// * `"proxy"` — call an external OpenAI-compatible `/v1/embeddings`/// endpoint ([`default_endpoint`]). Lets customers plug in ANY embedding-/// model / provider they already run.+/// model / provider they already run. An explicit `mode = "proxy"` with a+/// missing or unusable endpoint is a startup ERROR (#1189): silently+/// running lexical instead would write the wrong vectors into the index+/// while every embedding request appears to succeed./// * `"auto"` (default) — use the proxy when [`default_endpoint`] is set,-/// otherwise lexical. This preserves the historical behavior exactly.+/// otherwise lexical; if the endpoint fails to initialize, fall back to+/// lexical with a warning. The fallback exists ONLY on this path (#1189).////// **20 settings.**#[derive(Debug, Clone, Serialize, Deserialize)]@@ -1369,7 +1373,9 @@ pub struct EmbeddingConfig {pub default_endpoint: String,/// Model name to request from the endpoint (default: `""`).pub default_model: String,- /// Maximum documents per embedding API call (default: `64`).+ /// Maximum documents per embedding API call (default: `64`). Providers+ /// cap the request `input` array (often far below 64); the proxy splits+ /// larger batches into consecutive requests of at most this size (#1190).pub batch_size: usize,/// HTTP timeout for embedding requests in ms (default: `5000`).pub timeout_ms: u64,diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 698df1066..be6794c7d 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -35532,6 +35532,11 @@ pub(crate) fn embedding_execution_identity(const CONTRACT: &str = "semantic_text-derived-vector.v1";let requested = cfg.mode.trim().to_ascii_lowercase();+ // Explicit mode=proxy with an unusable endpoint is a configuration error,+ // not a lexical degrade (#1189): the identity must report what the+ // embedder would actually run, and after the make_embedder fix that+ // configuration no longer starts. Auto (any non-pinned mode) keeps the+ // historical fallback to lexical.let mut backend = match requested.as_str() {"neural" => {#[cfg(feature = "neural")]@@ -35545,13 +35550,25 @@ pub(crate) fn embedding_execution_identity(}"onnx-experimental" => "onnx-experimental","lexical" => "lexical",- "proxy" if cfg.default_endpoint.is_empty() => "lexical",+ "proxy" if cfg.default_endpoint.is_empty() => {+ return Err(EngineError::Common(xerj_common::XerjError::embedding(+ "embedding.mode=proxy requires embedding.default_endpoint to be set; \+ remove mode=proxy (use auto) if a lexical fallback is wanted",+ )));+ }"proxy" => "proxy",_ if cfg.default_endpoint.is_empty() => "lexical",_ => "proxy",};- if backend == "proxy" && !proxy_initializes(cfg) {- backend = "lexical";+ if backend == "proxy" {+ if let Err(e) = xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)) {+ if requested == "proxy" {+ return Err(EngineError::Common(xerj_common::XerjError::embedding(+ format!("embedding.mode=proxy failed to initialize: {e}"),+ )));+ }+ backend = "lexical";+ }}// Only report a width for the backends whose width this server actually// pins. `neural` reads it from the resolved model's `config.json`@@ -35655,7 +35672,10 @@ fn lexical_algorithm_probe_sha256() -> String {format!("{:x}", digest.finalize())}-fn proxy_config(+/// Build the proxy client config from engine embedding settings. Public so+/// `xerj-server` runs the same #1189 boot validation the engine runs at+/// index-create, instead of duplicating the mapping.+pub fn proxy_config(cfg: &xerj_common::config::EmbeddingConfig,) -> xerj_ai::embed::EmbeddingProxyConfig {xerj_ai::embed::EmbeddingProxyConfig {@@ -35665,12 +35685,11 @@ fn proxy_config(timeout_secs: cfg.timeout_ms / 1000,max_concurrent: 4,max_retries: 3,+ batch_size: cfg.batch_size.max(1),}}-fn proxy_initializes(cfg: &xerj_common::config::EmbeddingConfig) -> bool {- xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)).is_ok()-}+/// Fail closed rather than mixing vectors produced by different model,/// tokenizer, pooling, or backend configurations after a restart/resume.@@ -35995,23 +36014,88 @@ mod embedding_identity_tests {assert!(!encoded.contains("private-alias"));assert!(!encoded.contains("dimensions"), "{encoded}");+ // #1189: an EXPLICIT mode=proxy with an unusable endpoint is a+ // configuration error, not a silent lexical degrade. All three+ // failures previously reported backend "lexical" while the operator+ // believed the node was embedding through their provider.for endpoint in ["", "not a url", "file:///private/model"] {- let fallback = xerj_common::config::EmbeddingConfig {+ let broken = xerj_common::config::EmbeddingConfig {mode: "proxy".into(),default_endpoint: endpoint.into(),..Default::default()};+ let err = embedding_execution_identity(&broken)+ .err()+ .expect("explicit proxy with an unusable endpoint must error");+ let msg = format!("{err}");+ assert!(+ msg.contains("embedding.mode=proxy"),+ "the error must name the setting: {msg}"+ );+ }++ // The AUTO path keeps the documented fallback: an unset or unknown+ // mode with a broken endpoint resolves to lexical, never errors.+ for mode in ["auto", "", "somewhere-in-between"] {+ let fallback = xerj_common::config::EmbeddingConfig {+ mode: mode.into(),+ default_endpoint: "not a url".into(),+ ..Default::default()+ };let identity = embedding_execution_identity(&fallback).unwrap();- assert_eq!(identity.backend, "lexical", "{endpoint}");- assert!(identity.resumable, "{endpoint}");+ assert_eq!(identity.backend, "lexical", "{mode}");+ assert!(identity.resumable, "{mode}");// The fallback really did become lexical, so the pinned lexical// width is the truthful answer here.assert_eq!(identity.dimensions,Some(xerj_ai::local::DEFAULT_DIMS),- "{endpoint}"+ "{mode}"+ );+ }+ }++ /// #1189: explicit mode=proxy fails closed at the embedder too, with the+ /// proxy's own error carried through; the auto path keeps the documented+ /// lexical fallback.+ #[test]+ fn explicit_proxy_mode_fails_closed_at_the_embedder() {+ for endpoint in ["", "htp://typo.example/v1", "file:///etc/hosts"] {+ let cfg = xerj_common::config::EmbeddingConfig {+ mode: "proxy".into(),+ default_endpoint: endpoint.into(),+ ..Default::default()+ };+ let err = make_embedder(&cfg)+ .err()+ .expect("explicit proxy with an unusable endpoint must error");+ let msg = format!("{err}");+ assert!(+ msg.contains("embedding.mode=proxy"),+ "must name the setting: {msg}");+ // The typo case carries the underlying scheme error so the+ // operator sees what is actually wrong with the URL.+ if endpoint.starts_with("htp://") {+ assert!(+ msg.contains("http or https"),+ "must carry the proxy's own init error: {msg}"+ );+ }}++ // Auto with the same broken endpoint still boots as lexical — that+ // fallback is the documented auto behavior and stays.+ let auto = xerj_common::config::EmbeddingConfig {+ mode: "auto".into(),+ default_endpoint: "htp://typo.example/v1".into(),+ ..Default::default()+ };+ make_embedder(&auto).unwrap();+ assert_eq!(+ embedding_execution_identity(&auto).unwrap().backend,+ "lexical"+ );}/// #522: the #434 guard (`validate_embedding_execution`) was unit-tested at@@ -36048,16 +36132,21 @@ mod embedding_identity_tests {"the refusal must name both the recorded and the current backend: {msg}");- // Over-refusal guard: a proxy config with NO endpoint falls back to the- // lexical embedder, so its resolved identity is unchanged and reopening- // must NOT be refused.+ // Over-refusal guard (#1189 update): a proxy config with NO endpoint+ // used to silently resolve to lexical and reopen cleanly. It is now a+ // configuration error at the identity level, so reopening fails with+ // the same actionable message instead of a confusing mismatch refusal.let proxy_without_endpoint = xerj_common::config::EmbeddingConfig {mode: "proxy".into(),default_endpoint: String::new(),..Default::default()};- validate_embedding_execution(index_dir, &proxy_without_endpoint, false, true)- .expect("a proxy that falls back to lexical must not be refused");+ let err = validate_embedding_execution(index_dir, &proxy_without_endpoint, false, true)+ .expect_err("proxy with no endpoint is a config error, not a lexical reopen");+ assert!(+ format!("{err}").contains("embedding.mode=proxy"),+ "must name the setting: {err}"+ );}/// #533: the proxy `default_endpoint` is part of the embedder identity. Same@@ -36742,7 +36831,8 @@ mod embedding_identity_tests {fn make_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> Result<xerj_ai::Embedder> {let mode = cfg.mode.trim().to_ascii_lowercase();- let want_proxy = mode == "proxy"+ let explicit_proxy = mode == "proxy";+ let want_proxy = explicit_proxy|| (mode != "neural" && mode != "lexical" && !cfg.default_endpoint.is_empty());if mode == "neural" {@@ -36754,22 +36844,35 @@ fn make_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> Result<xerj_ai::}if want_proxy {+ // Explicit mode=proxy fails closed (#1189): a warn plus lexical+ // fallback left the node "healthy" while every embedding request+ // silently wrote lexical vectors into the index. Only the auto path+ // (endpoint set but mode not pinned) keeps the documented fallback.if cfg.default_endpoint.is_empty() {- warn!("embedding.mode=proxy but embedding.default_endpoint is empty — falling back to lexical");+ if explicit_proxy {+ return Err(EngineError::Common(xerj_common::XerjError::embedding(+ "embedding.mode=proxy requires embedding.default_endpoint to be set; \+ remove mode=proxy (use auto) if a lexical fallback is wanted",+ )));+ }+ warn!("embedding.default_endpoint is empty — falling back to lexical");return Ok(xerj_ai::Embedder::lexical());}- return Ok(- match xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)) {- Ok(p) => {- info!(endpoint = %cfg.default_endpoint, "embedding backend: external proxy");- xerj_ai::Embedder::proxy(p)- }- Err(e) => {- warn!(error = %e, "embedding proxy init failed — falling back to lexical");- xerj_ai::Embedder::lexical()+ return match xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)) {+ Ok(p) => {+ info!(endpoint = %cfg.default_endpoint, "embedding backend: external proxy");+ Ok(xerj_ai::Embedder::proxy(p))+ }+ Err(e) => {+ if explicit_proxy {+ return Err(EngineError::Common(xerj_common::XerjError::embedding(format!(+ "embedding.mode=proxy failed to initialize: {e}"+ ))));}- },- );+ warn!(error = %e, "embedding proxy init failed — falling back to lexical");+ Ok(xerj_ai::Embedder::lexical())+ }+ };}Ok(xerj_ai::Embedder::lexical())diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex ab34be228..976b8c206 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -734,6 +734,24 @@ fn load_config(args: &CliArgs) -> Result<Config> {}}+ // #1189: an explicit mode=proxy is validated at boot, the same+ // fail-closed contract as onnx-experimental above. The old warn-plus-+ // lexical fallback started a "healthy" node whose embedding requests+ // silently wrote lexical vectors. Auto keeps its documented fallback.+ if cfg.embedding.mode.trim().eq_ignore_ascii_case("proxy") {+ if cfg.embedding.default_endpoint.trim().is_empty() {+ anyhow::bail!(+ "embedding.mode=proxy requires embedding.default_endpoint to be set; \+ remove mode=proxy (use auto) if a lexical fallback is wanted"+ );+ }+ if let Err(e) = xerj_ai::embed::EmbeddingProxy::new(xerj_engine::index::proxy_config(+ &cfg.embedding,+ )) {+ anyhow::bail!("embedding.mode=proxy failed to initialize: {e}");+ }+ }+// ES/OpenSearch wire-compatibility override: `--compat-distribution` /// `XERJ_COMPAT_DISTRIBUTION` (flag wins). Left unset, xerj auto-detects// per request from the caller's User-Agent instead — this is only for