Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(embedding): mode=proxy fails closed; batch_size reaches the proxy

xerj-orgccf6351e60d0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f57ed00eb..eb6bb7508 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- **`embedding.mode = "proxy"` with an unusable endpoint silently ran
+ lexical** (issue [#1189](https://github.com/xerj-org/xerj/issues/1189)) —
+ an empty `default_endpoint` or a typo'd URL (`htp://…`) produced a
+ startup WARN and a lexical embedder; indexing and semantic search then
+ "succeeded" while writing lexical feature-hash vectors into the index, and
+ `/v1/embedding/identity` reported `backend: lexical` for a config that
+ said `proxy`. An explicit `mode = "proxy"` now fails closed at startup
+ with an actionable error in both the embedder and the identity path
+ (including the reporter's scheme typo, which carries the underlying
+ `http or https` message); the auto path (endpoint set, mode not pinned)
+ keeps the documented warn-plus-lexical fallback. `xerj autoindex` now
+ names the live embedding backend in its run output so a degraded backend
+ is visible where the work happens.
+
+- **`embedding.batch_size` was never read** (issue
+ [#1190](https://github.com/xerj-org/xerj/issues/1190)) — the setting
+ documented "Maximum documents per embedding API call (default: 64)" but
+ no code read it: one `_bulk` of 20 documents left the proxy as a single
+ request with 20 inputs, and an autoindex run was observed sending a
+ 192-input request. Providers cap the request `input` array (OpenAI 2048,
+ self-hosted gateways often far lower), so a large bulk could fail on
+ request shape, not load. The proxy now splits every batch into
+ concurrent requests of at most `batch_size` inputs (each with its own
+ retry/backoff and concurrency permit; the semaphore now limits in-flight
+ HTTP requests, which was its purpose) and reassembles the vectors in
+ input order. Verified by a mock-endpoint test: 10 texts with
+ `batch_size: 4` leave as three requests of 4/4/2 inputs and the returned
+ vectors keep input order across chunk boundaries.
+
- **Long BM25 queries silently fell back to per-segment IDF/avgdl**
(issue [#1186](https://github.com/xerj-org/xerj/issues/1186)) — the
index-wide statistics pre-pass (#188) budgeted `(fields + terms) ×
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 29cc0d6ac..8a76c39a4 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -7242,6 +7242,7 @@ dependencies = [
"candle-core",
"candle-nn",
"candle-transformers",
+ "futures-util",
"hf-hub",
"ort",
"reqwest",
diff --git a/engine/crates/xerj-ai/Cargo.toml b/engine/crates/xerj-ai/Cargo.toml
index 14cbb9348..60191a0d0 100644
--- a/engine/crates/xerj-ai/Cargo.toml
+++ b/engine/crates/xerj-ai/Cargo.toml
@@ -15,6 +15,9 @@ tokio.workspace = true
tracing.workspace = true
thiserror.workspace = true
anyhow.workspace = true
+# Only for try_join_all in the embedding proxy's batch chunking (#1190) —
+# already in the tree via xerj-engine, no new crate for the workspace.
+futures-util = "0.3"
# Built-in neural embedder (candle + MiniLM), download-on-first-run.
# Feature-gated so the default build stays dependency-light and the
diff --git a/engine/crates/xerj-ai/src/embed.rs b/engine/crates/xerj-ai/src/embed.rs
index 12eaa2bd0..9da5ce92c 100644
--- a/engine/crates/xerj-ai/src/embed.rs
+++ b/engine/crates/xerj-ai/src/embed.rs
@@ -39,6 +39,12 @@ pub struct EmbeddingProxyConfig {
/// Maximum number of retries on transient failures.
#[serde(default = "default_max_retries")]
pub max_retries: u32,
+ /// Maximum input texts per request (#1190). Providers cap
+ /// `input` array length (OpenAI: 2048, many self-hosted ones far lower);
+ /// one `_bulk` of N docs used to leave here as a single N-input request.
+ /// Default 64 mirrors `embedding.batch_size`.
+ #[serde(default = "default_batch_size")]
+ pub batch_size: usize,
}
fn default_timeout_secs() -> u64 {
@@ -50,6 +56,9 @@ fn default_max_concurrent() -> usize {
fn default_max_retries() -> u32 {
3
}
+fn default_batch_size() -> usize {
+ 64
+}
impl EmbeddingProxyConfig {
pub fn new(endpoint: impl Into<String>, model: impl Into<String>) -> Self {
@@ -60,6 +69,7 @@ impl EmbeddingProxyConfig {
timeout_secs: default_timeout_secs(),
max_concurrent: default_max_concurrent(),
max_retries: default_max_retries(),
+ batch_size: default_batch_size(),
}
}
@@ -178,14 +188,29 @@ impl EmbeddingProxy {
return Ok(vec![]);
}
- let _permit = self
- .semaphore
- .acquire()
- .await
- .map_err(|e| XerjError::embedding(format!("semaphore: {e}")))?;
+ // #1190: split into requests of at most `batch_size` inputs. The
+ // semaphore permit moved from here to per-chunk: it limits in-flight
+ // HTTP requests (its actual purpose), not logical batches.
+ let chunk_len = self.config.batch_size.max(1);
+ let chunks: Vec<&[String]> = if texts.len() <= chunk_len {
+ vec![texts.as_slice()]
+ } else {
+ texts.chunks(chunk_len).collect()
+ };
- let result = self.send_with_retry(&texts, model).await?;
- Ok(result)
+ let mut futs = Vec::with_capacity(chunks.len());
+ for chunk in &chunks {
+ futs.push(async {
+ let _permit = self
+ .semaphore
+ .acquire()
+ .await
+ .map_err(|e| XerjError::embedding(format!("semaphore: {e}")))?;
+ self.send_with_retry(chunk, model).await
+ });
+ }
+ let results = futures_util::future::try_join_all(futs).await?;
+ Ok(results.into_iter().flatten().collect())
}
async fn send_with_retry(&self, texts: &[String], model: &str) -> Result<Vec<Vec<f32>>> {
@@ -311,6 +336,7 @@ mod tests {
assert_eq!(cfg.timeout_secs, 30);
assert_eq!(cfg.max_concurrent, 4);
assert_eq!(cfg.max_retries, 3);
+ assert_eq!(cfg.batch_size, 64);
assert!(cfg.api_key.is_none());
}
@@ -398,4 +424,76 @@ mod tests {
);
drop(handle); // listener already closed by loop exit
}
+
+ // #1190: one embed_batch of 10 texts with batch_size 4 must leave as
+ // requests of at most 4 inputs, and the returned vectors stay in input
+ // order across chunk boundaries.
+ #[tokio::test]
+ async fn batch_size_splits_requests_and_preserves_order() {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let addr = listener.local_addr().unwrap();
+ let handle = std::thread::spawn(move || {
+ use std::io::{Read, Write};
+ let mut input_sizes = vec![];
+ for _ in 0..3 {
+ let Ok((mut s, _)) = listener.accept() else { break };
+ // Fixed single read, like permanent_4xx above: the client
+ // holds the connection open awaiting the response, so
+ // read_to_end would deadlock.
+ let mut buf = [0u8; 8192];
+ let n = s.read(&mut buf).unwrap();
+ let body = String::from_utf8_lossy(&buf[..n]);
+ let payload = body
+ .split("\r\n\r\n")
+ .nth(1)
+ .unwrap_or_default()
+ .to_string();
+ let parsed: serde_json::Value = serde_json::from_str(&payload).unwrap();
+ let n = parsed["input"].as_array().unwrap().len();
+ input_sizes.push(n);
+ // Embedding for input i is the vector [i, i, ...]: order is
+ // observable in the output itself.
+ let data: Vec<String> = parsed["input"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .enumerate()
+ .map(|(i, _)| {
+ // The provider sees per-request indices; recover the
+ // global index from the text itself ("t<global>").
+ let text = parsed["input"][i].as_str().unwrap();
+ let g: f32 = text[1..].parse().unwrap();
+ format!(
+ "{{\"object\":\"embedding\",\"index\":{i},\"embedding\":[{g},{g}]}}"
+ )
+ })
+ .collect();
+ let resp_body = format!(
+ "{{\"object\":\"list\",\"data\":[{}]}}",
+ data.join(",")
+ );
+ let resp = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
+ resp_body.len(),
+ resp_body
+ );
+ let _ = s.write_all(resp.as_bytes());
+ }
+ input_sizes
+ });
+
+ let mut cfg = EmbeddingProxyConfig::new(format!("http://{addr}/v1/embeddings"), "m");
+ cfg.batch_size = 4;
+ let proxy = EmbeddingProxy::new(cfg).unwrap();
+
+ let texts: Vec<String> = (0..10).map(|i| format!("t{i}")).collect();
+ let out = proxy.embed_batch(texts).await.unwrap();
+
+ assert_eq!(out.len(), 10);
+ for (i, v) in out.iter().enumerate() {
+ assert_eq!(v, &vec![i as f32, i as f32], "output {i} out of order");
+ }
+ let sizes = handle.join().unwrap();
+ assert_eq!(sizes, vec![4, 4, 2], "requests must respect batch_size");
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 90566d8c1..2ae2879e6 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -6568,6 +6568,14 @@ fn run_index_report_inner(
identity.resumable,
identity.non_resumable_reason.as_deref(),
)?;
+ // #1189: name the live backend in the run output. A server started
+ // with mode=proxy that had silently degraded to lexical was
+ // indistinguishable here until the vectors came back wrong.
+ pr.note(&format!(
+ "semantic autoindex embedding backend: {} ({}resumable)",
+ identity.backend,
+ if identity.resumable { "" } else { "non-" }
+ ));
}
// ── create indices with explicit mappings ────────────────────────────
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index 704a02d7e..0c58f6e9e 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -1354,9 +1354,13 @@ impl Default for LogsConfig {
/// to lexical.
/// * `"proxy"` — call an external OpenAI-compatible `/v1/embeddings`
/// endpoint ([`default_endpoint`]). Lets customers plug in ANY embedding
-/// model / provider they already run.
+/// model / provider they already run. An explicit `mode = "proxy"` with a
+/// missing or unusable endpoint is a startup ERROR (#1189): silently
+/// running lexical instead would write the wrong vectors into the index
+/// while every embedding request appears to succeed.
/// * `"auto"` (default) — use the proxy when [`default_endpoint`] is set,
-/// otherwise lexical. This preserves the historical behavior exactly.
+/// otherwise lexical; if the endpoint fails to initialize, fall back to
+/// lexical with a warning. The fallback exists ONLY on this path (#1189).
///
/// **20 settings.**
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -1369,7 +1373,9 @@ pub struct EmbeddingConfig {
pub default_endpoint: String,
/// Model name to request from the endpoint (default: `""`).
pub default_model: String,
- /// Maximum documents per embedding API call (default: `64`).
+ /// Maximum documents per embedding API call (default: `64`). Providers
+ /// cap the request `input` array (often far below 64); the proxy splits
+ /// larger batches into consecutive requests of at most this size (#1190).
pub batch_size: usize,
/// HTTP timeout for embedding requests in ms (default: `5000`).
pub timeout_ms: u64,
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 698df1066..be6794c7d 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -35532,6 +35532,11 @@ pub(crate) fn embedding_execution_identity(
const CONTRACT: &str = "semantic_text-derived-vector.v1";
let requested = cfg.mode.trim().to_ascii_lowercase();
+ // Explicit mode=proxy with an unusable endpoint is a configuration error,
+ // not a lexical degrade (#1189): the identity must report what the
+ // embedder would actually run, and after the make_embedder fix that
+ // configuration no longer starts. Auto (any non-pinned mode) keeps the
+ // historical fallback to lexical.
let mut backend = match requested.as_str() {
"neural" => {
#[cfg(feature = "neural")]
@@ -35545,13 +35550,25 @@ pub(crate) fn embedding_execution_identity(
}
"onnx-experimental" => "onnx-experimental",
"lexical" => "lexical",
- "proxy" if cfg.default_endpoint.is_empty() => "lexical",
+ "proxy" if cfg.default_endpoint.is_empty() => {
+ return Err(EngineError::Common(xerj_common::XerjError::embedding(
+ "embedding.mode=proxy requires embedding.default_endpoint to be set; \
+ remove mode=proxy (use auto) if a lexical fallback is wanted",
+ )));
+ }
"proxy" => "proxy",
_ if cfg.default_endpoint.is_empty() => "lexical",
_ => "proxy",
};
- if backend == "proxy" && !proxy_initializes(cfg) {
- backend = "lexical";
+ if backend == "proxy" {
+ if let Err(e) = xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)) {
+ if requested == "proxy" {
+ return Err(EngineError::Common(xerj_common::XerjError::embedding(
+ format!("embedding.mode=proxy failed to initialize: {e}"),
+ )));
+ }
+ backend = "lexical";
+ }
}
// Only report a width for the backends whose width this server actually
// pins. `neural` reads it from the resolved model's `config.json`
@@ -35655,7 +35672,10 @@ fn lexical_algorithm_probe_sha256() -> String {
format!("{:x}", digest.finalize())
}
-fn proxy_config(
+/// Build the proxy client config from engine embedding settings. Public so
+/// `xerj-server` runs the same #1189 boot validation the engine runs at
+/// index-create, instead of duplicating the mapping.
+pub fn proxy_config(
cfg: &xerj_common::config::EmbeddingConfig,
) -> xerj_ai::embed::EmbeddingProxyConfig {
xerj_ai::embed::EmbeddingProxyConfig {
@@ -35665,12 +35685,11 @@ fn proxy_config(
timeout_secs: cfg.timeout_ms / 1000,
max_concurrent: 4,
max_retries: 3,
+ batch_size: cfg.batch_size.max(1),
}
}
-fn proxy_initializes(cfg: &xerj_common::config::EmbeddingConfig) -> bool {
- xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)).is_ok()
-}
+
/// Fail closed rather than mixing vectors produced by different model,
/// tokenizer, pooling, or backend configurations after a restart/resume.
@@ -35995,23 +36014,88 @@ mod embedding_identity_tests {
assert!(!encoded.contains("private-alias"));
assert!(!encoded.contains("dimensions"), "{encoded}");
+ // #1189: an EXPLICIT mode=proxy with an unusable endpoint is a
+ // configuration error, not a silent lexical degrade. All three
+ // failures previously reported backend "lexical" while the operator
+ // believed the node was embedding through their provider.
for endpoint in ["", "not a url", "file:///private/model"] {
- let fallback = xerj_common::config::EmbeddingConfig {
+ let broken = xerj_common::config::EmbeddingConfig {
mode: "proxy".into(),
default_endpoint: endpoint.into(),
..Default::default()
};
+ let err = embedding_execution_identity(&broken)
+ .err()
+ .expect("explicit proxy with an unusable endpoint must error");
+ let msg = format!("{err}");
+ assert!(
+ msg.contains("embedding.mode=proxy"),
+ "the error must name the setting: {msg}"
+ );
+ }
+
+ // The AUTO path keeps the documented fallback: an unset or unknown
+ // mode with a broken endpoint resolves to lexical, never errors.
+ for mode in ["auto", "", "somewhere-in-between"] {
+ let fallback = xerj_common::config::EmbeddingConfig {
+ mode: mode.into(),
+ default_endpoint: "not a url".into(),
+ ..Default::default()
+ };
let identity = embedding_execution_identity(&fallback).unwrap();
- assert_eq!(identity.backend, "lexical", "{endpoint}");
- assert!(identity.resumable, "{endpoint}");
+ assert_eq!(identity.backend, "lexical", "{mode}");
+ assert!(identity.resumable, "{mode}");
// The fallback really did become lexical, so the pinned lexical
// width is the truthful answer here.
assert_eq!(
identity.dimensions,
Some(xerj_ai::local::DEFAULT_DIMS),
- "{endpoint}"
+ "{mode}"
+ );
+ }
+ }
+
+ /// #1189: explicit mode=proxy fails closed at the embedder too, with the
+ /// proxy's own error carried through; the auto path keeps the documented
+ /// lexical fallback.
+ #[test]
+ fn explicit_proxy_mode_fails_closed_at_the_embedder() {
+ for endpoint in ["", "htp://typo.example/v1", "file:///etc/hosts"] {
+ let cfg = xerj_common::config::EmbeddingConfig {
+ mode: "proxy".into(),
+ default_endpoint: endpoint.into(),
+ ..Default::default()
+ };
+ let err = make_embedder(&cfg)
+ .err()
+ .expect("explicit proxy with an unusable endpoint must error");
+ let msg = format!("{err}");
+ assert!(
+ msg.contains("embedding.mode=proxy"),
+ "must name the setting: {msg}"
);
+ // The typo case carries the underlying scheme error so the
+ // operator sees what is actually wrong with the URL.
+ if endpoint.starts_with("htp://") {
+ assert!(
+ msg.contains("http or https"),
+ "must carry the proxy's own init error: {msg}"
+ );
+ }
}
+
+ // Auto with the same broken endpoint still boots as lexical — that
+ // fallback is the documented auto behavior and stays.
+ let auto = xerj_common::config::EmbeddingConfig {
+ mode: "auto".into(),
+ default_endpoint: "htp://typo.example/v1".into(),
+ ..Default::default()
+ };
+ make_embedder(&auto).unwrap();
+ assert_eq!(
+ embedding_execution_identity(&auto).unwrap().backend,
+ "lexical"
+ );
}
/// #522: the #434 guard (`validate_embedding_execution`) was unit-tested at
@@ -36048,16 +36132,21 @@ mod embedding_identity_tests {
"the refusal must name both the recorded and the current backend: {msg}"
);
- // Over-refusal guard: a proxy config with NO endpoint falls back to the
- // lexical embedder, so its resolved identity is unchanged and reopening
- // must NOT be refused.
+ // Over-refusal guard (#1189 update): a proxy config with NO endpoint
+ // used to silently resolve to lexical and reopen cleanly. It is now a
+ // configuration error at the identity level, so reopening fails with
+ // the same actionable message instead of a confusing mismatch refusal.
let proxy_without_endpoint = xerj_common::config::EmbeddingConfig {
mode: "proxy".into(),
default_endpoint: String::new(),
..Default::default()
};
- validate_embedding_execution(index_dir, &proxy_without_endpoint, false, true)
- .expect("a proxy that falls back to lexical must not be refused");
+ let err = validate_embedding_execution(index_dir, &proxy_without_endpoint, false, true)
+ .expect_err("proxy with no endpoint is a config error, not a lexical reopen");
+ assert!(
+ format!("{err}").contains("embedding.mode=proxy"),
+ "must name the setting: {err}"
+ );
}
/// #533: the proxy `default_endpoint` is part of the embedder identity. Same
@@ -36742,7 +36831,8 @@ mod embedding_identity_tests {
fn make_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> Result<xerj_ai::Embedder> {
let mode = cfg.mode.trim().to_ascii_lowercase();
- let want_proxy = mode == "proxy"
+ let explicit_proxy = mode == "proxy";
+ let want_proxy = explicit_proxy
|| (mode != "neural" && mode != "lexical" && !cfg.default_endpoint.is_empty());
if mode == "neural" {
@@ -36754,22 +36844,35 @@ fn make_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> Result<xerj_ai::
}
if want_proxy {
+ // Explicit mode=proxy fails closed (#1189): a warn plus lexical
+ // fallback left the node "healthy" while every embedding request
+ // silently wrote lexical vectors into the index. Only the auto path
+ // (endpoint set but mode not pinned) keeps the documented fallback.
if cfg.default_endpoint.is_empty() {
- warn!("embedding.mode=proxy but embedding.default_endpoint is empty — falling back to lexical");
+ if explicit_proxy {
+ return Err(EngineError::Common(xerj_common::XerjError::embedding(
+ "embedding.mode=proxy requires embedding.default_endpoint to be set; \
+ remove mode=proxy (use auto) if a lexical fallback is wanted",
+ )));
+ }
+ warn!("embedding.default_endpoint is empty — falling back to lexical");
return Ok(xerj_ai::Embedder::lexical());
}
- return Ok(
- match xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)) {
- Ok(p) => {
- info!(endpoint = %cfg.default_endpoint, "embedding backend: external proxy");
- xerj_ai::Embedder::proxy(p)
- }
- Err(e) => {
- warn!(error = %e, "embedding proxy init failed — falling back to lexical");
- xerj_ai::Embedder::lexical()
+ return match xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)) {
+ Ok(p) => {
+ info!(endpoint = %cfg.default_endpoint, "embedding backend: external proxy");
+ Ok(xerj_ai::Embedder::proxy(p))
+ }
+ Err(e) => {
+ if explicit_proxy {
+ return Err(EngineError::Common(xerj_common::XerjError::embedding(format!(
+ "embedding.mode=proxy failed to initialize: {e}"
+ ))));
}
- },
- );
+ warn!(error = %e, "embedding proxy init failed — falling back to lexical");
+ Ok(xerj_ai::Embedder::lexical())
+ }
+ };
}
Ok(xerj_ai::Embedder::lexical())
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index ab34be228..976b8c206 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -734,6 +734,24 @@ fn load_config(args: &CliArgs) -> Result<Config> {
}
}
+ // #1189: an explicit mode=proxy is validated at boot, the same
+ // fail-closed contract as onnx-experimental above. The old warn-plus-
+ // lexical fallback started a "healthy" node whose embedding requests
+ // silently wrote lexical vectors. Auto keeps its documented fallback.
+ if cfg.embedding.mode.trim().eq_ignore_ascii_case("proxy") {
+ if cfg.embedding.default_endpoint.trim().is_empty() {
+ anyhow::bail!(
+ "embedding.mode=proxy requires embedding.default_endpoint to be set; \
+ remove mode=proxy (use auto) if a lexical fallback is wanted"
+ );
+ }
+ if let Err(e) = xerj_ai::embed::EmbeddingProxy::new(xerj_engine::index::proxy_config(
+ &cfg.embedding,
+ )) {
+ anyhow::bail!("embedding.mode=proxy failed to initialize: {e}");
+ }
+ }
+
// ES/OpenSearch wire-compatibility override: `--compat-distribution` /
// `XERJ_COMPAT_DISTRIBUTION` (flag wins). Left unset, xerj auto-detects
// per request from the caller's User-Agent instead — this is only for