AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api,engine): size a bare semantic_text at the backend's confirmed width (#2476)
Cristian Grozadf665e9296e6
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 76f57a407..cd9508c89 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -179,6 +179,20 @@ continues in #2427 and #1122, which stay open.### Fixed+- A `semantic_text` field declared without `dimensions` is now sized at the+ width the server's embedding backend actually produces, when that width is+ confirmed (#2407, defect 1). It used to be always 384, so behind a `proxy`+ returning another width (768, 1024, …) every document failed ingest with+ "embedding backend returned width N, mapping requires 384", and the new+ index stayed green with no semantic documents. Index creation,+ `PUT _mapping` and templates (including indices that `_bulk` creates+ automatically) write the width into the mapping, so `GET _mapping` shows it.+ An explicit `dimensions` still wins. Indices created before this change keep+ their stored 384 mapping. Re-declaring a bare `semantic_text` on one of them+ while running a backend of another width is now refused by the immutable+ embedding contract. When the width cannot be confirmed, the field is still+ sized at 384, and the server logs a warning.+- `xerj search` names an unknown flag on one line and exits 2 instead offollowing "unexpected argument" with the whole usage text (#2458). Anagent that guessed `--index` and `--max-hits` read the help dump as thediff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 86acb6958..013d927b6 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -1512,13 +1512,23 @@ pub async fn create_index(// (#2281). A schema-only pass cannot: the stored mapping was the request// body, so a template `semantic_text` was rendered back as `object`, and// the template's settings were never copied.- let (mappings_val, merged_settings) = state.engine.merge_index_template(+ let (mut mappings_val, merged_settings) = state.engine.merge_index_template(&index,body.get("mappings").cloned().unwrap_or(Value::Null),body.get("settings").cloned().unwrap_or(Value::Null),);let settings_ref = (!merged_settings.is_null()).then_some(&merged_settings);+ // #2407: size a bare `semantic_text` at the width the server's embedding+ // backend actually produces, before validation, schema build and the+ // mapping's persistence all read this merged value.+ if let Some(props) = mappings_val.get_mut("properties") {+ if properties_have_bare_semantic_text(props) {+ let dimensions = confirmed_semantic_dimensions(&state);+ materialize_semantic_dimensions(props, dimensions);+ }+ }+// Validate mapping field types before creating — ES returns// mapper_parsing_exception for unsupported types.if let Some(props) = mappings_val.get("properties").and_then(Value::as_object) {@@ -3130,7 +3140,7 @@ pub struct EsPutMappingBody {pub async fn put_mapping(State(state): State<AppState>,Path(index): Path<String>,- Json(body): Json<Value>,+ Json(mut body): Json<Value>,) -> impl IntoResponse {// Wildcard-expand index list.let targets = resolve_index_selector(&state, &index).await;@@ -3184,6 +3194,16 @@ pub async fn put_mapping(}}+ // #2407: size a bare `semantic_text` at the width the server's embedding+ // backend actually produces, before validation, the field parse and the+ // merge into each target's stored mapping all read the request body.+ if let Some(props) = body.get_mut("properties") {+ if properties_have_bare_semantic_text(props) {+ let dimensions = confirmed_semantic_dimensions(&state);+ materialize_semantic_dimensions(props, dimensions);+ }+ }+// Validate any `properties` field types once, up front.if let Some(props) = body.get("properties").and_then(Value::as_object) {if let Some(refusal) = empty_field_name_refusal(props) {@@ -3436,6 +3456,60 @@ fn embedder_warnings(state: &AppState, body: &Value) -> Vec<String> {crate::embed_warning::semantic_embedder_warnings(body, &resolved)}+/// True when `properties` (recursively) declares a `semantic_text` field+/// without an explicit `dimensions`/`dims` — the shape #2407 is about.+fn properties_have_bare_semantic_text(properties: &Value) -> bool {+ let Some(obj) = properties.as_object() else {+ return false;+ };+ obj.values().any(|spec| {+ (spec.get("type").and_then(Value::as_str) == Some("semantic_text")+ && spec.get("dimensions").is_none()+ && spec.get("dims").is_none())+ || spec+ .get("properties")+ .is_some_and(properties_have_bare_semantic_text)+ })+}++/// Write `dimensions` into every bare `semantic_text` property, recursively.+///+/// #2407: a bare `semantic_text` was always sized at 384 (the built-in+/// embedder's width) even when the server's own backend returns another+/// width, so every document failed ingest with "embedding backend returned+/// width N, mapping requires 384" and the just-created index sat green with+/// 0 documents. Materializing the width into the mapping at creation time+/// keeps the embedding contract explicit and immutable as designed, makes+/// `GET _mapping` echo the real width, and leaves an explicit declaration+/// untouched.+fn materialize_semantic_dimensions(properties: &mut Value, dimensions: usize) {+ let Some(obj) = properties.as_object_mut() else {+ return;+ };+ for spec in obj.values_mut() {+ if spec.get("type").and_then(Value::as_str) == Some("semantic_text")+ && spec.get("dimensions").is_none()+ && spec.get("dims").is_none()+ {+ if let Some(map) = spec.as_object_mut() {+ map.insert("dimensions".into(), json!(dimensions));+ }+ }+ if let Some(sub) = spec.get_mut("properties") {+ materialize_semantic_dimensions(sub, dimensions);+ }+ }+}++/// The width this server's embedding backend actually produces: the confirmed+/// identity width (for `proxy` this runs the #2306 probe, cached after the+/// first confirmation), else 384 with a warning — the built-in default a bare+/// mapping was always sized at. Resolved only when a mapping actually carries+/// a bare `semantic_text`, so mappings without one never pay for a probe.+fn confirmed_semantic_dimensions(state: &AppState) -> usize {+ state.engine.confirmed_semantic_dimensions()+}+fn semantic_mapping_contract(field_name: &str,definition: &serde_json::Map<String, Value>,diff --git a/engine/crates/xerj-api/tests/semantic_text_width_follows_backend.rs b/engine/crates/xerj-api/tests/semantic_text_width_follows_backend.rsnew file mode 100644index 000000000..ab1dbd79e--- /dev/null+++ b/engine/crates/xerj-api/tests/semantic_text_width_follows_backend.rs@@ -0,0 +1,241 @@+//! Issue #2407: a `semantic_text` field created without `dimensions` was+//! always sized at 384, the built-in embedder's width, even when the server's+//! own embedding backend is a `proxy` whose confirmed width (#2306, published+//! at `GET /v1/embedding/identity`) is something else. Every document then+//! failed ingest with "embedding backend returned width N, mapping requires+//! 384", and an index created a moment earlier sat green with 0 documents.+//!+//! These tests pin the fix: a bare `semantic_text` is sized at the backend's+//! confirmed width at creation time — explicit create, `PUT _mapping`, and an+//! index template applied on create — the materialized width is echoed by+//! `GET _mapping`, and a document ingests end to end. An explicit+//! `dimensions` in the mapping still wins, unchanged.+//!+//! The mock answers OpenAI `/v1/embeddings` with 768-wide vectors, the shape+//! of the report's `embeddinggemma` gateway; nothing here needs a GPU or a+//! network beyond the loopback mock.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++const DIMS: usize = 768;++async fn mock_embedder() -> String {+ let app = axum::Router::new().route(+ "/v1/embeddings",+ axum::routing::post(|axum::Json(body): axum::Json<Value>| async move {+ let count = match body.get("input") {+ Some(Value::Array(items)) => items.len(),+ Some(Value::String(_)) => 1,+ _ => 0,+ };+ let data: Vec<Value> = (0..count)+ .map(|index| {+ let mut embedding = vec![0.0_f64; DIMS];+ embedding[0] = 1.0;+ json!({"index": index, "embedding": embedding})+ })+ .collect();+ (StatusCode::OK, axum::Json(json!({"data": data})))+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind mock embedder");+ let addr = listener.local_addr().expect("mock embedder addr");+ tokio::spawn(async move {+ let _ = axum::serve(listener, app).await;+ });+ format!("http://{addr}/v1/embeddings")+}++fn proxy_node(endpoint: String) -> axum::Router {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.auth.enabled = false;+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ config.embedding.mode = "proxy".into();+ config.embedding.default_endpoint = endpoint;+ config.embedding.default_model = "mock-embed".into();+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ std::mem::forget(dir);+ xerj_api::router::build_es_compat_router(state)+}++async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+}++/// One document through the semantic path proves the mapping and the backend+/// agree: before the fix the bulk item fails with `mapper_parsing_exception`+/// ("embedding backend returned width 768, mapping requires 384").+async fn ingest_one(app: &axum::Router, index: &str) {+ // `call` sends JSON; `_bulk` is NDJSON, so this request is built by hand.+ let ndjson = format!(+ "{}\n{}\n",+ json!({"index": {"_index": index, "_id": "1"}}),+ json!({"body": "The quarterly revenue report describes how the operating team closed the books."})+ );+ let request = Request::builder()+ .method("POST")+ .uri("/_bulk?refresh=true")+ .header("content-type", "application/x-ndjson")+ .body(Body::from(ndjson))+ .unwrap();+ let response = app.clone().oneshot(request).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ let body: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);+ assert_eq!(status, StatusCode::OK, "bulk failed: {body}");+ assert_eq!(+ body.get("errors"),+ Some(&Value::Bool(false)),+ "bulk item refused: {body}"+ );+ let (status, body) = call(app, "GET", &format!("/{index}/_count"), Value::Null).await;+ assert_eq!(status, StatusCode::OK, "count failed: {body}");+ assert_eq!(body.get("count"), Some(&json!(1)), "count: {body}");+}++fn mapped_dimensions(mapping: &Value, index: &str, field: &str) -> Option<u64> {+ mapping+ .pointer(&format!("/{index}/mappings/properties/{field}/dimensions"))+ .and_then(Value::as_u64)+}++#[tokio::test(flavor = "multi_thread")]+async fn create_index_sizes_bare_semantic_text_at_the_backend_width() {+ let endpoint = mock_embedder().await;+ let app = proxy_node(endpoint);+ let (status, body) = call(+ &app,+ "PUT",+ "/kb",+ json!({"mappings": {"properties": {"body": {"type": "semantic_text"}}}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "create failed: {body}");+ let (status, mapping) = call(&app, "GET", "/kb/_mapping", Value::Null).await;+ assert_eq!(status, StatusCode::OK, "mapping failed: {mapping}");+ assert_eq!(+ mapped_dimensions(&mapping, "kb", "body"),+ Some(DIMS as u64),+ "a bare semantic_text must be sized at the confirmed backend width: {mapping}"+ );+ ingest_one(&app, "kb").await;+}++#[tokio::test(flavor = "multi_thread")]+async fn put_mapping_sizes_bare_semantic_text_at_the_backend_width() {+ let endpoint = mock_embedder().await;+ let app = proxy_node(endpoint);+ let (status, body) = call(&app, "PUT", "/kb2", json!({})).await;+ assert_eq!(status, StatusCode::OK, "create failed: {body}");+ let (status, body) = call(+ &app,+ "PUT",+ "/kb2/_mapping",+ json!({"properties": {"body": {"type": "semantic_text"}}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "put_mapping failed: {body}");+ let (status, mapping) = call(&app, "GET", "/kb2/_mapping", Value::Null).await;+ assert_eq!(status, StatusCode::OK, "mapping failed: {mapping}");+ assert_eq!(+ mapped_dimensions(&mapping, "kb2", "body"),+ Some(DIMS as u64),+ "PUT _mapping must size a bare semantic_text at the backend width: {mapping}"+ );+ ingest_one(&app, "kb2").await;+}++#[tokio::test(flavor = "multi_thread")]+async fn template_applied_on_create_sizes_bare_semantic_text_at_the_backend_width() {+ let endpoint = mock_embedder().await;+ let app = proxy_node(endpoint);+ let (status, body) = call(+ &app,+ "PUT",+ "/_index_template/kb-template",+ json!({+ "index_patterns": ["kb3*"],+ "template": {"mappings": {"properties": {"body": {"type": "semantic_text"}}}}+ }),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "template failed: {body}");+ let (status, body) = call(&app, "PUT", "/kb3", json!({})).await;+ assert_eq!(status, StatusCode::OK, "create failed: {body}");+ let (status, mapping) = call(&app, "GET", "/kb3/_mapping", Value::Null).await;+ assert_eq!(status, StatusCode::OK, "mapping failed: {mapping}");+ assert_eq!(+ mapped_dimensions(&mapping, "kb3", "body"),+ Some(DIMS as u64),+ "a template's bare semantic_text must be sized at the backend width: {mapping}"+ );+ ingest_one(&app, "kb3").await;+}++/// `_bulk` auto-create does not pass through the create-index handler: the+/// engine applies the matching template itself, so the engine-side sizing+/// (`apply_index_template`) is what this test pins.+#[tokio::test(flavor = "multi_thread")]+async fn bulk_auto_create_from_a_template_ingests_at_the_backend_width() {+ let endpoint = mock_embedder().await;+ let app = proxy_node(endpoint);+ let (status, body) = call(+ &app,+ "PUT",+ "/_index_template/kb5-template",+ json!({+ "index_patterns": ["kb5*"],+ "template": {"mappings": {"properties": {"body": {"type": "semantic_text"}}}}+ }),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "template failed: {body}");+ ingest_one(&app, "kb5").await;+}++#[tokio::test(flavor = "multi_thread")]+async fn an_explicit_dimensions_still_wins() {+ let endpoint = mock_embedder().await;+ let app = proxy_node(endpoint);+ let (status, body) = call(+ &app,+ "PUT",+ "/kb4",+ json!({"mappings": {"properties": {"body": {"type": "semantic_text", "dimensions": 32}}}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "create failed: {body}");+ let (status, mapping) = call(&app, "GET", "/kb4/_mapping", Value::Null).await;+ assert_eq!(status, StatusCode::OK, "mapping failed: {mapping}");+ assert_eq!(+ mapped_dimensions(&mapping, "kb4", "body"),+ Some(32),+ "an explicit dimensions declaration is never overridden: {mapping}"+ );+}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 12a7002f9..0bdfa3c6d 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -1495,6 +1495,11 @@ impl Engine {pub(crate) fn apply_index_template(&self, name: &str, schema: Schema) -> Schema {let mut effective_schema = schema;let mut new_semantic_targets = Vec::new();+ // #2407: a template's bare `semantic_text` is sized at the width this+ // server's embedding backend actually produces, not always 384.+ // Resolved once per create and only when a bare field appears, so a+ // template without one never pays for the proxy width probe (#2306).+ let mut confirmed_dimensions: Option<usize> = None;if let Some(tmpl) = self.best_matching_template(name) {if let Some(props) = tmpl.mappings.get("properties").and_then(Value::as_object) {for (field_name, field_def) in props {@@ -1505,7 +1510,18 @@ impl Engine {{continue;}- let fc = field_from_template_property(field_name, field_def);+ let bare_semantic = field_def.get("type").and_then(Value::as_str)+ == Some("semantic_text")+ && field_def.get("dimensions").is_none()+ && field_def.get("dims").is_none();+ let semantic_dimensions = if bare_semantic {+ *confirmed_dimensions+ .get_or_insert_with(|| self.confirmed_semantic_dimensions())+ } else {+ xerj_ai::local::DEFAULT_DIMS+ };+ let fc =+ field_from_template_property(field_name, field_def, semantic_dimensions);if let Some(embedding) = fc.embedding.as_ref() {new_semantic_targets.push(embedding@@ -4370,6 +4386,26 @@ impl Engine {/// Return the effective embedding identity without exposing configuration/// secrets or local asset paths.+ /// The width a bare `semantic_text` is sized at (#2407): the backend's+ /// confirmed width when one exists, else the built-in default, with a+ /// warning, since that fallback only matches a 384-wide backend.+ pub fn confirmed_semantic_dimensions(&self) -> usize {+ let unconfirmed = match self.embedding_execution_identity() {+ Ok(identity) => match identity.dimensions {+ Some(dimensions) => return dimensions,+ None => format!("the {} backend has not confirmed a width", identity.backend),+ },+ Err(error) => error.to_string(),+ };+ warn!(+ reason = %unconfirmed,+ fallback = xerj_ai::local::DEFAULT_DIMS,+ "bare semantic_text sized at the built-in default width; declare `dimensions` \+ if the embedding backend returns another width"+ );+ xerj_ai::local::DEFAULT_DIMS+ }+pub fn embedding_execution_identity(&self) -> Result<EmbeddingExecutionIdentity> {// #2306: a proxy resume needs a confirmed vector width. The probe runs// here, on the identity request autoindex makes before it writes, and@@ -5626,7 +5662,11 @@ fn merge_template_settings(template: &Value, request: Value) -> Value {/// One template property, in the same native shape `es_properties_to_fields`/// builds for an explicit mapping. `semantic_text` is text plus an embedding/// contract — not `object` (#2281).-fn field_from_template_property(name: &str, field_def: &Value) -> xerj_common::types::FieldConfig {+fn field_from_template_property(+ name: &str,+ field_def: &Value,+ semantic_dimensions: usize,+) -> xerj_common::types::FieldConfig {use xerj_common::types::{EmbeddingConfig, FieldConfig, FieldType};let es_type = field_def.get("type")@@ -5651,7 +5691,7 @@ fn field_from_template_property(name: &str, field_def: &Value) -> xerj_common::t.or_else(|| field_def.get("dims")).and_then(Value::as_u64).map(|d| d as usize)- .unwrap_or(xerj_ai::local::DEFAULT_DIMS);+ .unwrap_or(semantic_dimensions);fc.options.dimensions = Some(dims);fc.options.similarity = Some(field_def@@ -6278,6 +6318,7 @@ mod index_template_merge_tests {let semantic = field_from_template_property("diagnostic_semantic",&json!({ "type": "semantic_text" }),+ 768, // #2407: a bare semantic_text takes the caller's backend width);assert_eq!(semantic.field_type, FieldType::Text);assert!(!semantic.options.doc_values);@@ -6286,11 +6327,12 @@ mod index_template_merge_tests {embedding.target_field.as_deref(),Some("diagnostic_semantic_vector"));- assert_eq!(semantic.options.dimensions, Some(384));+ assert_eq!(semantic.options.dimensions, Some(768));let keyword = field_from_template_property("diagnostic_template_marker",&json!({ "type": "keyword" }),+ xerj_ai::local::DEFAULT_DIMS,);assert_eq!(keyword.field_type, FieldType::Keyword);assert!(keyword.embedding.is_none());