Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api,engine): size a bare semantic_text at the backend's confirmed width (#2476)

Cristian Grozadf665e9296e6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 76f57a407..cd9508c89 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -179,6 +179,20 @@ continues in #2427 and #1122, which stay open.
### Fixed
+- A `semantic_text` field declared without `dimensions` is now sized at the
+ width the server's embedding backend actually produces, when that width is
+ confirmed (#2407, defect 1). It used to be always 384, so behind a `proxy`
+ returning another width (768, 1024, …) every document failed ingest with
+ "embedding backend returned width N, mapping requires 384", and the new
+ index stayed green with no semantic documents. Index creation,
+ `PUT _mapping` and templates (including indices that `_bulk` creates
+ automatically) write the width into the mapping, so `GET _mapping` shows it.
+ An explicit `dimensions` still wins. Indices created before this change keep
+ their stored 384 mapping. Re-declaring a bare `semantic_text` on one of them
+ while running a backend of another width is now refused by the immutable
+ embedding contract. When the width cannot be confirmed, the field is still
+ sized at 384, and the server logs a warning.
+
- `xerj search` names an unknown flag on one line and exits 2 instead of
following "unexpected argument" with the whole usage text (#2458). An
agent that guessed `--index` and `--max-hits` read the help dump as the
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 86acb6958..013d927b6 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -1512,13 +1512,23 @@ pub async fn create_index(
// (#2281). A schema-only pass cannot: the stored mapping was the request
// body, so a template `semantic_text` was rendered back as `object`, and
// the template's settings were never copied.
- let (mappings_val, merged_settings) = state.engine.merge_index_template(
+ let (mut mappings_val, merged_settings) = state.engine.merge_index_template(
&index,
body.get("mappings").cloned().unwrap_or(Value::Null),
body.get("settings").cloned().unwrap_or(Value::Null),
);
let settings_ref = (!merged_settings.is_null()).then_some(&merged_settings);
+ // #2407: size a bare `semantic_text` at the width the server's embedding
+ // backend actually produces, before validation, schema build and the
+ // mapping's persistence all read this merged value.
+ if let Some(props) = mappings_val.get_mut("properties") {
+ if properties_have_bare_semantic_text(props) {
+ let dimensions = confirmed_semantic_dimensions(&state);
+ materialize_semantic_dimensions(props, dimensions);
+ }
+ }
+
// Validate mapping field types before creating — ES returns
// mapper_parsing_exception for unsupported types.
if let Some(props) = mappings_val.get("properties").and_then(Value::as_object) {
@@ -3130,7 +3140,7 @@ pub struct EsPutMappingBody {
pub async fn put_mapping(
State(state): State<AppState>,
Path(index): Path<String>,
- Json(body): Json<Value>,
+ Json(mut body): Json<Value>,
) -> impl IntoResponse {
// Wildcard-expand index list.
let targets = resolve_index_selector(&state, &index).await;
@@ -3184,6 +3194,16 @@ pub async fn put_mapping(
}
}
+ // #2407: size a bare `semantic_text` at the width the server's embedding
+ // backend actually produces, before validation, the field parse and the
+ // merge into each target's stored mapping all read the request body.
+ if let Some(props) = body.get_mut("properties") {
+ if properties_have_bare_semantic_text(props) {
+ let dimensions = confirmed_semantic_dimensions(&state);
+ materialize_semantic_dimensions(props, dimensions);
+ }
+ }
+
// Validate any `properties` field types once, up front.
if let Some(props) = body.get("properties").and_then(Value::as_object) {
if let Some(refusal) = empty_field_name_refusal(props) {
@@ -3436,6 +3456,60 @@ fn embedder_warnings(state: &AppState, body: &Value) -> Vec<String> {
crate::embed_warning::semantic_embedder_warnings(body, &resolved)
}
+/// True when `properties` (recursively) declares a `semantic_text` field
+/// without an explicit `dimensions`/`dims` — the shape #2407 is about.
+fn properties_have_bare_semantic_text(properties: &Value) -> bool {
+ let Some(obj) = properties.as_object() else {
+ return false;
+ };
+ obj.values().any(|spec| {
+ (spec.get("type").and_then(Value::as_str) == Some("semantic_text")
+ && spec.get("dimensions").is_none()
+ && spec.get("dims").is_none())
+ || spec
+ .get("properties")
+ .is_some_and(properties_have_bare_semantic_text)
+ })
+}
+
+/// Write `dimensions` into every bare `semantic_text` property, recursively.
+///
+/// #2407: a bare `semantic_text` was always sized at 384 (the built-in
+/// embedder's width) even when the server's own backend returns another
+/// width, so every document failed ingest with "embedding backend returned
+/// width N, mapping requires 384" and the just-created index sat green with
+/// 0 documents. Materializing the width into the mapping at creation time
+/// keeps the embedding contract explicit and immutable as designed, makes
+/// `GET _mapping` echo the real width, and leaves an explicit declaration
+/// untouched.
+fn materialize_semantic_dimensions(properties: &mut Value, dimensions: usize) {
+ let Some(obj) = properties.as_object_mut() else {
+ return;
+ };
+ for spec in obj.values_mut() {
+ if spec.get("type").and_then(Value::as_str) == Some("semantic_text")
+ && spec.get("dimensions").is_none()
+ && spec.get("dims").is_none()
+ {
+ if let Some(map) = spec.as_object_mut() {
+ map.insert("dimensions".into(), json!(dimensions));
+ }
+ }
+ if let Some(sub) = spec.get_mut("properties") {
+ materialize_semantic_dimensions(sub, dimensions);
+ }
+ }
+}
+
+/// The width this server's embedding backend actually produces: the confirmed
+/// identity width (for `proxy` this runs the #2306 probe, cached after the
+/// first confirmation), else 384 with a warning — the built-in default a bare
+/// mapping was always sized at. Resolved only when a mapping actually carries
+/// a bare `semantic_text`, so mappings without one never pay for a probe.
+fn confirmed_semantic_dimensions(state: &AppState) -> usize {
+ state.engine.confirmed_semantic_dimensions()
+}
+
fn semantic_mapping_contract(
field_name: &str,
definition: &serde_json::Map<String, Value>,
diff --git a/engine/crates/xerj-api/tests/semantic_text_width_follows_backend.rs b/engine/crates/xerj-api/tests/semantic_text_width_follows_backend.rs
new file mode 100644
index 000000000..ab1dbd79e
--- /dev/null
+++ b/engine/crates/xerj-api/tests/semantic_text_width_follows_backend.rs
@@ -0,0 +1,241 @@
+//! Issue #2407: a `semantic_text` field created without `dimensions` was
+//! always sized at 384, the built-in embedder's width, even when the server's
+//! own embedding backend is a `proxy` whose confirmed width (#2306, published
+//! at `GET /v1/embedding/identity`) is something else. Every document then
+//! failed ingest with "embedding backend returned width N, mapping requires
+//! 384", and an index created a moment earlier sat green with 0 documents.
+//!
+//! These tests pin the fix: a bare `semantic_text` is sized at the backend's
+//! confirmed width at creation time — explicit create, `PUT _mapping`, and an
+//! index template applied on create — the materialized width is echoed by
+//! `GET _mapping`, and a document ingests end to end. An explicit
+//! `dimensions` in the mapping still wins, unchanged.
+//!
+//! The mock answers OpenAI `/v1/embeddings` with 768-wide vectors, the shape
+//! of the report's `embeddinggemma` gateway; nothing here needs a GPU or a
+//! network beyond the loopback mock.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+const DIMS: usize = 768;
+
+async fn mock_embedder() -> String {
+ let app = axum::Router::new().route(
+ "/v1/embeddings",
+ axum::routing::post(|axum::Json(body): axum::Json<Value>| async move {
+ let count = match body.get("input") {
+ Some(Value::Array(items)) => items.len(),
+ Some(Value::String(_)) => 1,
+ _ => 0,
+ };
+ let data: Vec<Value> = (0..count)
+ .map(|index| {
+ let mut embedding = vec![0.0_f64; DIMS];
+ embedding[0] = 1.0;
+ json!({"index": index, "embedding": embedding})
+ })
+ .collect();
+ (StatusCode::OK, axum::Json(json!({"data": data})))
+ }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("bind mock embedder");
+ let addr = listener.local_addr().expect("mock embedder addr");
+ tokio::spawn(async move {
+ let _ = axum::serve(listener, app).await;
+ });
+ format!("http://{addr}/v1/embeddings")
+}
+
+fn proxy_node(endpoint: String) -> axum::Router {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.auth.enabled = false;
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ config.embedding.mode = "proxy".into();
+ config.embedding.default_endpoint = endpoint;
+ config.embedding.default_model = "mock-embed".into();
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ std::mem::forget(dir);
+ xerj_api::router::build_es_compat_router(state)
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+/// One document through the semantic path proves the mapping and the backend
+/// agree: before the fix the bulk item fails with `mapper_parsing_exception`
+/// ("embedding backend returned width 768, mapping requires 384").
+async fn ingest_one(app: &axum::Router, index: &str) {
+ // `call` sends JSON; `_bulk` is NDJSON, so this request is built by hand.
+ let ndjson = format!(
+ "{}\n{}\n",
+ json!({"index": {"_index": index, "_id": "1"}}),
+ json!({"body": "The quarterly revenue report describes how the operating team closed the books."})
+ );
+ let request = Request::builder()
+ .method("POST")
+ .uri("/_bulk?refresh=true")
+ .header("content-type", "application/x-ndjson")
+ .body(Body::from(ndjson))
+ .unwrap();
+ let response = app.clone().oneshot(request).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let body: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ assert_eq!(status, StatusCode::OK, "bulk failed: {body}");
+ assert_eq!(
+ body.get("errors"),
+ Some(&Value::Bool(false)),
+ "bulk item refused: {body}"
+ );
+ let (status, body) = call(app, "GET", &format!("/{index}/_count"), Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "count failed: {body}");
+ assert_eq!(body.get("count"), Some(&json!(1)), "count: {body}");
+}
+
+fn mapped_dimensions(mapping: &Value, index: &str, field: &str) -> Option<u64> {
+ mapping
+ .pointer(&format!("/{index}/mappings/properties/{field}/dimensions"))
+ .and_then(Value::as_u64)
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn create_index_sizes_bare_semantic_text_at_the_backend_width() {
+ let endpoint = mock_embedder().await;
+ let app = proxy_node(endpoint);
+ let (status, body) = call(
+ &app,
+ "PUT",
+ "/kb",
+ json!({"mappings": {"properties": {"body": {"type": "semantic_text"}}}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "create failed: {body}");
+ let (status, mapping) = call(&app, "GET", "/kb/_mapping", Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "mapping failed: {mapping}");
+ assert_eq!(
+ mapped_dimensions(&mapping, "kb", "body"),
+ Some(DIMS as u64),
+ "a bare semantic_text must be sized at the confirmed backend width: {mapping}"
+ );
+ ingest_one(&app, "kb").await;
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn put_mapping_sizes_bare_semantic_text_at_the_backend_width() {
+ let endpoint = mock_embedder().await;
+ let app = proxy_node(endpoint);
+ let (status, body) = call(&app, "PUT", "/kb2", json!({})).await;
+ assert_eq!(status, StatusCode::OK, "create failed: {body}");
+ let (status, body) = call(
+ &app,
+ "PUT",
+ "/kb2/_mapping",
+ json!({"properties": {"body": {"type": "semantic_text"}}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "put_mapping failed: {body}");
+ let (status, mapping) = call(&app, "GET", "/kb2/_mapping", Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "mapping failed: {mapping}");
+ assert_eq!(
+ mapped_dimensions(&mapping, "kb2", "body"),
+ Some(DIMS as u64),
+ "PUT _mapping must size a bare semantic_text at the backend width: {mapping}"
+ );
+ ingest_one(&app, "kb2").await;
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn template_applied_on_create_sizes_bare_semantic_text_at_the_backend_width() {
+ let endpoint = mock_embedder().await;
+ let app = proxy_node(endpoint);
+ let (status, body) = call(
+ &app,
+ "PUT",
+ "/_index_template/kb-template",
+ json!({
+ "index_patterns": ["kb3*"],
+ "template": {"mappings": {"properties": {"body": {"type": "semantic_text"}}}}
+ }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "template failed: {body}");
+ let (status, body) = call(&app, "PUT", "/kb3", json!({})).await;
+ assert_eq!(status, StatusCode::OK, "create failed: {body}");
+ let (status, mapping) = call(&app, "GET", "/kb3/_mapping", Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "mapping failed: {mapping}");
+ assert_eq!(
+ mapped_dimensions(&mapping, "kb3", "body"),
+ Some(DIMS as u64),
+ "a template's bare semantic_text must be sized at the backend width: {mapping}"
+ );
+ ingest_one(&app, "kb3").await;
+}
+
+/// `_bulk` auto-create does not pass through the create-index handler: the
+/// engine applies the matching template itself, so the engine-side sizing
+/// (`apply_index_template`) is what this test pins.
+#[tokio::test(flavor = "multi_thread")]
+async fn bulk_auto_create_from_a_template_ingests_at_the_backend_width() {
+ let endpoint = mock_embedder().await;
+ let app = proxy_node(endpoint);
+ let (status, body) = call(
+ &app,
+ "PUT",
+ "/_index_template/kb5-template",
+ json!({
+ "index_patterns": ["kb5*"],
+ "template": {"mappings": {"properties": {"body": {"type": "semantic_text"}}}}
+ }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "template failed: {body}");
+ ingest_one(&app, "kb5").await;
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn an_explicit_dimensions_still_wins() {
+ let endpoint = mock_embedder().await;
+ let app = proxy_node(endpoint);
+ let (status, body) = call(
+ &app,
+ "PUT",
+ "/kb4",
+ json!({"mappings": {"properties": {"body": {"type": "semantic_text", "dimensions": 32}}}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "create failed: {body}");
+ let (status, mapping) = call(&app, "GET", "/kb4/_mapping", Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "mapping failed: {mapping}");
+ assert_eq!(
+ mapped_dimensions(&mapping, "kb4", "body"),
+ Some(32),
+ "an explicit dimensions declaration is never overridden: {mapping}"
+ );
+}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 12a7002f9..0bdfa3c6d 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -1495,6 +1495,11 @@ impl Engine {
pub(crate) fn apply_index_template(&self, name: &str, schema: Schema) -> Schema {
let mut effective_schema = schema;
let mut new_semantic_targets = Vec::new();
+ // #2407: a template's bare `semantic_text` is sized at the width this
+ // server's embedding backend actually produces, not always 384.
+ // Resolved once per create and only when a bare field appears, so a
+ // template without one never pays for the proxy width probe (#2306).
+ let mut confirmed_dimensions: Option<usize> = None;
if let Some(tmpl) = self.best_matching_template(name) {
if let Some(props) = tmpl.mappings.get("properties").and_then(Value::as_object) {
for (field_name, field_def) in props {
@@ -1505,7 +1510,18 @@ impl Engine {
{
continue;
}
- let fc = field_from_template_property(field_name, field_def);
+ let bare_semantic = field_def.get("type").and_then(Value::as_str)
+ == Some("semantic_text")
+ && field_def.get("dimensions").is_none()
+ && field_def.get("dims").is_none();
+ let semantic_dimensions = if bare_semantic {
+ *confirmed_dimensions
+ .get_or_insert_with(|| self.confirmed_semantic_dimensions())
+ } else {
+ xerj_ai::local::DEFAULT_DIMS
+ };
+ let fc =
+ field_from_template_property(field_name, field_def, semantic_dimensions);
if let Some(embedding) = fc.embedding.as_ref() {
new_semantic_targets.push(
embedding
@@ -4370,6 +4386,26 @@ impl Engine {
/// Return the effective embedding identity without exposing configuration
/// secrets or local asset paths.
+ /// The width a bare `semantic_text` is sized at (#2407): the backend's
+ /// confirmed width when one exists, else the built-in default, with a
+ /// warning, since that fallback only matches a 384-wide backend.
+ pub fn confirmed_semantic_dimensions(&self) -> usize {
+ let unconfirmed = match self.embedding_execution_identity() {
+ Ok(identity) => match identity.dimensions {
+ Some(dimensions) => return dimensions,
+ None => format!("the {} backend has not confirmed a width", identity.backend),
+ },
+ Err(error) => error.to_string(),
+ };
+ warn!(
+ reason = %unconfirmed,
+ fallback = xerj_ai::local::DEFAULT_DIMS,
+ "bare semantic_text sized at the built-in default width; declare `dimensions` \
+ if the embedding backend returns another width"
+ );
+ xerj_ai::local::DEFAULT_DIMS
+ }
+
pub fn embedding_execution_identity(&self) -> Result<EmbeddingExecutionIdentity> {
// #2306: a proxy resume needs a confirmed vector width. The probe runs
// here, on the identity request autoindex makes before it writes, and
@@ -5626,7 +5662,11 @@ fn merge_template_settings(template: &Value, request: Value) -> Value {
/// One template property, in the same native shape `es_properties_to_fields`
/// builds for an explicit mapping. `semantic_text` is text plus an embedding
/// contract — not `object` (#2281).
-fn field_from_template_property(name: &str, field_def: &Value) -> xerj_common::types::FieldConfig {
+fn field_from_template_property(
+ name: &str,
+ field_def: &Value,
+ semantic_dimensions: usize,
+) -> xerj_common::types::FieldConfig {
use xerj_common::types::{EmbeddingConfig, FieldConfig, FieldType};
let es_type = field_def
.get("type")
@@ -5651,7 +5691,7 @@ fn field_from_template_property(name: &str, field_def: &Value) -> xerj_common::t
.or_else(|| field_def.get("dims"))
.and_then(Value::as_u64)
.map(|d| d as usize)
- .unwrap_or(xerj_ai::local::DEFAULT_DIMS);
+ .unwrap_or(semantic_dimensions);
fc.options.dimensions = Some(dims);
fc.options.similarity = Some(
field_def
@@ -6278,6 +6318,7 @@ mod index_template_merge_tests {
let semantic = field_from_template_property(
"diagnostic_semantic",
&json!({ "type": "semantic_text" }),
+ 768, // #2407: a bare semantic_text takes the caller's backend width
);
assert_eq!(semantic.field_type, FieldType::Text);
assert!(!semantic.options.doc_values);
@@ -6286,11 +6327,12 @@ mod index_template_merge_tests {
embedding.target_field.as_deref(),
Some("diagnostic_semantic_vector")
);
- assert_eq!(semantic.options.dimensions, Some(384));
+ assert_eq!(semantic.options.dimensions, Some(768));
let keyword = field_from_template_property(
"diagnostic_template_marker",
&json!({ "type": "keyword" }),
+ xerj_ai::local::DEFAULT_DIMS,
);
assert_eq!(keyword.field_type, FieldType::Keyword);
assert!(keyword.embedding.is_none());