Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(semantic): publish companion vectors to HNSW

buger983bd4c9d00b
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 335830bcb..2e090d1cc 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -683,6 +683,10 @@ pub async fn create_index(
let e = xerj_common::XerjError::invalid_mapping(reason);
return ApiError::new(e).into_response();
}
+ if let Err(reason) = validate_semantic_companion_targets(props) {
+ let e = xerj_common::XerjError::invalid_mapping(reason);
+ return ApiError::new(e).into_response();
+ }
}
// Runtime mapping type validation too.
if let Some(rt_obj) = mappings_val.get("runtime").and_then(Value::as_object) {
@@ -839,6 +843,135 @@ fn validate_properties(props: &serde_json::Map<String, Value>) -> Result<(), Str
Ok(())
}
+fn validate_semantic_companion_targets(
+ props: &serde_json::Map<String, Value>,
+) -> Result<(), String> {
+ fn walk(props: &serde_json::Map<String, Value>, parent: Option<&str>) -> Result<(), String> {
+ let mut producer_by_target: std::collections::HashMap<String, String> =
+ std::collections::HashMap::new();
+ for (name, spec) in props {
+ let Some(definition) = spec.as_object() else {
+ continue;
+ };
+ if definition.get("type").and_then(Value::as_str) == Some("semantic_text") {
+ if let Some(parent) = parent {
+ return Err(format!(
+ "nested semantic_text field [{parent}.{name}] is not supported; \
+ map semantic_text as a top-level field"
+ ));
+ }
+ let target = definition
+ .get("target_field")
+ .and_then(Value::as_str)
+ .map(str::to_string)
+ .unwrap_or_else(|| format!("{name}_vector"));
+ if let Some(first) = producer_by_target.insert(target.clone(), name.clone()) {
+ return Err(format!(
+ "semantic_text fields [{first}] and [{name}] both target [{target}]; \
+ each semantic field must use a distinct target_field"
+ ));
+ }
+ let explicit_targets = mapping_definitions_for_logical_path(props, &target);
+ if explicit_targets.len() > 1 {
+ return Err(format!(
+ "semantic_text field [{name}] target [{target}] is ambiguous because both \
+ literal and nested mapping paths resolve to that name; keep only one"
+ ));
+ }
+ if props.get(&target).is_none() && !explicit_targets.is_empty() {
+ return Err(format!(
+ "semantic_text field [{name}] target [{target}] resolves to a nested mapping; \
+ custom dotted target_field values must be declared as a literal top-level \
+ property key"
+ ));
+ }
+ if let Some(explicit) = explicit_targets.first() {
+ let explicit_type = explicit
+ .get("type")
+ .and_then(Value::as_str)
+ .unwrap_or("object");
+ if explicit_type != "dense_vector" {
+ return Err(format!(
+ "semantic_text field [{name}] targets [{target}], but that field is \
+ explicitly mapped as [{explicit_type}]; map it as [dense_vector] or \
+ choose another target_field"
+ ));
+ }
+ let semantic_dims = definition
+ .get("dimensions")
+ .or_else(|| definition.get("dims"))
+ .and_then(Value::as_u64)
+ .unwrap_or(384);
+ if let Some(target_dims) = explicit.get("dims").and_then(Value::as_u64) {
+ if target_dims != semantic_dims {
+ return Err(format!(
+ "semantic_text field [{name}] produces {semantic_dims} dimensions, \
+ but target field [{target}] declares {target_dims}"
+ ));
+ }
+ }
+ let semantic_similarity = definition
+ .get("similarity")
+ .and_then(Value::as_str)
+ .unwrap_or("cosine");
+ let target_similarity = explicit
+ .get("similarity")
+ .and_then(Value::as_str)
+ .unwrap_or("cosine");
+ if semantic_similarity != target_similarity {
+ return Err(format!(
+ "semantic_text field [{name}] uses similarity [{semantic_similarity}], \
+ but target field [{target}] declares [{target_similarity}]"
+ ));
+ }
+ }
+ }
+ if let Some(sub) = definition.get("properties").and_then(Value::as_object) {
+ let path = parent
+ .map(|prefix| format!("{prefix}.{name}"))
+ .unwrap_or_else(|| name.clone());
+ walk(sub, Some(&path))?;
+ }
+ }
+ Ok(())
+ }
+ walk(props, None)
+}
+
+fn mapping_definitions_for_logical_path<'a>(
+ props: &'a serde_json::Map<String, Value>,
+ path: &str,
+) -> Vec<&'a serde_json::Map<String, Value>> {
+ let mut matches = Vec::new();
+ if let Some(definition) = props.get(path).and_then(Value::as_object) {
+ matches.push(definition);
+ }
+ if path.contains('.') {
+ let segments: Vec<&str> = path.split('.').collect();
+ let mut current = props;
+ let mut found = None;
+ for (position, segment) in segments.iter().enumerate() {
+ let Some(definition) = current.get(*segment).and_then(Value::as_object) else {
+ found = None;
+ break;
+ };
+ if position + 1 == segments.len() {
+ found = Some(definition);
+ break;
+ }
+ let Some(children) = definition.get("properties").and_then(Value::as_object) else {
+ found = None;
+ break;
+ };
+ current = children;
+ }
+ if let Some(definition) = found {
+ matches.push(definition);
+ }
+ }
+ matches
+}
+
fn validate_runtime_fields(rt: &serde_json::Map<String, Value>) -> Result<(), String> {
for (name, spec) in rt {
let Some(obj) = spec.as_object() else {
@@ -1363,6 +1496,10 @@ pub async fn put_mapping(
let e = xerj_common::XerjError::invalid_mapping(reason);
return ApiError::new(e).into_response();
}
+ if let Err(reason) = validate_semantic_companion_targets(props) {
+ let e = xerj_common::XerjError::invalid_mapping(reason);
+ return ApiError::new(e).into_response();
+ }
}
// Reject type changes on existing fields. ES disallows mutating a
@@ -1405,6 +1542,42 @@ pub async fn put_mapping(
)
.into_response();
}
+ if new_type == "semantic_text" {
+ let old_definition = mapping
+ .get("properties")
+ .and_then(Value::as_object)
+ .and_then(|properties| properties.get(key))
+ .and_then(Value::as_object);
+ let new_definition = val.as_object();
+ if let (Some(old_definition), Some(new_definition)) =
+ (old_definition, new_definition)
+ {
+ let old_contract = semantic_mapping_contract(key, old_definition);
+ let new_contract = semantic_mapping_contract(key, new_definition);
+ if old_contract != new_contract {
+ let reason = format!(
+ "semantic_text field [{key}] embedding contract cannot be \
+ changed in place (target_field, dimensions, and similarity \
+ are immutable); create a new field or reindex"
+ );
+ return (
+ StatusCode::BAD_REQUEST,
+ Json(json!({
+ "error": {
+ "root_cause": [{
+ "type": "illegal_argument_exception",
+ "reason": reason,
+ }],
+ "type": "illegal_argument_exception",
+ "reason": reason,
+ },
+ "status": 400,
+ })),
+ )
+ .into_response();
+ }
+ }
+ }
}
}
}
@@ -1415,12 +1588,6 @@ pub async fn put_mapping(
Ok(i) => i,
Err(_) => continue,
};
- if let Some(properties) = body.get("properties") {
- let fields = es_properties_to_fields(properties);
- for field in fields {
- let _ = idx.add_field(field).await;
- }
- }
// Merge the new properties into the stored mappings blob so
// round-trip GET /_mapping reflects both create-time and
// put_mapping-time field definitions. Dotted keys in the request
@@ -1457,6 +1624,37 @@ pub async fn put_mapping(
} else {
existing = body.clone();
}
+
+ // Semantic companion validation must see the complete post-update
+ // mapping, not only this request. Otherwise a target declared by an
+ // earlier request can collide with a newly added semantic source.
+ if let Some(merged_props) = existing.get("properties").and_then(Value::as_object) {
+ if let Err(reason) = validate_semantic_companion_targets(merged_props) {
+ let e = xerj_common::XerjError::invalid_mapping(reason);
+ return ApiError::new(e).into_response();
+ }
+ }
+
+ if let Some(properties) = body.get("properties") {
+ let current = idx.schema().await;
+ let mut additions = Vec::new();
+ for field in es_properties_to_fields(properties) {
+ if let Some(existing_field) = current.field(&field.name) {
+ if existing_field.field_type != field.field_type {
+ let e = xerj_common::XerjError::invalid_mapping(format!(
+ "field [{}] already exists as [{}], cannot add [{}]",
+ field.name, existing_field.field_type, field.field_type
+ ));
+ return ApiError::new(e).into_response();
+ }
+ continue;
+ }
+ additions.push(field);
+ }
+ if let Err(error) = idx.add_fields(additions).await {
+ return ApiError::new(xerj_common::XerjError::from(error)).into_response();
+ }
+ }
// Persists es_mapping.json into the index dir (atomic) so the
// merged mapping survives a restart.
state.engine.put_index_mapping(idx_name, existing);
@@ -1465,6 +1663,28 @@ pub async fn put_mapping(
Json(json!({ "acknowledged": true })).into_response()
}
+fn semantic_mapping_contract(
+ field_name: &str,
+ definition: &serde_json::Map<String, Value>,
+) -> (String, u64, String) {
+ let target = definition
+ .get("target_field")
+ .and_then(Value::as_str)
+ .map(str::to_string)
+ .unwrap_or_else(|| format!("{field_name}_vector"));
+ let dimensions = definition
+ .get("dimensions")
+ .or_else(|| definition.get("dims"))
+ .and_then(Value::as_u64)
+ .unwrap_or(384);
+ let similarity = definition
+ .get("similarity")
+ .and_then(Value::as_str)
+ .unwrap_or("cosine")
+ .to_string();
+ (target, dimensions, similarity)
+}
+
/// Walk a mapping blob's `properties` tree and build a flat map of
/// dotted-path → leaf type. Both literal-dotted keys (subobjects:false
/// leaves) and nested `{properties:{...}}` children are accommodated,
@@ -12572,9 +12792,20 @@ fn es_properties_to_fields(properties: &Value) -> Vec<FieldConfig> {
.map(|d| d as usize)
.unwrap_or(384);
fc.options.dimensions = Some(dims);
- // Built-in embeddings are L2-normalised → cosine similarity.
- fc.options.similarity = Some("cosine".to_string());
- let target_field = format!("{field_name}_vector");
+ // Built-in embeddings are L2-normalised, so cosine is the
+ // default. Honour an explicit similarity for proxy/custom
+ // embedders and carry it onto the internal vector companion.
+ let similarity = field_def
+ .get("similarity")
+ .and_then(Value::as_str)
+ .unwrap_or("cosine")
+ .to_string();
+ fc.options.similarity = Some(similarity);
+ let target_field = field_def
+ .get("target_field")
+ .and_then(Value::as_str)
+ .map(str::to_string)
+ .unwrap_or_else(|| format!("{field_name}_vector"));
// Allow a mapping to point at an external inference endpoint/model
// (`inference_id` mirrors ES 8.x semantic_text wiring); when absent
// the engine falls back to the zero-config built-in embedder.
@@ -12638,9 +12869,48 @@ fn es_properties_to_fields(properties: &Value) -> Vec<FieldConfig> {
fields.push(fc);
}
+
+ // A semantic_text field writes its embedding to a companion field. That
+ // companion is internal when the caller did not declare it explicitly,
+ // but it still has to be a native Vector in the engine schema: HNSW pins
+ // only mapped vector fields and deliberately ignores arbitrary numeric
+ // arrays. Add companions after parsing all public fields so declaration
+ // order cannot overwrite an explicit target.
+ let semantic_companions: Vec<FieldConfig> = fields
+ .iter()
+ .filter_map(|field| {
+ let embedding = field.embedding.as_ref()?;
+ let target = embedding
+ .target_field
+ .clone()
+ .unwrap_or_else(|| format!("{}_vector", field.name));
+ if field_configs_contain_logical_path(&fields, &target) {
+ return None;
+ }
+ let mut companion = FieldConfig::new(target, FieldType::Vector);
+ companion.options.dimensions = field.options.dimensions;
+ companion.options.similarity = field.options.similarity.clone();
+ Some(companion)
+ })
+ .collect();
+ fields.extend(semantic_companions);
fields
}
+fn field_configs_contain_logical_path(fields: &[FieldConfig], target: &str) -> bool {
+ fn walk(fields: &[FieldConfig], prefix: &str, target: &str) -> bool {
+ fields.iter().any(|field| {
+ let path = if prefix.is_empty() {
+ field.name.clone()
+ } else {
+ format!("{prefix}.{}", field.name)
+ };
+ path == target || (!field.fields.is_empty() && walk(&field.fields, &path, target))
+ })
+ }
+ walk(fields, "", target)
+}
+
fn es_type_to_native(es_type: &str) -> FieldType {
match es_type {
// `semantic_text` carries plain text in `_source` (the derived
@@ -12667,6 +12937,713 @@ fn es_type_to_native(es_type: &str) -> FieldType {
}
}
+#[cfg(test)]
+mod semantic_companion_schema_tests {
+ use super::*;
+
+ #[test]
+ fn implicit_companion_is_a_native_vector_with_semantic_options() {
+ let fields = es_properties_to_fields(&json!({
+ "body": {
+ "type": "semantic_text",
+ "dimensions": 96,
+ "similarity": "dot_product"
+ },
+ "ports": { "type": "object" }
+ }));
+
+ let body = fields.iter().find(|field| field.name == "body").unwrap();
+ assert_eq!(
+ body.embedding.as_ref().unwrap().target_field.as_deref(),
+ Some("body_vector")
+ );
+ let companion = fields
+ .iter()
+ .find(|field| field.name == "body_vector")
+ .unwrap();
+ assert_eq!(companion.field_type, FieldType::Vector);
+ assert_eq!(companion.options.dimensions, Some(96));
+ assert_eq!(companion.options.similarity.as_deref(), Some("dot_product"));
+ assert!(
+ fields.iter().all(|field| field.name != "ports_vector"),
+ "ordinary fields and numeric arrays must not acquire vector mappings"
+ );
+ }
+
+ #[test]
+ fn custom_target_is_registered_without_overwriting_an_explicit_companion() {
+ let fields = es_properties_to_fields(&json!({
+ "body": {
+ "type": "semantic_text",
+ "target_field": "model_output",
+ "dimensions": 7,
+ "similarity": "l2_norm"
+ },
+ "model_output": {
+ "type": "dense_vector",
+ "dims": 7,
+ "similarity": "l2_norm"
+ }
+ }));
+
+ assert_eq!(
+ fields
+ .iter()
+ .filter(|field| field.name == "model_output")
+ .count(),
+ 1,
+ "the explicit target must not be replaced or duplicated"
+ );
+ let companion = fields
+ .iter()
+ .find(|field| field.name == "model_output")
+ .unwrap();
+ assert_eq!(companion.field_type, FieldType::Vector);
+ assert_eq!(companion.options.dimensions, Some(7));
+ assert_eq!(companion.options.similarity.as_deref(), Some("l2_norm"));
+ }
+
+ #[test]
+ fn explicit_companion_collisions_report_actionable_mismatches() {
+ let wrong_type = json!({
+ "body": { "type": "semantic_text", "target_field": "model_output" },
+ "model_output": { "type": "double" }
+ });
+ let error =
+ validate_semantic_companion_targets(wrong_type.as_object().unwrap()).unwrap_err();
+ assert!(error.contains("explicitly mapped as [double]"), "{error}");
+ assert!(error.contains("map it as [dense_vector]"), "{error}");
+
+ let wrong_dims = json!({
+ "body": {
+ "type": "semantic_text",
+ "target_field": "model_output",
+ "dimensions": 384
+ },
+ "model_output": { "type": "dense_vector", "dims": 768 }
+ });
+ let error =
+ validate_semantic_companion_targets(wrong_dims.as_object().unwrap()).unwrap_err();
+ assert!(error.contains("produces 384 dimensions"), "{error}");
+ assert!(error.contains("declares 768"), "{error}");
+
+ let wrong_similarity = json!({
+ "body": {
+ "type": "semantic_text",
+ "target_field": "model_output",
+ "similarity": "dot_product"
+ },
+ "model_output": {
+ "type": "dense_vector",
+ "dims": 384,
+ "similarity": "cosine"
+ }
+ });
+ let error =
+ validate_semantic_companion_targets(wrong_similarity.as_object().unwrap()).unwrap_err();
+ assert!(error.contains("uses similarity [dot_product]"), "{error}");
+ assert!(error.contains("declares [cosine]"), "{error}");
+
+ let nested = json!({
+ "document": {
+ "type": "object",
+ "properties": {
+ "body": { "type": "semantic_text" }
+ }
+ }
+ });
+ let error = validate_semantic_companion_targets(nested.as_object().unwrap()).unwrap_err();
+ assert!(
+ error.contains("nested semantic_text field [document.body] is not supported"),
+ "{error}"
+ );
+
+ let nested_target = json!({
+ "body": {
+ "type": "semantic_text",
+ "target_field": "semantic.embedding",
+ "dimensions": 32
+ },
+ "semantic": {
+ "properties": {
+ "embedding": { "type": "dense_vector", "dims": 32 }
+ }
+ }
+ });
+ let error =
+ validate_semantic_companion_targets(nested_target.as_object().unwrap()).unwrap_err();
+ assert!(
+ error.contains("must be declared as a literal top-level property key"),
+ "{error}"
+ );
+ }
+
+ #[test]
+ fn multiple_semantic_fields_pin_only_declared_companions() {
+ let fields = es_properties_to_fields(&json!({
+ "body": { "type": "semantic_text", "dimensions": 8 },
+ "summary": {
+ "type": "semantic_text",
+ "target_field": "summary_embedding",
+ "dimensions": 16
+ },
+ "measurements": { "type": "object" }
+ }));
+
+ let vectors: Vec<_> = fields
+ .iter()
+ .filter(|field| field.field_type == FieldType::Vector)
+ .map(|field| (field.name.as_str(), field.options.dimensions))
+ .collect();
+ assert_eq!(
+ vectors,
+ vec![("body_vector", Some(8)), ("summary_embedding", Some(16))]
+ );
+ }
+}
+
+#[cfg(test)]
+mod semantic_companion_hnsw_api_tests {
+ use super::*;
+ use axum::{
+ body::{to_bytes, Body},
+ http::{Request, StatusCode},
+ };
+ use tower::ServiceExt;
+ use xerj_common::{
+ config::{Config, WalSync},
+ metrics::Metrics,
+ };
+ use xerj_engine::Engine;
+
+ async fn request_json(
+ state: AppState,
+ method: &str,
+ path: &str,
+ content_type: Option<&str>,
+ body: impl Into<Body>,
+ ) -> (StatusCode, Value) {
+ let mut builder = Request::builder().method(method).uri(path);
+ if let Some(content_type) = content_type {
+ builder = builder.header("content-type", content_type);
+ }
+ let response = crate::router::build_es_compat_router(state)
+ .oneshot(builder.body(body.into()).unwrap())
+ .await
+ .unwrap();
+ let status = response.status();
+ let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
+ let json = serde_json::from_slice(&bytes).unwrap_or_else(|error| {
+ panic!(
+ "{method} {path} returned non-JSON ({error}): {}",
+ String::from_utf8_lossy(&bytes)
+ )
+ });
+ (status, json)
+ }
+
+ fn state(config: &Config) -> AppState {
+ let engine = Engine::new(config.clone()).unwrap();
+ let metrics = Metrics::new().unwrap();
+ AppState::new(config.clone(), engine, metrics)
+ }
+
+ async fn assert_complete_semantic_graph(state: &AppState) {
+ let index = state.engine.get_index("semantic-api").unwrap();
+ let stats = index.hnsw_stats().await;
+ assert_eq!(stats["present"], true, "{stats}");
+ assert_eq!(stats["field"], "body_vector", "{stats}");
+ assert_eq!(stats["nodes"], 1024, "{stats}");
+ assert_eq!(stats["doc_coverage"], 1024, "{stats}");
+ assert_eq!(stats["vector_doc_count"], 1024, "{stats}");
+ assert_eq!(stats["covered"], true, "{stats}");
+ }
+
+ #[tokio::test]
+ async fn api_semantic_bulk_builds_complete_hnsw_and_survives_restart() {
+ let data_dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = data_dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = WalSync::Async;
+
+ {
+ let state = state(&config);
+ let (status, _) = request_json(
+ state.clone(),
+ "PUT",
+ "/semantic-api",
+ Some("application/json"),
+ json!({
+ "mappings": {
+ "properties": {
+ "body": { "type": "semantic_text", "dimensions": 32 },
+ "measurements": { "type": "object" }
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+
+ // Public mapping round-trips exactly what the caller declared:
+ // the engine-only companion is not exposed as another field.
+ let (status, mapping) =
+ request_json(state.clone(), "GET", "/semantic-api/_mapping", None, "").await;
+ assert_eq!(status, StatusCode::OK);
+ let properties = mapping["semantic-api"]["mappings"]["properties"]
+ .as_object()
+ .unwrap();
+ assert_eq!(properties.len(), 2, "{mapping}");
+ assert!(!properties.contains_key("body_vector"), "{mapping}");
+
+ let mut bulk = String::new();
+ for id in 0..1024 {
+ bulk.push_str(&format!(
+ "{{\"index\":{{\"_index\":\"semantic-api\",\"_id\":\"{id}\"}}}}\n"
+ ));
+ bulk.push_str(&format!(
+ "{{\"body\":\"quarterly finance report revenue item {id}\",\
+ \"measurements\":[{id},{}]}}\n",
+ id + 1
+ ));
+ }
+ let (status, response) = request_json(
+ state.clone(),
+ "POST",
+ "/_bulk",
+ Some("application/x-ndjson"),
+ bulk,
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(response["errors"], false, "{response}");
+ assert_eq!(response["items"].as_array().unwrap().len(), 1024);
+ assert_complete_semantic_graph(&state).await;
+
+ let (status, response) = request_json(
+ state.clone(),
+ "POST",
+ "/semantic-api/_search",
+ Some("application/json"),
+ json!({
+ "size": 5,
+ "query": {
+ "semantic": {
+ "field": "body",
+ "query": "quarterly revenue",
+ "k": 5
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{response}");
+ assert_eq!(response["hits"]["hits"].as_array().unwrap().len(), 5);
+
+ state.engine.flush_all_force().await;
+ }
+
+ let restarted = state(&config);
+ assert_complete_semantic_graph(&restarted).await;
+ let (status, response) = request_json(
+ restarted.clone(),
+ "POST",
+ "/semantic-api/_search",
+ Some("application/json"),
+ json!({
+ "size": 5,
+ "query": {
+ "semantic": {
+ "field": "body",
+ "query": "quarterly revenue",
+ "k": 5
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{response}");
+ assert_eq!(response["hits"]["hits"].as_array().unwrap().len(), 5);
+ restarted.engine.flush_all_force().await;
+ }
+
+ #[tokio::test]
+ async fn put_mapping_rejects_existing_companion_collision_atomically() {
+ let data_dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = data_dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = WalSync::Async;
+ let state = state(&config);
+
+ let (status, _) = request_json(
+ state.clone(),
+ "PUT",
+ "/semantic-collision",
+ Some("application/json"),
+ json!({
+ "mappings": {
+ "properties": {
+ "body_vector": { "type": "double" }
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ let schema_before = state
+ .engine
+ .get_index("semantic-collision")
+ .unwrap()
+ .schema()
+ .await;
+
+ let (status, error) = request_json(
+ state.clone(),
+ "PUT",
+ "/semantic-collision/_mapping",
+ Some("application/json"),
+ json!({
+ "properties": {
+ "body": { "type": "semantic_text" }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");
+ assert!(
+ error["error"]["reason"]
+ .as_str()
+ .unwrap()
+ .contains("explicitly mapped as [double]"),
+ "{error}"
+ );
+ let schema_after = state
+ .engine
+ .get_index("semantic-collision")
+ .unwrap()
+ .schema()
+ .await;
+ assert_eq!(schema_after, schema_before, "schema update must be atomic");
+
+ let (_, mapping) = request_json(
+ state.clone(),
+ "GET",
+ "/semantic-collision/_mapping",
+ None,
+ "",
+ )
+ .await;
+ assert!(
+ mapping["semantic-collision"]["mappings"]["properties"]
+ .get("body")
+ .is_none(),
+ "{mapping}"
+ );
+ state.engine.flush_all_force().await;
+ }
+
+ #[tokio::test]
+ async fn put_mapping_rejects_semantic_contract_mutation_atomically() {
+ let data_dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = data_dir.path().to_string_lossy().into_owned();
+ let state = state(&config);
+ let (status, _) = request_json(
+ state.clone(),
+ "PUT",
+ "/semantic-contract",
+ Some("application/json"),
+ json!({
+ "mappings": {
+ "properties": {
+ "body": { "type": "semantic_text", "dimensions": 32 }
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ let schema_before = state
+ .engine
+ .get_index("semantic-contract")
+ .unwrap()
+ .schema()
+ .await;
+ let (_, mapping_before) = request_json(
+ state.clone(),
+ "GET",
+ "/semantic-contract/_mapping",
+ None,
+ "",
+ )
+ .await;
+
+ for mutation in [
+ json!({"type": "semantic_text", "dimensions": 64}),
+ json!({
+ "type": "semantic_text",
+ "dimensions": 32,
+ "target_field": "replacement_vector"
+ }),
+ json!({
+ "type": "semantic_text",
+ "dimensions": 32,
+ "similarity": "dot_product"
+ }),
+ ] {
+ let (status, error) = request_json(
+ state.clone(),
+ "PUT",
+ "/semantic-contract/_mapping",
+ Some("application/json"),
+ json!({"properties": {"body": mutation}}).to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");
+ assert!(
+ error["error"]["reason"]
+ .as_str()
+ .unwrap()
+ .contains("embedding contract cannot be changed in place"),
+ "{error}"
+ );
+ assert_eq!(
+ state
+ .engine
+ .get_index("semantic-contract")
+ .unwrap()
+ .schema()
+ .await,
+ schema_before
+ );
+ let (_, mapping_after) = request_json(
+ state.clone(),
+ "GET",
+ "/semantic-contract/_mapping",
+ None,
+ "",
+ )
+ .await;
+ assert_eq!(mapping_after, mapping_before);
+ }
+ state.engine.flush_all_force().await;
+ }
+
+ #[tokio::test]
+ async fn shared_semantic_target_is_rejected_before_index_creation() {
+ let data_dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = data_dir.path().to_string_lossy().into_owned();
+ let state = state(&config);
+ let (status, error) = request_json(
+ state.clone(),
+ "PUT",
+ "/shared-target",
+ Some("application/json"),
+ json!({
+ "mappings": {
+ "properties": {
+ "body": {
+ "type": "semantic_text",
+ "target_field": "shared_vector"
+ },
+ "summary": {
+ "type": "semantic_text",
+ "target_field": "shared_vector"
+ }
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");
+ assert!(
+ error["error"]["reason"]
+ .as_str()
+ .unwrap()
+ .contains("each semantic field must use a distinct target_field"),
+ "{error}"
+ );
+ assert!(state.engine.get_index("shared-target").is_err());
+
+ let (status, error) = request_json(
+ state.clone(),
+ "PUT",
+ "/nested-custom-target",
+ Some("application/json"),
+ json!({
+ "mappings": {
+ "properties": {
+ "body": {
+ "type": "semantic_text",
+ "target_field": "semantic.embedding",
+ "dimensions": 32
+ },
+ "semantic": {
+ "properties": {
+ "embedding": { "type": "dense_vector", "dims": 32 }
+ }
+ }
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");
+ assert!(
+ error["error"]["reason"]
+ .as_str()
+ .unwrap()
+ .contains("literal top-level property key"),
+ "{error}"
+ );
+ assert!(state.engine.get_index("nested-custom-target").is_err());
+
+ let (status, _) = request_json(
+ state.clone(),
+ "PUT",
+ "/nested-put-target",
+ Some("application/json"),
+ json!({
+ "mappings": {
+ "properties": {
+ "semantic": {
+ "properties": {
+ "embedding": { "type": "dense_vector", "dims": 32 }
+ }
+ }
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ let before = state
+ .engine
+ .get_index("nested-put-target")
+ .unwrap()
+ .schema()
+ .await;
+ let (status, error) = request_json(
+ state.clone(),
+ "PUT",
+ "/nested-put-target/_mapping",
+ Some("application/json"),
+ json!({
+ "properties": {
+ "body": {
+ "type": "semantic_text",
+ "target_field": "semantic.embedding",
+ "dimensions": 32
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");
+ assert_eq!(
+ state
+ .engine
+ .get_index("nested-put-target")
+ .unwrap()
+ .schema()
+ .await,
+ before
+ );
+ }
+
+ #[tokio::test]
+ async fn literal_dotted_custom_target_builds_and_queries_hnsw() {
+ let data_dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = data_dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = WalSync::Async;
+ let state = state(&config);
+ let (status, _) = request_json(
+ state.clone(),
+ "PUT",
+ "/dotted-target",
+ Some("application/json"),
+ json!({
+ "mappings": {
+ "properties": {
+ "body": {
+ "type": "semantic_text",
+ "target_field": "semantic.embedding",
+ "dimensions": 32
+ }
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+
+ let mut bulk = String::new();
+ for id in 0..1024 {
+ bulk.push_str(&format!(
+ "{{\"index\":{{\"_index\":\"dotted-target\",\"_id\":\"{id}\"}}}}\n"
+ ));
+ bulk.push_str(&format!(
+ "{{\"body\":\"dotted semantic revenue item {id}\"}}\n"
+ ));
+ }
+ let (status, response) = request_json(
+ state.clone(),
+ "POST",
+ "/_bulk",
+ Some("application/x-ndjson"),
+ bulk,
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(response["errors"], false, "{response}");
+ let stats = state
+ .engine
+ .get_index("dotted-target")
+ .unwrap()
+ .hnsw_stats()
+ .await;
+ assert_eq!(stats["field"], "semantic.embedding", "{stats}");
+ assert_eq!(stats["doc_coverage"], 1024, "{stats}");
+ assert_eq!(stats["covered"], true, "{stats}");
+
+ let (status, response) = request_json(
+ state.clone(),
+ "POST",
+ "/dotted-target/_search",
+ Some("application/json"),
+ json!({
+ "size": 5,
+ "query": {
+ "semantic": {
+ "field": "body",
+ "query": "semantic revenue",
+ "k": 5
+ }
+ }
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{response}");
+ assert_eq!(response["hits"]["total"]["value"], 5);
+ assert_eq!(response["hits"]["hits"].as_array().unwrap().len(), 5);
+ state.engine.flush_all_force().await;
+ }
+}
+
pub fn schema_to_es_properties(schema: &Schema) -> serde_json::Map<String, Value> {
let mut props = serde_json::Map::new();
for field in &schema.fields {
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 620ca6275..25801b650 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -546,6 +546,255 @@ mod semantic_deadline_regression_tests {
idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
}
+ #[tokio::test]
+ async fn fully_covered_semantic_companion_uses_ann_with_exact_rescore() {
+ let dir = TempDir::new().unwrap();
+ let engine_instance = engine(&dir);
+ let mut schema = Schema::empty();
+ let mut body = FieldConfig::new("body", FieldType::Text);
+ body.options.dimensions = Some(32);
+ body.options.similarity = Some("cosine".into());
+ body.embedding = Some(xerj_common::types::EmbeddingConfig {
+ endpoint: None,
+ model: None,
+ target_field: Some("body_vector".into()),
+ });
+ schema.add_field(body).unwrap();
+ let mut companion = FieldConfig::new("body_vector", FieldType::Vector);
+ companion.options.dimensions = Some(32);
+ companion.options.similarity = Some("cosine".into());
+ schema.add_field(companion).unwrap();
+ let mut summary = FieldConfig::new("summary", FieldType::Text);
+ summary.options.dimensions = Some(32);
+ summary.options.similarity = Some("cosine".into());
+ summary.embedding = Some(xerj_common::types::EmbeddingConfig {
+ endpoint: None,
+ model: None,
+ target_field: Some("summary_vector".into()),
+ });
+ schema.add_field(summary).unwrap();
+ let mut summary_companion = FieldConfig::new("summary_vector", FieldType::Vector);
+ summary_companion.options.dimensions = Some(32);
+ summary_companion.options.similarity = Some("cosine".into());
+ schema.add_field(summary_companion).unwrap();
+ engine_instance
+ .create_index("semantic-ann", schema)
+ .unwrap();
+ let idx = engine_instance.get_index("semantic-ann").unwrap();
+
+ for n in 0..1024 {
+ idx.index_document(
+ Some(format!("s{n}")),
+ serde_json::json!({
+ "body": format!("quarterly revenue item {n}"),
+ "summary": format!("cash flow note {n}")
+ }),
+ )
+ .await
+ .unwrap();
+ }
+ let stats = idx.hnsw_stats().await;
+ assert_eq!(stats["field"], "body_vector", "{stats}");
+ assert_eq!(stats["doc_coverage"], 1024, "{stats}");
+ assert_eq!(stats["vector_doc_count"], 1024, "{stats}");
+ assert_eq!(stats["covered"], true, "{stats}");
+
+ let semantic_request = SearchRequest {
+ query: QueryNode::SemanticSearch {
+ field: "body".into(),
+ text: "quarterly revenue item 777".into(),
+ k: 5,
+ filter: None,
+ boost: None,
+ },
+ size: 5,
+ ..SearchRequest::default()
+ };
+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
+ let body_ann = idx.search(&semantic_request).await.unwrap();
+ assert_eq!(body_ann.hits.len(), 5);
+ assert_eq!(body_ann.total.value, 5);
+ assert_eq!(body_ann.hits[0].id, "s777");
+ assert_eq!(
+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed),
+ 0,
+ "an eligible semantic query must use HNSW candidates plus exact rescore, \
+ not the brute-force document scan"
+ );
+
+ let exact = idx
+ .run_knn_brute_force(
+ &semantic_request,
+ "body_vector",
+ &xerj_ai::local::local_embed("quarterly revenue item 777", 32),
+ 5,
+ None,
+ "cosine",
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ assert_eq!(exact.hits[0].id, "s777");
+ assert_eq!(exact.hits[0].score, body_ann.hits[0].score);
+ let exact_ids: HashSet<_> = exact.hits.iter().map(|hit| hit.id.as_str()).collect();
+ let recalled = body_ann
+ .hits
+ .iter()
+ .filter(|hit| exact_ids.contains(hit.id.as_str()))
+ .count();
+ assert!(
+ recalled >= 4,
+ "fixed-oracle recall@5 fell below 0.8: ANN={:?}, exact={:?}",
+ body_ann
+ .hits
+ .iter()
+ .map(|hit| (&hit.id, hit.score))
+ .collect::<Vec<_>>(),
+ exact
+ .hits
+ .iter()
+ .map(|hit| (&hit.id, hit.score))
+ .collect::<Vec<_>>()
+ );
+ // Only the unique exact match has a stable order. Remaining documents
+ // tie under this deliberately repetitive corpus; graph-return order is
+ // the documented ANN tie contract, while total remains exactly k.
+ let cached = idx.search(&semantic_request).await.unwrap();
+ assert_eq!(
+ cached
+ .hits
+ .iter()
+ .map(|hit| (&hit.id, hit.score))
+ .collect::<Vec<_>>(),
+ body_ann
+ .hits
+ .iter()
+ .map(|hit| (&hit.id, hit.score))
+ .collect::<Vec<_>>(),
+ "cached semantic ANN must preserve IDs and exact-rescored scores"
+ );
+
+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
+ let expired = idx
+ .search(&SearchRequest {
+ query: QueryNode::SemanticSearch {
+ field: "body".into(),
+ text: "a distinct already-expired semantic request".into(),
+ k: 5,
+ filter: None,
+ boost: None,
+ },
+ timeout_ms: Some(0),
+ size: 5,
+ ..SearchRequest::default()
+ })
+ .await
+ .unwrap();
+ assert!(expired.timed_out);
+ assert_eq!(expired.total.relation, TotalHitsRelation::Gte);
+ assert!(
+ expired.hits.is_empty(),
+ "an expired request must not publish ANN candidates"
+ );
+ assert!(
+ idx.query_cache.iter().all(|entry| !entry.value().timed_out),
+ "timed-out semantic results must never enter the query cache"
+ );
+
+ let mut filtered_request = semantic_request.clone();
+ filtered_request.query = QueryNode::SemanticSearch {
+ field: "body".into(),
+ text: "quarterly revenue item 777".into(),
+ k: 5,
+ filter: Some(Box::new(QueryNode::MatchAll)),
+ boost: None,
+ };
+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
+ let filtered = idx.search(&filtered_request).await.unwrap();
+ assert_eq!(filtered.hits.len(), 5);
+ assert!(
+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed) > 0,
+ "filtered semantic queries must keep the exact scan"
+ );
+
+ let mut aggregate_request = semantic_request.clone();
+ aggregate_request.aggs = Some(serde_json::json!({
+ "ids": { "terms": { "field": "summary" } }
+ }));
+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
+ let aggregated = idx.search(&aggregate_request).await.unwrap();
+ assert!(aggregated.aggs.is_some());
+ assert!(
+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed) > 0,
+ "semantic aggregations must keep the exact scan"
+ );
+
+ // One graph has one pinned field. A second semantic companion must
+ // never be mixed into that graph; its query safely stays exact.
+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
+ let summary_exact = idx
+ .search(&SearchRequest {
+ query: QueryNode::SemanticSearch {
+ field: "summary".into(),
+ text: "cash flow".into(),
+ k: 5,
+ filter: None,
+ boost: None,
+ },
+ size: 5,
+ ..SearchRequest::default()
+ })
+ .await
+ .unwrap();
+ assert_eq!(summary_exact.hits.len(), 5);
+ assert!(
+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed) > 0,
+ "the non-pinned semantic companion must use the safe exact fallback"
+ );
+
+ idx.index_document(
+ Some("s777".into()),
+ serde_json::json!({
+ "body": "unrelated replacement",
+ "summary": "cash flow note 777"
+ }),
+ )
+ .await
+ .unwrap();
+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);
+ let after_update = idx.search(&semantic_request).await.unwrap();
+ assert_ne!(
+ after_update.hits[0].id, "s777",
+ "a write must invalidate the cached semantic result"
+ );
+ assert_eq!(
+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed),
+ 0,
+ "cache invalidation must recompute through ANN, not force brute"
+ );
+
+ idx.flush().await.unwrap();
+ drop(idx);
+ drop(engine_instance);
+ let restarted_engine = engine(&dir);
+ let restarted = restarted_engine.get_index("semantic-ann").unwrap();
+ let stats = restarted.hnsw_stats().await;
+ assert_eq!(stats["field"], "body_vector", "{stats}");
+ assert_eq!(stats["covered"], true, "{stats}");
+ restarted
+ .test_scan_checkpoint_count
+ .store(0, Ordering::Relaxed);
+ let restarted_result = restarted.search(&semantic_request).await.unwrap();
+ assert_eq!(restarted_result.hits[0].id, after_update.hits[0].id);
+ assert_eq!(restarted_result.hits[0].score, after_update.hits[0].score);
+ assert_eq!(
+ restarted.test_scan_checkpoint_count.load(Ordering::Relaxed),
+ 0,
+ "the restored graph must serve semantic ANN after restart"
+ );
+ }
+
#[tokio::test]
async fn timed_out_leader_is_not_cached() {
let dir = TempDir::new().unwrap();
@@ -6545,9 +6794,11 @@ impl Index {
/// paths. Accepted divergences from brute (commit body documents them):
/// tie order is graph-return order, and `total.value` may be < k when
/// the graph returns fewer candidates (ES ANN behavior class; no pad).
+ #[allow(clippy::too_many_arguments)] // query contract plus cooperative deadline
async fn run_knn_hnsw(
&self,
request: &SearchRequest,
+ deadline: std::time::Instant,
field: &str,
query_vec: &[f32],
k: usize,
@@ -6555,6 +6806,9 @@ impl Index {
similarity: &str,
) -> Option<SearchResult> {
let started = std::time::Instant::now();
+ if std::time::Instant::now() >= deadline {
+ return None;
+ }
if similarity != "cosine" {
return None;
}
@@ -6623,6 +6877,9 @@ impl Index {
};
#[cfg(test)]
self.publication_test_point("", PublicationTestPoint::AnnAfterGraphSearch);
+ if std::time::Instant::now() >= deadline {
+ return None;
+ }
// node_id → doc_id under a short id_rev read. Nodes without a rev
// entry were deleted between search and mapping — skip them (brute
@@ -6646,10 +6903,15 @@ impl Index {
let chunk_field = format!("{field}_chunks");
let mut scored: Vec<(String, f32, Value, Option<u32>)> = Vec::with_capacity(doc_ids.len());
for doc_id in doc_ids {
- let src = match self.get_document_uncounted(&doc_id).await {
- Ok(Some(s)) => s,
- _ => return None,
- };
+ let remaining = deadline.saturating_duration_since(std::time::Instant::now());
+ if remaining.is_zero() {
+ return None;
+ }
+ let src =
+ match tokio::time::timeout(remaining, self.get_document_uncounted(&doc_id)).await {
+ Ok(Ok(Some(source))) => source,
+ _ => return None,
+ };
// Multi-chunk semantic docs score by best passage in brute —
// single-vector rescore would diverge, so hand the request back.
if get_field_value(&src, &chunk_field).is_some() {
@@ -6668,6 +6930,9 @@ impl Index {
let score = compute_vector_similarity(similarity, query_vec, &doc_vec);
scored.push((doc_id, score, src, None));
}
+ if std::time::Instant::now() >= deadline {
+ return None;
+ }
// A writer may start after the pre-check and finish while ANN searches
// or hydrates. Never return candidates from a graph generation that
@@ -7312,8 +7577,41 @@ impl Index {
tracing::info!(index=%self.name, elapsed_ms=semantic_started.elapsed().as_millis() as u64, "semantic_phase=embed_complete");
}
- let result = self
- .run_knn_brute_force_with_deadline(
+ // Match the top-level kNN serving policy: plain, unfiltered semantic
+ // queries may use the fully covered HNSW graph and are exact-rescored
+ // against stored vectors. Filters and aggregations remain exact scans
+ // so approximation cannot change filter/analytics membership.
+ let result = if filter.is_none() && request.aggs.is_none() {
+ match self
+ .run_knn_hnsw(
+ request,
+ deadline,
+ &knn_field,
+ &query_vec,
+ k,
+ None,
+ &similarity,
+ )
+ .await
+ {
+ Some(result) => Ok(result),
+ None => {
+ self.run_knn_brute_force_with_deadline(
+ request,
+ deadline,
+ &knn_field,
+ &query_vec,
+ k,
+ filter,
+ &similarity,
+ None,
+ None,
+ )
+ .await
+ }
+ }
+ } else {
+ self.run_knn_brute_force_with_deadline(
request,
deadline,
&knn_field,
@@ -7324,7 +7622,8 @@ impl Index {
None,
None,
)
- .await;
+ .await
+ };
if trace_phases {
tracing::info!(index=%self.name, elapsed_ms=semantic_started.elapsed().as_millis() as u64, ok=result.is_ok(), "semantic_phase=request_complete");
}
@@ -9335,7 +9634,15 @@ impl Index {
&& request.aggs.is_none()
{
if let Some(result) = self
- .run_knn_hnsw(request, &field, &query_vec, k, num_candidates, &similarity)
+ .run_knn_hnsw(
+ request,
+ search_deadline,
+ &field,
+ &query_vec,
+ k,
+ num_candidates,
+ &similarity,
+ )
.await
{
return Ok(result);
@@ -13078,29 +13385,47 @@ impl Index {
/// Add a field to the schema.
pub async fn add_field(&self, field: FieldConfig) -> Result<()> {
- let activates_embedder = schema_needs_embedder(std::slice::from_ref(&field))
- && !schema_needs_embedder(&self.schema.read().await.schema.fields);
+ self.add_fields(vec![field]).await
+ }
+
+ /// Atomically add several fields to the schema.
+ ///
+ /// Mapping updates can introduce a semantic source and its derived vector
+ /// companion together. Validate and persist the complete candidate schema
+ /// before publishing it so a later-field failure cannot leave half of that
+ /// contract visible.
+ pub async fn add_fields(&self, fields: Vec<FieldConfig>) -> Result<()> {
+ if fields.is_empty() {
+ return Ok(());
+ }
+ let mut schema = self.schema.write().await;
+ let current = schema.clone();
+ let activates_embedder =
+ schema_needs_embedder(&fields) && !schema_needs_embedder(&current.schema.fields);
let activated = if activates_embedder {
Some(make_embedder(&self.embedding_config)?)
} else {
None
};
- if schema_has_embeddings(std::slice::from_ref(&field)) {
+ if schema_has_embeddings(&fields) {
if self
.embedding_config
.mode
.eq_ignore_ascii_case("onnx-experimental")
{
- validate_onnx_dimensions(std::slice::from_ref(&field))?;
+ validate_onnx_dimensions(&fields)?;
}
ensure_embedding_identity_for_new_field(&self.data_dir, &self.embedding_config)?;
}
+ let mut candidate = current;
+ for field in fields {
+ candidate.add_field(field)?;
+ }
+ self.save_schema(&candidate).await?;
if let Some(embedder) = activated {
*self.embedder.write().await = embedder;
}
- let mut schema = self.schema.write().await;
- schema.add_field(field)?;
- self.save_schema(&schema).await?;
+ *schema = candidate;
Ok(())
}
diff --git a/engine/reports/2026-07-28_semantic-companion-hnsw.md b/engine/reports/2026-07-28_semantic-companion-hnsw.md
new file mode 100644
index 000000000..bc1c350da
--- /dev/null
+++ b/engine/reports/2026-07-28_semantic-companion-hnsw.md
@@ -0,0 +1,121 @@
+# Semantic companion HNSW publication
+
+## Outcome
+
+`semantic_text` mappings now register their derived embedding target as an
+internal native `Vector` field. The engine can therefore pin that target as
+the index HNSW field, insert every eligible semantic document, persist the
+graph, restore it after restart, and serve plain semantic queries through HNSW
+with exact rescoring.
+
+This is a correctness/path-selection fix, not a throughput benchmark. No
+speedup claim is made here.
+
+## Root cause
+
+The API mapping converter attached `EmbeddingConfig` to the public text field
+but did not add its derived target to the engine schema. HNSW intentionally
+pins only native `FieldType::Vector` mappings so arbitrary numeric arrays do
+not become graphs. Consequently semantic embeddings were written into
+`_source`, but HNSW skipped them and semantic queries always used the exact
+document scan.
+
+Separately, the semantic executor called the brute-force function directly.
+Even a correctly populated graph could not serve a `semantic` query.
+
+## Change
+
+- Add an internal vector companion for an implicit or custom semantic target,
+ preserving dimensions and similarity.
+- Never overwrite an explicit companion.
+- Keep the raw public mapping as the mapping round-trip; the implicit
+ companion is engine state, not a second user-declared field.
+- Route eligible plain semantic queries through the existing HNSW gate and
+ exact-rescore tail. Filtered queries, aggregation queries, non-pinned fields,
+ incomplete/stale graphs, non-cosine mappings, small indices, and all other
+ ineligible shapes retain exact fallback.
+- Thread the request deadline through graph admission and candidate hydration.
+ Expired work falls back to the cooperative exact timeout path and never
+ publishes partial ANN candidates or caches a timed-out result.
+- Add atomic multi-field schema publication so a semantic source and companion
+ cannot become partially visible.
+
+## Mapping policy and pre-existing inconsistencies now rejected
+
+The flagship fix exposed mapping states that previously returned success while
+raw mapping and engine schema diverged. This change rejects them before
+mutation:
+
+- a companion explicitly mapped to a non-vector type;
+- explicit target dimensions or similarity that differ from the semantic
+ producer;
+- multiple semantic fields sharing one target;
+- changing an existing semantic field's target, dimensions, or similarity via
+ `PUT _mapping`;
+- nested `semantic_text` fields, which the ingest collector does not support;
+- a custom dotted target resolved only through nested properties, or an
+ ambiguous mapping containing both literal and nested definitions for the
+ same logical path.
+
+A literal top-level dotted target such as `"semantic.embedding"` remains
+supported and has an API bulk/query regression test.
+
+## Focused proof
+
+The engine fixture indexes 1,024 documents with two semantic fields and proves:
+
+- the first companion is pinned with 1,024/1,024 graph coverage;
+- the body semantic query takes ANN (zero exact-scan checkpoints);
+- the fixed exact winner is `s777`, with the same exact-rescored score as the
+ brute oracle;
+- measured recall@5 is at least 0.8 on the fixed repetitive fixture (the
+ unique winner is stable; remaining ties intentionally do not promise
+ ordering);
+- `hits.total.value` remains `k`;
+- expired queries return timed-out partial semantics and are not cached;
+- filtered and aggregation semantic requests stay exact;
+- a second, non-pinned semantic field stays exact instead of mixing fields;
+- cached IDs/scores are stable and a write invalidates the cached answer;
+- the graph remains covered and serves ANN after flush and fresh-engine
+ restart.
+
+The API fixture uses real create-index and bulk routes for 1,024 documents and
+proves full HNSW coverage, semantic query success, public mapping round-trip,
+flush/restart restoration, and literal dotted custom-target operation.
+Existing phase-attribution coverage also proves successful semantic bulk
+publication emits ordered Embed/WAL/Memtable/HNSW records with HNSW
+`outcome=ok` and `vectors=1`.
+
+## Verification
+
+All commands ran in
+`/workspace/xerj-fix-semantic-companion-hnsw/engine`.
+
+- `cargo clippy -p xerj-engine -p xerj-api --all-targets -- -D warnings`
+ passed.
+- `cargo test -p xerj-engine -p xerj-api --lib` passed:
+ `237` engine and `74` API tests.
+- `cargo test --release -p xerj-engine fully_covered_semantic_companion_uses_ann_with_exact_rescore --lib -- --nocapture`
+ passed `1/1`.
+- `cargo test --release -p xerj-api semantic_companion --lib -- --nocapture`
+ passed `9/9`.
+- `cargo fmt --all -- --check` and `git diff --check` passed.
+- A fresh branch-local
+ `cargo build --release -j 32 -p xerj-server -p es-yaml-runner` passed.
+ Server SHA-256:
+ `ea94d57d59064e9846030178ad0913199c19fe04d4ae321eeb66f2913b7dc7e6`.
+ Runner SHA-256:
+ `7adb54815499b793b5bdbe7f4340427e81c9e101b34e04a76bc2fd5965d1987b`.
+- The built server used fresh data directory
+ `/tmp/xerj-semantic-conformance.wh7jtb`.
+ `./target/release/es-yaml-runner --dir tests/es-compat-yaml/yaml --verbose`
+ passed `1360`, failed `0`, skipped `3` (`1363` total).
+
+## Excluded attempt
+
+An earlier default-feature server build was interrupted with exit code `130`
+after concurrent worktree build provenance was initially misidentified. Its
+remaining branch-local compiler process was terminated, no artifact was used,
+and no result from that attempt is included above. The recorded hashes and
+conformance result come only from the later clean branch-local build after all
+competing Cargo/Rust compiler activity had cleared.