AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(semantic): publish companion vectors to HNSW
buger983bd4c9d00b
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 335830bcb..2e090d1cc 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -683,6 +683,10 @@ pub async fn create_index(let e = xerj_common::XerjError::invalid_mapping(reason);return ApiError::new(e).into_response();}+ if let Err(reason) = validate_semantic_companion_targets(props) {+ let e = xerj_common::XerjError::invalid_mapping(reason);+ return ApiError::new(e).into_response();+ }}// Runtime mapping type validation too.if let Some(rt_obj) = mappings_val.get("runtime").and_then(Value::as_object) {@@ -839,6 +843,135 @@ fn validate_properties(props: &serde_json::Map<String, Value>) -> Result<(), StrOk(())}+fn validate_semantic_companion_targets(+ props: &serde_json::Map<String, Value>,+) -> Result<(), String> {+ fn walk(props: &serde_json::Map<String, Value>, parent: Option<&str>) -> Result<(), String> {+ let mut producer_by_target: std::collections::HashMap<String, String> =+ std::collections::HashMap::new();+ for (name, spec) in props {+ let Some(definition) = spec.as_object() else {+ continue;+ };+ if definition.get("type").and_then(Value::as_str) == Some("semantic_text") {+ if let Some(parent) = parent {+ return Err(format!(+ "nested semantic_text field [{parent}.{name}] is not supported; \+ map semantic_text as a top-level field"+ ));+ }+ let target = definition+ .get("target_field")+ .and_then(Value::as_str)+ .map(str::to_string)+ .unwrap_or_else(|| format!("{name}_vector"));+ if let Some(first) = producer_by_target.insert(target.clone(), name.clone()) {+ return Err(format!(+ "semantic_text fields [{first}] and [{name}] both target [{target}]; \+ each semantic field must use a distinct target_field"+ ));+ }+ let explicit_targets = mapping_definitions_for_logical_path(props, &target);+ if explicit_targets.len() > 1 {+ return Err(format!(+ "semantic_text field [{name}] target [{target}] is ambiguous because both \+ literal and nested mapping paths resolve to that name; keep only one"+ ));+ }+ if props.get(&target).is_none() && !explicit_targets.is_empty() {+ return Err(format!(+ "semantic_text field [{name}] target [{target}] resolves to a nested mapping; \+ custom dotted target_field values must be declared as a literal top-level \+ property key"+ ));+ }+ if let Some(explicit) = explicit_targets.first() {+ let explicit_type = explicit+ .get("type")+ .and_then(Value::as_str)+ .unwrap_or("object");+ if explicit_type != "dense_vector" {+ return Err(format!(+ "semantic_text field [{name}] targets [{target}], but that field is \+ explicitly mapped as [{explicit_type}]; map it as [dense_vector] or \+ choose another target_field"+ ));+ }+ let semantic_dims = definition+ .get("dimensions")+ .or_else(|| definition.get("dims"))+ .and_then(Value::as_u64)+ .unwrap_or(384);+ if let Some(target_dims) = explicit.get("dims").and_then(Value::as_u64) {+ if target_dims != semantic_dims {+ return Err(format!(+ "semantic_text field [{name}] produces {semantic_dims} dimensions, \+ but target field [{target}] declares {target_dims}"+ ));+ }+ }+ let semantic_similarity = definition+ .get("similarity")+ .and_then(Value::as_str)+ .unwrap_or("cosine");+ let target_similarity = explicit+ .get("similarity")+ .and_then(Value::as_str)+ .unwrap_or("cosine");+ if semantic_similarity != target_similarity {+ return Err(format!(+ "semantic_text field [{name}] uses similarity [{semantic_similarity}], \+ but target field [{target}] declares [{target_similarity}]"+ ));+ }+ }+ }+ if let Some(sub) = definition.get("properties").and_then(Value::as_object) {+ let path = parent+ .map(|prefix| format!("{prefix}.{name}"))+ .unwrap_or_else(|| name.clone());+ walk(sub, Some(&path))?;+ }+ }+ Ok(())+ }+ walk(props, None)+}++fn mapping_definitions_for_logical_path<'a>(+ props: &'a serde_json::Map<String, Value>,+ path: &str,+) -> Vec<&'a serde_json::Map<String, Value>> {+ let mut matches = Vec::new();+ if let Some(definition) = props.get(path).and_then(Value::as_object) {+ matches.push(definition);+ }+ if path.contains('.') {+ let segments: Vec<&str> = path.split('.').collect();+ let mut current = props;+ let mut found = None;+ for (position, segment) in segments.iter().enumerate() {+ let Some(definition) = current.get(*segment).and_then(Value::as_object) else {+ found = None;+ break;+ };+ if position + 1 == segments.len() {+ found = Some(definition);+ break;+ }+ let Some(children) = definition.get("properties").and_then(Value::as_object) else {+ found = None;+ break;+ };+ current = children;+ }+ if let Some(definition) = found {+ matches.push(definition);+ }+ }+ matches+}+fn validate_runtime_fields(rt: &serde_json::Map<String, Value>) -> Result<(), String> {for (name, spec) in rt {let Some(obj) = spec.as_object() else {@@ -1363,6 +1496,10 @@ pub async fn put_mapping(let e = xerj_common::XerjError::invalid_mapping(reason);return ApiError::new(e).into_response();}+ if let Err(reason) = validate_semantic_companion_targets(props) {+ let e = xerj_common::XerjError::invalid_mapping(reason);+ return ApiError::new(e).into_response();+ }}// Reject type changes on existing fields. ES disallows mutating a@@ -1405,6 +1542,42 @@ pub async fn put_mapping().into_response();}+ if new_type == "semantic_text" {+ let old_definition = mapping+ .get("properties")+ .and_then(Value::as_object)+ .and_then(|properties| properties.get(key))+ .and_then(Value::as_object);+ let new_definition = val.as_object();+ if let (Some(old_definition), Some(new_definition)) =+ (old_definition, new_definition)+ {+ let old_contract = semantic_mapping_contract(key, old_definition);+ let new_contract = semantic_mapping_contract(key, new_definition);+ if old_contract != new_contract {+ let reason = format!(+ "semantic_text field [{key}] embedding contract cannot be \+ changed in place (target_field, dimensions, and similarity \+ are immutable); create a new field or reindex"+ );+ return (+ StatusCode::BAD_REQUEST,+ Json(json!({+ "error": {+ "root_cause": [{+ "type": "illegal_argument_exception",+ "reason": reason,+ }],+ "type": "illegal_argument_exception",+ "reason": reason,+ },+ "status": 400,+ })),+ )+ .into_response();+ }+ }+ }}}}@@ -1415,12 +1588,6 @@ pub async fn put_mapping(Ok(i) => i,Err(_) => continue,};- if let Some(properties) = body.get("properties") {- let fields = es_properties_to_fields(properties);- for field in fields {- let _ = idx.add_field(field).await;- }- }// Merge the new properties into the stored mappings blob so// round-trip GET /_mapping reflects both create-time and// put_mapping-time field definitions. Dotted keys in the request@@ -1457,6 +1624,37 @@ pub async fn put_mapping(} else {existing = body.clone();}++ // Semantic companion validation must see the complete post-update+ // mapping, not only this request. Otherwise a target declared by an+ // earlier request can collide with a newly added semantic source.+ if let Some(merged_props) = existing.get("properties").and_then(Value::as_object) {+ if let Err(reason) = validate_semantic_companion_targets(merged_props) {+ let e = xerj_common::XerjError::invalid_mapping(reason);+ return ApiError::new(e).into_response();+ }+ }++ if let Some(properties) = body.get("properties") {+ let current = idx.schema().await;+ let mut additions = Vec::new();+ for field in es_properties_to_fields(properties) {+ if let Some(existing_field) = current.field(&field.name) {+ if existing_field.field_type != field.field_type {+ let e = xerj_common::XerjError::invalid_mapping(format!(+ "field [{}] already exists as [{}], cannot add [{}]",+ field.name, existing_field.field_type, field.field_type+ ));+ return ApiError::new(e).into_response();+ }+ continue;+ }+ additions.push(field);+ }+ if let Err(error) = idx.add_fields(additions).await {+ return ApiError::new(xerj_common::XerjError::from(error)).into_response();+ }+ }// Persists es_mapping.json into the index dir (atomic) so the// merged mapping survives a restart.state.engine.put_index_mapping(idx_name, existing);@@ -1465,6 +1663,28 @@ pub async fn put_mapping(Json(json!({ "acknowledged": true })).into_response()}+fn semantic_mapping_contract(+ field_name: &str,+ definition: &serde_json::Map<String, Value>,+) -> (String, u64, String) {+ let target = definition+ .get("target_field")+ .and_then(Value::as_str)+ .map(str::to_string)+ .unwrap_or_else(|| format!("{field_name}_vector"));+ let dimensions = definition+ .get("dimensions")+ .or_else(|| definition.get("dims"))+ .and_then(Value::as_u64)+ .unwrap_or(384);+ let similarity = definition+ .get("similarity")+ .and_then(Value::as_str)+ .unwrap_or("cosine")+ .to_string();+ (target, dimensions, similarity)+}+/// Walk a mapping blob's `properties` tree and build a flat map of/// dotted-path → leaf type. Both literal-dotted keys (subobjects:false/// leaves) and nested `{properties:{...}}` children are accommodated,@@ -12572,9 +12792,20 @@ fn es_properties_to_fields(properties: &Value) -> Vec<FieldConfig> {.map(|d| d as usize).unwrap_or(384);fc.options.dimensions = Some(dims);- // Built-in embeddings are L2-normalised → cosine similarity.- fc.options.similarity = Some("cosine".to_string());- let target_field = format!("{field_name}_vector");+ // Built-in embeddings are L2-normalised, so cosine is the+ // default. Honour an explicit similarity for proxy/custom+ // embedders and carry it onto the internal vector companion.+ let similarity = field_def+ .get("similarity")+ .and_then(Value::as_str)+ .unwrap_or("cosine")+ .to_string();+ fc.options.similarity = Some(similarity);+ let target_field = field_def+ .get("target_field")+ .and_then(Value::as_str)+ .map(str::to_string)+ .unwrap_or_else(|| format!("{field_name}_vector"));// Allow a mapping to point at an external inference endpoint/model// (`inference_id` mirrors ES 8.x semantic_text wiring); when absent// the engine falls back to the zero-config built-in embedder.@@ -12638,9 +12869,48 @@ fn es_properties_to_fields(properties: &Value) -> Vec<FieldConfig> {fields.push(fc);}++ // A semantic_text field writes its embedding to a companion field. That+ // companion is internal when the caller did not declare it explicitly,+ // but it still has to be a native Vector in the engine schema: HNSW pins+ // only mapped vector fields and deliberately ignores arbitrary numeric+ // arrays. Add companions after parsing all public fields so declaration+ // order cannot overwrite an explicit target.+ let semantic_companions: Vec<FieldConfig> = fields+ .iter()+ .filter_map(|field| {+ let embedding = field.embedding.as_ref()?;+ let target = embedding+ .target_field+ .clone()+ .unwrap_or_else(|| format!("{}_vector", field.name));+ if field_configs_contain_logical_path(&fields, &target) {+ return None;+ }+ let mut companion = FieldConfig::new(target, FieldType::Vector);+ companion.options.dimensions = field.options.dimensions;+ companion.options.similarity = field.options.similarity.clone();+ Some(companion)+ })+ .collect();+ fields.extend(semantic_companions);fields}+fn field_configs_contain_logical_path(fields: &[FieldConfig], target: &str) -> bool {+ fn walk(fields: &[FieldConfig], prefix: &str, target: &str) -> bool {+ fields.iter().any(|field| {+ let path = if prefix.is_empty() {+ field.name.clone()+ } else {+ format!("{prefix}.{}", field.name)+ };+ path == target || (!field.fields.is_empty() && walk(&field.fields, &path, target))+ })+ }+ walk(fields, "", target)+}+fn es_type_to_native(es_type: &str) -> FieldType {match es_type {// `semantic_text` carries plain text in `_source` (the derived@@ -12667,6 +12937,713 @@ fn es_type_to_native(es_type: &str) -> FieldType {}}+#[cfg(test)]+mod semantic_companion_schema_tests {+ use super::*;++ #[test]+ fn implicit_companion_is_a_native_vector_with_semantic_options() {+ let fields = es_properties_to_fields(&json!({+ "body": {+ "type": "semantic_text",+ "dimensions": 96,+ "similarity": "dot_product"+ },+ "ports": { "type": "object" }+ }));++ let body = fields.iter().find(|field| field.name == "body").unwrap();+ assert_eq!(+ body.embedding.as_ref().unwrap().target_field.as_deref(),+ Some("body_vector")+ );+ let companion = fields+ .iter()+ .find(|field| field.name == "body_vector")+ .unwrap();+ assert_eq!(companion.field_type, FieldType::Vector);+ assert_eq!(companion.options.dimensions, Some(96));+ assert_eq!(companion.options.similarity.as_deref(), Some("dot_product"));+ assert!(+ fields.iter().all(|field| field.name != "ports_vector"),+ "ordinary fields and numeric arrays must not acquire vector mappings"+ );+ }++ #[test]+ fn custom_target_is_registered_without_overwriting_an_explicit_companion() {+ let fields = es_properties_to_fields(&json!({+ "body": {+ "type": "semantic_text",+ "target_field": "model_output",+ "dimensions": 7,+ "similarity": "l2_norm"+ },+ "model_output": {+ "type": "dense_vector",+ "dims": 7,+ "similarity": "l2_norm"+ }+ }));++ assert_eq!(+ fields+ .iter()+ .filter(|field| field.name == "model_output")+ .count(),+ 1,+ "the explicit target must not be replaced or duplicated"+ );+ let companion = fields+ .iter()+ .find(|field| field.name == "model_output")+ .unwrap();+ assert_eq!(companion.field_type, FieldType::Vector);+ assert_eq!(companion.options.dimensions, Some(7));+ assert_eq!(companion.options.similarity.as_deref(), Some("l2_norm"));+ }++ #[test]+ fn explicit_companion_collisions_report_actionable_mismatches() {+ let wrong_type = json!({+ "body": { "type": "semantic_text", "target_field": "model_output" },+ "model_output": { "type": "double" }+ });+ let error =+ validate_semantic_companion_targets(wrong_type.as_object().unwrap()).unwrap_err();+ assert!(error.contains("explicitly mapped as [double]"), "{error}");+ assert!(error.contains("map it as [dense_vector]"), "{error}");++ let wrong_dims = json!({+ "body": {+ "type": "semantic_text",+ "target_field": "model_output",+ "dimensions": 384+ },+ "model_output": { "type": "dense_vector", "dims": 768 }+ });+ let error =+ validate_semantic_companion_targets(wrong_dims.as_object().unwrap()).unwrap_err();+ assert!(error.contains("produces 384 dimensions"), "{error}");+ assert!(error.contains("declares 768"), "{error}");++ let wrong_similarity = json!({+ "body": {+ "type": "semantic_text",+ "target_field": "model_output",+ "similarity": "dot_product"+ },+ "model_output": {+ "type": "dense_vector",+ "dims": 384,+ "similarity": "cosine"+ }+ });+ let error =+ validate_semantic_companion_targets(wrong_similarity.as_object().unwrap()).unwrap_err();+ assert!(error.contains("uses similarity [dot_product]"), "{error}");+ assert!(error.contains("declares [cosine]"), "{error}");++ let nested = json!({+ "document": {+ "type": "object",+ "properties": {+ "body": { "type": "semantic_text" }+ }+ }+ });+ let error = validate_semantic_companion_targets(nested.as_object().unwrap()).unwrap_err();+ assert!(+ error.contains("nested semantic_text field [document.body] is not supported"),+ "{error}"+ );++ let nested_target = json!({+ "body": {+ "type": "semantic_text",+ "target_field": "semantic.embedding",+ "dimensions": 32+ },+ "semantic": {+ "properties": {+ "embedding": { "type": "dense_vector", "dims": 32 }+ }+ }+ });+ let error =+ validate_semantic_companion_targets(nested_target.as_object().unwrap()).unwrap_err();+ assert!(+ error.contains("must be declared as a literal top-level property key"),+ "{error}"+ );+ }++ #[test]+ fn multiple_semantic_fields_pin_only_declared_companions() {+ let fields = es_properties_to_fields(&json!({+ "body": { "type": "semantic_text", "dimensions": 8 },+ "summary": {+ "type": "semantic_text",+ "target_field": "summary_embedding",+ "dimensions": 16+ },+ "measurements": { "type": "object" }+ }));++ let vectors: Vec<_> = fields+ .iter()+ .filter(|field| field.field_type == FieldType::Vector)+ .map(|field| (field.name.as_str(), field.options.dimensions))+ .collect();+ assert_eq!(+ vectors,+ vec![("body_vector", Some(8)), ("summary_embedding", Some(16))]+ );+ }+}++#[cfg(test)]+mod semantic_companion_hnsw_api_tests {+ use super::*;+ use axum::{+ body::{to_bytes, Body},+ http::{Request, StatusCode},+ };+ use tower::ServiceExt;+ use xerj_common::{+ config::{Config, WalSync},+ metrics::Metrics,+ };+ use xerj_engine::Engine;++ async fn request_json(+ state: AppState,+ method: &str,+ path: &str,+ content_type: Option<&str>,+ body: impl Into<Body>,+ ) -> (StatusCode, Value) {+ let mut builder = Request::builder().method(method).uri(path);+ if let Some(content_type) = content_type {+ builder = builder.header("content-type", content_type);+ }+ let response = crate::router::build_es_compat_router(state)+ .oneshot(builder.body(body.into()).unwrap())+ .await+ .unwrap();+ let status = response.status();+ let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();+ let json = serde_json::from_slice(&bytes).unwrap_or_else(|error| {+ panic!(+ "{method} {path} returned non-JSON ({error}): {}",+ String::from_utf8_lossy(&bytes)+ )+ });+ (status, json)+ }++ fn state(config: &Config) -> AppState {+ let engine = Engine::new(config.clone()).unwrap();+ let metrics = Metrics::new().unwrap();+ AppState::new(config.clone(), engine, metrics)+ }++ async fn assert_complete_semantic_graph(state: &AppState) {+ let index = state.engine.get_index("semantic-api").unwrap();+ let stats = index.hnsw_stats().await;+ assert_eq!(stats["present"], true, "{stats}");+ assert_eq!(stats["field"], "body_vector", "{stats}");+ assert_eq!(stats["nodes"], 1024, "{stats}");+ assert_eq!(stats["doc_coverage"], 1024, "{stats}");+ assert_eq!(stats["vector_doc_count"], 1024, "{stats}");+ assert_eq!(stats["covered"], true, "{stats}");+ }++ #[tokio::test]+ async fn api_semantic_bulk_builds_complete_hnsw_and_survives_restart() {+ let data_dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = data_dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = WalSync::Async;++ {+ let state = state(&config);+ let (status, _) = request_json(+ state.clone(),+ "PUT",+ "/semantic-api",+ Some("application/json"),+ json!({+ "mappings": {+ "properties": {+ "body": { "type": "semantic_text", "dimensions": 32 },+ "measurements": { "type": "object" }+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::OK);++ // Public mapping round-trips exactly what the caller declared:+ // the engine-only companion is not exposed as another field.+ let (status, mapping) =+ request_json(state.clone(), "GET", "/semantic-api/_mapping", None, "").await;+ assert_eq!(status, StatusCode::OK);+ let properties = mapping["semantic-api"]["mappings"]["properties"]+ .as_object()+ .unwrap();+ assert_eq!(properties.len(), 2, "{mapping}");+ assert!(!properties.contains_key("body_vector"), "{mapping}");++ let mut bulk = String::new();+ for id in 0..1024 {+ bulk.push_str(&format!(+ "{{\"index\":{{\"_index\":\"semantic-api\",\"_id\":\"{id}\"}}}}\n"+ ));+ bulk.push_str(&format!(+ "{{\"body\":\"quarterly finance report revenue item {id}\",\+ \"measurements\":[{id},{}]}}\n",+ id + 1+ ));+ }+ let (status, response) = request_json(+ state.clone(),+ "POST",+ "/_bulk",+ Some("application/x-ndjson"),+ bulk,+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(response["errors"], false, "{response}");+ assert_eq!(response["items"].as_array().unwrap().len(), 1024);+ assert_complete_semantic_graph(&state).await;++ let (status, response) = request_json(+ state.clone(),+ "POST",+ "/semantic-api/_search",+ Some("application/json"),+ json!({+ "size": 5,+ "query": {+ "semantic": {+ "field": "body",+ "query": "quarterly revenue",+ "k": 5+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{response}");+ assert_eq!(response["hits"]["hits"].as_array().unwrap().len(), 5);++ state.engine.flush_all_force().await;+ }++ let restarted = state(&config);+ assert_complete_semantic_graph(&restarted).await;+ let (status, response) = request_json(+ restarted.clone(),+ "POST",+ "/semantic-api/_search",+ Some("application/json"),+ json!({+ "size": 5,+ "query": {+ "semantic": {+ "field": "body",+ "query": "quarterly revenue",+ "k": 5+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{response}");+ assert_eq!(response["hits"]["hits"].as_array().unwrap().len(), 5);+ restarted.engine.flush_all_force().await;+ }++ #[tokio::test]+ async fn put_mapping_rejects_existing_companion_collision_atomically() {+ let data_dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = data_dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = WalSync::Async;+ let state = state(&config);++ let (status, _) = request_json(+ state.clone(),+ "PUT",+ "/semantic-collision",+ Some("application/json"),+ json!({+ "mappings": {+ "properties": {+ "body_vector": { "type": "double" }+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ let schema_before = state+ .engine+ .get_index("semantic-collision")+ .unwrap()+ .schema()+ .await;++ let (status, error) = request_json(+ state.clone(),+ "PUT",+ "/semantic-collision/_mapping",+ Some("application/json"),+ json!({+ "properties": {+ "body": { "type": "semantic_text" }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");+ assert!(+ error["error"]["reason"]+ .as_str()+ .unwrap()+ .contains("explicitly mapped as [double]"),+ "{error}"+ );+ let schema_after = state+ .engine+ .get_index("semantic-collision")+ .unwrap()+ .schema()+ .await;+ assert_eq!(schema_after, schema_before, "schema update must be atomic");++ let (_, mapping) = request_json(+ state.clone(),+ "GET",+ "/semantic-collision/_mapping",+ None,+ "",+ )+ .await;+ assert!(+ mapping["semantic-collision"]["mappings"]["properties"]+ .get("body")+ .is_none(),+ "{mapping}"+ );+ state.engine.flush_all_force().await;+ }++ #[tokio::test]+ async fn put_mapping_rejects_semantic_contract_mutation_atomically() {+ let data_dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = data_dir.path().to_string_lossy().into_owned();+ let state = state(&config);+ let (status, _) = request_json(+ state.clone(),+ "PUT",+ "/semantic-contract",+ Some("application/json"),+ json!({+ "mappings": {+ "properties": {+ "body": { "type": "semantic_text", "dimensions": 32 }+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ let schema_before = state+ .engine+ .get_index("semantic-contract")+ .unwrap()+ .schema()+ .await;+ let (_, mapping_before) = request_json(+ state.clone(),+ "GET",+ "/semantic-contract/_mapping",+ None,+ "",+ )+ .await;++ for mutation in [+ json!({"type": "semantic_text", "dimensions": 64}),+ json!({+ "type": "semantic_text",+ "dimensions": 32,+ "target_field": "replacement_vector"+ }),+ json!({+ "type": "semantic_text",+ "dimensions": 32,+ "similarity": "dot_product"+ }),+ ] {+ let (status, error) = request_json(+ state.clone(),+ "PUT",+ "/semantic-contract/_mapping",+ Some("application/json"),+ json!({"properties": {"body": mutation}}).to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");+ assert!(+ error["error"]["reason"]+ .as_str()+ .unwrap()+ .contains("embedding contract cannot be changed in place"),+ "{error}"+ );+ assert_eq!(+ state+ .engine+ .get_index("semantic-contract")+ .unwrap()+ .schema()+ .await,+ schema_before+ );+ let (_, mapping_after) = request_json(+ state.clone(),+ "GET",+ "/semantic-contract/_mapping",+ None,+ "",+ )+ .await;+ assert_eq!(mapping_after, mapping_before);+ }+ state.engine.flush_all_force().await;+ }++ #[tokio::test]+ async fn shared_semantic_target_is_rejected_before_index_creation() {+ let data_dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = data_dir.path().to_string_lossy().into_owned();+ let state = state(&config);+ let (status, error) = request_json(+ state.clone(),+ "PUT",+ "/shared-target",+ Some("application/json"),+ json!({+ "mappings": {+ "properties": {+ "body": {+ "type": "semantic_text",+ "target_field": "shared_vector"+ },+ "summary": {+ "type": "semantic_text",+ "target_field": "shared_vector"+ }+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");+ assert!(+ error["error"]["reason"]+ .as_str()+ .unwrap()+ .contains("each semantic field must use a distinct target_field"),+ "{error}"+ );+ assert!(state.engine.get_index("shared-target").is_err());++ let (status, error) = request_json(+ state.clone(),+ "PUT",+ "/nested-custom-target",+ Some("application/json"),+ json!({+ "mappings": {+ "properties": {+ "body": {+ "type": "semantic_text",+ "target_field": "semantic.embedding",+ "dimensions": 32+ },+ "semantic": {+ "properties": {+ "embedding": { "type": "dense_vector", "dims": 32 }+ }+ }+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");+ assert!(+ error["error"]["reason"]+ .as_str()+ .unwrap()+ .contains("literal top-level property key"),+ "{error}"+ );+ assert!(state.engine.get_index("nested-custom-target").is_err());++ let (status, _) = request_json(+ state.clone(),+ "PUT",+ "/nested-put-target",+ Some("application/json"),+ json!({+ "mappings": {+ "properties": {+ "semantic": {+ "properties": {+ "embedding": { "type": "dense_vector", "dims": 32 }+ }+ }+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ let before = state+ .engine+ .get_index("nested-put-target")+ .unwrap()+ .schema()+ .await;+ let (status, error) = request_json(+ state.clone(),+ "PUT",+ "/nested-put-target/_mapping",+ Some("application/json"),+ json!({+ "properties": {+ "body": {+ "type": "semantic_text",+ "target_field": "semantic.embedding",+ "dimensions": 32+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");+ assert_eq!(+ state+ .engine+ .get_index("nested-put-target")+ .unwrap()+ .schema()+ .await,+ before+ );+ }++ #[tokio::test]+ async fn literal_dotted_custom_target_builds_and_queries_hnsw() {+ let data_dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = data_dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = WalSync::Async;+ let state = state(&config);+ let (status, _) = request_json(+ state.clone(),+ "PUT",+ "/dotted-target",+ Some("application/json"),+ json!({+ "mappings": {+ "properties": {+ "body": {+ "type": "semantic_text",+ "target_field": "semantic.embedding",+ "dimensions": 32+ }+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::OK);++ let mut bulk = String::new();+ for id in 0..1024 {+ bulk.push_str(&format!(+ "{{\"index\":{{\"_index\":\"dotted-target\",\"_id\":\"{id}\"}}}}\n"+ ));+ bulk.push_str(&format!(+ "{{\"body\":\"dotted semantic revenue item {id}\"}}\n"+ ));+ }+ let (status, response) = request_json(+ state.clone(),+ "POST",+ "/_bulk",+ Some("application/x-ndjson"),+ bulk,+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(response["errors"], false, "{response}");+ let stats = state+ .engine+ .get_index("dotted-target")+ .unwrap()+ .hnsw_stats()+ .await;+ assert_eq!(stats["field"], "semantic.embedding", "{stats}");+ assert_eq!(stats["doc_coverage"], 1024, "{stats}");+ assert_eq!(stats["covered"], true, "{stats}");++ let (status, response) = request_json(+ state.clone(),+ "POST",+ "/dotted-target/_search",+ Some("application/json"),+ json!({+ "size": 5,+ "query": {+ "semantic": {+ "field": "body",+ "query": "semantic revenue",+ "k": 5+ }+ }+ })+ .to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{response}");+ assert_eq!(response["hits"]["total"]["value"], 5);+ assert_eq!(response["hits"]["hits"].as_array().unwrap().len(), 5);+ state.engine.flush_all_force().await;+ }+}+pub fn schema_to_es_properties(schema: &Schema) -> serde_json::Map<String, Value> {let mut props = serde_json::Map::new();for field in &schema.fields {diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 620ca6275..25801b650 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -546,6 +546,255 @@ mod semantic_deadline_regression_tests {idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);}+ #[tokio::test]+ async fn fully_covered_semantic_companion_uses_ann_with_exact_rescore() {+ let dir = TempDir::new().unwrap();+ let engine_instance = engine(&dir);+ let mut schema = Schema::empty();+ let mut body = FieldConfig::new("body", FieldType::Text);+ body.options.dimensions = Some(32);+ body.options.similarity = Some("cosine".into());+ body.embedding = Some(xerj_common::types::EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: Some("body_vector".into()),+ });+ schema.add_field(body).unwrap();+ let mut companion = FieldConfig::new("body_vector", FieldType::Vector);+ companion.options.dimensions = Some(32);+ companion.options.similarity = Some("cosine".into());+ schema.add_field(companion).unwrap();+ let mut summary = FieldConfig::new("summary", FieldType::Text);+ summary.options.dimensions = Some(32);+ summary.options.similarity = Some("cosine".into());+ summary.embedding = Some(xerj_common::types::EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: Some("summary_vector".into()),+ });+ schema.add_field(summary).unwrap();+ let mut summary_companion = FieldConfig::new("summary_vector", FieldType::Vector);+ summary_companion.options.dimensions = Some(32);+ summary_companion.options.similarity = Some("cosine".into());+ schema.add_field(summary_companion).unwrap();+ engine_instance+ .create_index("semantic-ann", schema)+ .unwrap();+ let idx = engine_instance.get_index("semantic-ann").unwrap();++ for n in 0..1024 {+ idx.index_document(+ Some(format!("s{n}")),+ serde_json::json!({+ "body": format!("quarterly revenue item {n}"),+ "summary": format!("cash flow note {n}")+ }),+ )+ .await+ .unwrap();+ }+ let stats = idx.hnsw_stats().await;+ assert_eq!(stats["field"], "body_vector", "{stats}");+ assert_eq!(stats["doc_coverage"], 1024, "{stats}");+ assert_eq!(stats["vector_doc_count"], 1024, "{stats}");+ assert_eq!(stats["covered"], true, "{stats}");++ let semantic_request = SearchRequest {+ query: QueryNode::SemanticSearch {+ field: "body".into(),+ text: "quarterly revenue item 777".into(),+ k: 5,+ filter: None,+ boost: None,+ },+ size: 5,+ ..SearchRequest::default()+ };+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);+ let body_ann = idx.search(&semantic_request).await.unwrap();+ assert_eq!(body_ann.hits.len(), 5);+ assert_eq!(body_ann.total.value, 5);+ assert_eq!(body_ann.hits[0].id, "s777");+ assert_eq!(+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed),+ 0,+ "an eligible semantic query must use HNSW candidates plus exact rescore, \+ not the brute-force document scan"+ );++ let exact = idx+ .run_knn_brute_force(+ &semantic_request,+ "body_vector",+ &xerj_ai::local::local_embed("quarterly revenue item 777", 32),+ 5,+ None,+ "cosine",+ None,+ None,+ )+ .await+ .unwrap();+ assert_eq!(exact.hits[0].id, "s777");+ assert_eq!(exact.hits[0].score, body_ann.hits[0].score);+ let exact_ids: HashSet<_> = exact.hits.iter().map(|hit| hit.id.as_str()).collect();+ let recalled = body_ann+ .hits+ .iter()+ .filter(|hit| exact_ids.contains(hit.id.as_str()))+ .count();+ assert!(+ recalled >= 4,+ "fixed-oracle recall@5 fell below 0.8: ANN={:?}, exact={:?}",+ body_ann+ .hits+ .iter()+ .map(|hit| (&hit.id, hit.score))+ .collect::<Vec<_>>(),+ exact+ .hits+ .iter()+ .map(|hit| (&hit.id, hit.score))+ .collect::<Vec<_>>()+ );+ // Only the unique exact match has a stable order. Remaining documents+ // tie under this deliberately repetitive corpus; graph-return order is+ // the documented ANN tie contract, while total remains exactly k.+ let cached = idx.search(&semantic_request).await.unwrap();+ assert_eq!(+ cached+ .hits+ .iter()+ .map(|hit| (&hit.id, hit.score))+ .collect::<Vec<_>>(),+ body_ann+ .hits+ .iter()+ .map(|hit| (&hit.id, hit.score))+ .collect::<Vec<_>>(),+ "cached semantic ANN must preserve IDs and exact-rescored scores"+ );++ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);+ let expired = idx+ .search(&SearchRequest {+ query: QueryNode::SemanticSearch {+ field: "body".into(),+ text: "a distinct already-expired semantic request".into(),+ k: 5,+ filter: None,+ boost: None,+ },+ timeout_ms: Some(0),+ size: 5,+ ..SearchRequest::default()+ })+ .await+ .unwrap();+ assert!(expired.timed_out);+ assert_eq!(expired.total.relation, TotalHitsRelation::Gte);+ assert!(+ expired.hits.is_empty(),+ "an expired request must not publish ANN candidates"+ );+ assert!(+ idx.query_cache.iter().all(|entry| !entry.value().timed_out),+ "timed-out semantic results must never enter the query cache"+ );++ let mut filtered_request = semantic_request.clone();+ filtered_request.query = QueryNode::SemanticSearch {+ field: "body".into(),+ text: "quarterly revenue item 777".into(),+ k: 5,+ filter: Some(Box::new(QueryNode::MatchAll)),+ boost: None,+ };+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);+ let filtered = idx.search(&filtered_request).await.unwrap();+ assert_eq!(filtered.hits.len(), 5);+ assert!(+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed) > 0,+ "filtered semantic queries must keep the exact scan"+ );++ let mut aggregate_request = semantic_request.clone();+ aggregate_request.aggs = Some(serde_json::json!({+ "ids": { "terms": { "field": "summary" } }+ }));+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);+ let aggregated = idx.search(&aggregate_request).await.unwrap();+ assert!(aggregated.aggs.is_some());+ assert!(+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed) > 0,+ "semantic aggregations must keep the exact scan"+ );++ // One graph has one pinned field. A second semantic companion must+ // never be mixed into that graph; its query safely stays exact.+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);+ let summary_exact = idx+ .search(&SearchRequest {+ query: QueryNode::SemanticSearch {+ field: "summary".into(),+ text: "cash flow".into(),+ k: 5,+ filter: None,+ boost: None,+ },+ size: 5,+ ..SearchRequest::default()+ })+ .await+ .unwrap();+ assert_eq!(summary_exact.hits.len(), 5);+ assert!(+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed) > 0,+ "the non-pinned semantic companion must use the safe exact fallback"+ );++ idx.index_document(+ Some("s777".into()),+ serde_json::json!({+ "body": "unrelated replacement",+ "summary": "cash flow note 777"+ }),+ )+ .await+ .unwrap();+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);+ let after_update = idx.search(&semantic_request).await.unwrap();+ assert_ne!(+ after_update.hits[0].id, "s777",+ "a write must invalidate the cached semantic result"+ );+ assert_eq!(+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed),+ 0,+ "cache invalidation must recompute through ANN, not force brute"+ );++ idx.flush().await.unwrap();+ drop(idx);+ drop(engine_instance);+ let restarted_engine = engine(&dir);+ let restarted = restarted_engine.get_index("semantic-ann").unwrap();+ let stats = restarted.hnsw_stats().await;+ assert_eq!(stats["field"], "body_vector", "{stats}");+ assert_eq!(stats["covered"], true, "{stats}");+ restarted+ .test_scan_checkpoint_count+ .store(0, Ordering::Relaxed);+ let restarted_result = restarted.search(&semantic_request).await.unwrap();+ assert_eq!(restarted_result.hits[0].id, after_update.hits[0].id);+ assert_eq!(restarted_result.hits[0].score, after_update.hits[0].score);+ assert_eq!(+ restarted.test_scan_checkpoint_count.load(Ordering::Relaxed),+ 0,+ "the restored graph must serve semantic ANN after restart"+ );+ }+#[tokio::test]async fn timed_out_leader_is_not_cached() {let dir = TempDir::new().unwrap();@@ -6545,9 +6794,11 @@ impl Index {/// paths. Accepted divergences from brute (commit body documents them):/// tie order is graph-return order, and `total.value` may be < k when/// the graph returns fewer candidates (ES ANN behavior class; no pad).+ #[allow(clippy::too_many_arguments)] // query contract plus cooperative deadlineasync fn run_knn_hnsw(&self,request: &SearchRequest,+ deadline: std::time::Instant,field: &str,query_vec: &[f32],k: usize,@@ -6555,6 +6806,9 @@ impl Index {similarity: &str,) -> Option<SearchResult> {let started = std::time::Instant::now();+ if std::time::Instant::now() >= deadline {+ return None;+ }if similarity != "cosine" {return None;}@@ -6623,6 +6877,9 @@ impl Index {};#[cfg(test)]self.publication_test_point("", PublicationTestPoint::AnnAfterGraphSearch);+ if std::time::Instant::now() >= deadline {+ return None;+ }// node_id → doc_id under a short id_rev read. Nodes without a rev// entry were deleted between search and mapping — skip them (brute@@ -6646,10 +6903,15 @@ impl Index {let chunk_field = format!("{field}_chunks");let mut scored: Vec<(String, f32, Value, Option<u32>)> = Vec::with_capacity(doc_ids.len());for doc_id in doc_ids {- let src = match self.get_document_uncounted(&doc_id).await {- Ok(Some(s)) => s,- _ => return None,- };+ let remaining = deadline.saturating_duration_since(std::time::Instant::now());+ if remaining.is_zero() {+ return None;+ }+ let src =+ match tokio::time::timeout(remaining, self.get_document_uncounted(&doc_id)).await {+ Ok(Ok(Some(source))) => source,+ _ => return None,+ };// Multi-chunk semantic docs score by best passage in brute —// single-vector rescore would diverge, so hand the request back.if get_field_value(&src, &chunk_field).is_some() {@@ -6668,6 +6930,9 @@ impl Index {let score = compute_vector_similarity(similarity, query_vec, &doc_vec);scored.push((doc_id, score, src, None));}+ if std::time::Instant::now() >= deadline {+ return None;+ }// A writer may start after the pre-check and finish while ANN searches// or hydrates. Never return candidates from a graph generation that@@ -7312,8 +7577,41 @@ impl Index {tracing::info!(index=%self.name, elapsed_ms=semantic_started.elapsed().as_millis() as u64, "semantic_phase=embed_complete");}- let result = self- .run_knn_brute_force_with_deadline(+ // Match the top-level kNN serving policy: plain, unfiltered semantic+ // queries may use the fully covered HNSW graph and are exact-rescored+ // against stored vectors. Filters and aggregations remain exact scans+ // so approximation cannot change filter/analytics membership.+ let result = if filter.is_none() && request.aggs.is_none() {+ match self+ .run_knn_hnsw(+ request,+ deadline,+ &knn_field,+ &query_vec,+ k,+ None,+ &similarity,+ )+ .await+ {+ Some(result) => Ok(result),+ None => {+ self.run_knn_brute_force_with_deadline(+ request,+ deadline,+ &knn_field,+ &query_vec,+ k,+ filter,+ &similarity,+ None,+ None,+ )+ .await+ }+ }+ } else {+ self.run_knn_brute_force_with_deadline(request,deadline,&knn_field,@@ -7324,7 +7622,8 @@ impl Index {None,None,)- .await;+ .await+ };if trace_phases {tracing::info!(index=%self.name, elapsed_ms=semantic_started.elapsed().as_millis() as u64, ok=result.is_ok(), "semantic_phase=request_complete");}@@ -9335,7 +9634,15 @@ impl Index {&& request.aggs.is_none(){if let Some(result) = self- .run_knn_hnsw(request, &field, &query_vec, k, num_candidates, &similarity)+ .run_knn_hnsw(+ request,+ search_deadline,+ &field,+ &query_vec,+ k,+ num_candidates,+ &similarity,+ ).await{return Ok(result);@@ -13078,29 +13385,47 @@ impl Index {/// Add a field to the schema.pub async fn add_field(&self, field: FieldConfig) -> Result<()> {- let activates_embedder = schema_needs_embedder(std::slice::from_ref(&field))- && !schema_needs_embedder(&self.schema.read().await.schema.fields);+ self.add_fields(vec![field]).await+ }++ /// Atomically add several fields to the schema.+ ///+ /// Mapping updates can introduce a semantic source and its derived vector+ /// companion together. Validate and persist the complete candidate schema+ /// before publishing it so a later-field failure cannot leave half of that+ /// contract visible.+ pub async fn add_fields(&self, fields: Vec<FieldConfig>) -> Result<()> {+ if fields.is_empty() {+ return Ok(());+ }+ let mut schema = self.schema.write().await;+ let current = schema.clone();+ let activates_embedder =+ schema_needs_embedder(&fields) && !schema_needs_embedder(¤t.schema.fields);let activated = if activates_embedder {Some(make_embedder(&self.embedding_config)?)} else {None};- if schema_has_embeddings(std::slice::from_ref(&field)) {+ if schema_has_embeddings(&fields) {if self.embedding_config.mode.eq_ignore_ascii_case("onnx-experimental"){- validate_onnx_dimensions(std::slice::from_ref(&field))?;+ validate_onnx_dimensions(&fields)?;}ensure_embedding_identity_for_new_field(&self.data_dir, &self.embedding_config)?;}+ let mut candidate = current;+ for field in fields {+ candidate.add_field(field)?;+ }+ self.save_schema(&candidate).await?;if let Some(embedder) = activated {*self.embedder.write().await = embedder;}- let mut schema = self.schema.write().await;- schema.add_field(field)?;- self.save_schema(&schema).await?;+ *schema = candidate;Ok(())}diff --git a/engine/reports/2026-07-28_semantic-companion-hnsw.md b/engine/reports/2026-07-28_semantic-companion-hnsw.mdnew file mode 100644index 000000000..bc1c350da--- /dev/null+++ b/engine/reports/2026-07-28_semantic-companion-hnsw.md@@ -0,0 +1,121 @@+# Semantic companion HNSW publication++## Outcome++`semantic_text` mappings now register their derived embedding target as an+internal native `Vector` field. The engine can therefore pin that target as+the index HNSW field, insert every eligible semantic document, persist the+graph, restore it after restart, and serve plain semantic queries through HNSW+with exact rescoring.++This is a correctness/path-selection fix, not a throughput benchmark. No+speedup claim is made here.++## Root cause++The API mapping converter attached `EmbeddingConfig` to the public text field+but did not add its derived target to the engine schema. HNSW intentionally+pins only native `FieldType::Vector` mappings so arbitrary numeric arrays do+not become graphs. Consequently semantic embeddings were written into+`_source`, but HNSW skipped them and semantic queries always used the exact+document scan.++Separately, the semantic executor called the brute-force function directly.+Even a correctly populated graph could not serve a `semantic` query.++## Change++- Add an internal vector companion for an implicit or custom semantic target,+ preserving dimensions and similarity.+- Never overwrite an explicit companion.+- Keep the raw public mapping as the mapping round-trip; the implicit+ companion is engine state, not a second user-declared field.+- Route eligible plain semantic queries through the existing HNSW gate and+ exact-rescore tail. Filtered queries, aggregation queries, non-pinned fields,+ incomplete/stale graphs, non-cosine mappings, small indices, and all other+ ineligible shapes retain exact fallback.+- Thread the request deadline through graph admission and candidate hydration.+ Expired work falls back to the cooperative exact timeout path and never+ publishes partial ANN candidates or caches a timed-out result.+- Add atomic multi-field schema publication so a semantic source and companion+ cannot become partially visible.++## Mapping policy and pre-existing inconsistencies now rejected++The flagship fix exposed mapping states that previously returned success while+raw mapping and engine schema diverged. This change rejects them before+mutation:++- a companion explicitly mapped to a non-vector type;+- explicit target dimensions or similarity that differ from the semantic+ producer;+- multiple semantic fields sharing one target;+- changing an existing semantic field's target, dimensions, or similarity via+ `PUT _mapping`;+- nested `semantic_text` fields, which the ingest collector does not support;+- a custom dotted target resolved only through nested properties, or an+ ambiguous mapping containing both literal and nested definitions for the+ same logical path.++A literal top-level dotted target such as `"semantic.embedding"` remains+supported and has an API bulk/query regression test.++## Focused proof++The engine fixture indexes 1,024 documents with two semantic fields and proves:++- the first companion is pinned with 1,024/1,024 graph coverage;+- the body semantic query takes ANN (zero exact-scan checkpoints);+- the fixed exact winner is `s777`, with the same exact-rescored score as the+ brute oracle;+- measured recall@5 is at least 0.8 on the fixed repetitive fixture (the+ unique winner is stable; remaining ties intentionally do not promise+ ordering);+- `hits.total.value` remains `k`;+- expired queries return timed-out partial semantics and are not cached;+- filtered and aggregation semantic requests stay exact;+- a second, non-pinned semantic field stays exact instead of mixing fields;+- cached IDs/scores are stable and a write invalidates the cached answer;+- the graph remains covered and serves ANN after flush and fresh-engine+ restart.++The API fixture uses real create-index and bulk routes for 1,024 documents and+proves full HNSW coverage, semantic query success, public mapping round-trip,+flush/restart restoration, and literal dotted custom-target operation.+Existing phase-attribution coverage also proves successful semantic bulk+publication emits ordered Embed/WAL/Memtable/HNSW records with HNSW+`outcome=ok` and `vectors=1`.++## Verification++All commands ran in+`/workspace/xerj-fix-semantic-companion-hnsw/engine`.++- `cargo clippy -p xerj-engine -p xerj-api --all-targets -- -D warnings`+ passed.+- `cargo test -p xerj-engine -p xerj-api --lib` passed:+ `237` engine and `74` API tests.+- `cargo test --release -p xerj-engine fully_covered_semantic_companion_uses_ann_with_exact_rescore --lib -- --nocapture`+ passed `1/1`.+- `cargo test --release -p xerj-api semantic_companion --lib -- --nocapture`+ passed `9/9`.+- `cargo fmt --all -- --check` and `git diff --check` passed.+- A fresh branch-local+ `cargo build --release -j 32 -p xerj-server -p es-yaml-runner` passed.+ Server SHA-256:+ `ea94d57d59064e9846030178ad0913199c19fe04d4ae321eeb66f2913b7dc7e6`.+ Runner SHA-256:+ `7adb54815499b793b5bdbe7f4340427e81c9e101b34e04a76bc2fd5965d1987b`.+- The built server used fresh data directory+ `/tmp/xerj-semantic-conformance.wh7jtb`.+ `./target/release/es-yaml-runner --dir tests/es-compat-yaml/yaml --verbose`+ passed `1360`, failed `0`, skipped `3` (`1363` total).++## Excluded attempt++An earlier default-feature server build was interrupted with exit code `130`+after concurrent worktree build provenance was initially misidentified. Its+remaining branch-local compiler process was terminated, no artifact was used,+and no result from that attempt is included above. The recorded hashes and+conformance result come only from the later clean branch-local build after all+competing Cargo/Rust compiler activity had cleared.