Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Feat/embedding execution identity (#160)

Leonid Bugaev96f6d1ab6ef4
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d0e9cbf64..590ef201e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -78,6 +78,44 @@ jobs:
working-directory: engine
run: cargo test --release --workspace -- --test-threads=2
+ onnx-feature:
+ name: ONNX feature compile + tests
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install Rust
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ components: clippy
+
+ - name: Cache
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry/cache
+ ~/.cargo/registry/index
+ ~/.cargo/git/db
+ engine/target
+ key: ci-onnx-cargo-${{ hashFiles('engine/Cargo.lock') }}
+ restore-keys: ci-onnx-cargo-
+
+ - name: Compile the server ONNX feature path
+ working-directory: engine
+ run: cargo check -p xerj-server --no-default-features --features onnx-experimental
+
+ - name: Test the ONNX embedder
+ working-directory: engine
+ run: cargo test -p xerj-ai --features onnx-experimental --lib -- --test-threads=2
+
+ - name: Test engine embedding identity with ONNX
+ working-directory: engine
+ run: cargo test -p xerj-engine --features onnx-experimental --lib index::embedding_identity_tests -- --test-threads=2
+
+ - name: Clippy the ONNX implementation
+ working-directory: engine
+ run: cargo clippy -p xerj-ai -p xerj-engine --all-targets --features onnx-experimental -- -D warnings
+
api-checks:
name: API smoke + liveness + benchmark
runs-on: ubuntu-latest
diff --git a/CHANGELOG.md b/CHANGELOG.md
index abb5113ed..d4eb2e17b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,52 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Added
+
+- **`xerj autoindex` pins the server's embedding execution identity** (#160).
+ Vectors produced by two different embedding backends — or by two different
+ models, tokenizers, or vector widths within one backend — are not comparable,
+ so mixing them in one index silently degrades every similarity result rather
+ than failing. A semantic autoindex run now fetches
+ `GET /v1/embedding/identity`, records the returned opaque digest in its
+ journal before the index is created, and requires the same digest on every
+ resume; a mismatch aborts before the next index-create or `_bulk`. There is
+ no new command or setup step — `xerj autoindex ./reports` calls the endpoint
+ itself over the already-configured `--url`, and plans with no semantic fields
+ skip it entirely.
+
+ The endpoint is authenticated (cluster-read) and deliberately opaque: it
+ returns a version, the *effective* backend, a digest, the semantic contract,
+ whether that backend is resumable, and a sanitized reason when it is not. It
+ never returns credentials, provider URLs, model names, or local paths.
+ `dimensions` is reported only for the backends whose width the server
+ actually pins (`lexical`, and `onnx-experimental` at 384); it is omitted for
+ `neural`, whose width comes from the loaded model's `hidden_size`, and for
+ `proxy`, whose width is whatever the remote returns.
+
+ `lexical` and `onnx-experimental` are resumable. `neural` and `proxy` are
+ not: neither attests immutable model bytes, so a fresh semantic run is
+ allowed but a resume is refused.
+
+### Changed
+
+- **BREAKING: an existing semantic `autoindex` state directory cannot be
+ resumed** (#160). A journal written before this change carries no pinned
+ identity, so there is no way to prove the server still produces the same
+ vector space; rather than assume it, the resume fails closed with *"this
+ semantic autoindex journal predates embedding identity pinning and cannot be
+ resumed safely"*. The same path fires when a previous `--no-semantic` run is
+ followed by a semantic run over the same state directory. Recover by
+ rebuilding with `--fresh` and a new `--prefix`; before reusing the old
+ prefix, delete and recreate its prior autoindex indices. Non-semantic runs
+ and brand-new state directories are unaffected.
+
+- `EmbeddingProxy::new` now rejects an endpoint that is not `http(s)` at
+ construction time instead of at first request, so an invalid
+ `embedding.default_endpoint` falls back to lexical at startup and
+ `/v1/embedding/identity` reports `lexical` — the backend the server will
+ really use — rather than `proxy`.
+
## [1.0.0-rc.10] - 2026-08-03
### Security
diff --git a/engine/crates/xerj-ai/src/embed.rs b/engine/crates/xerj-ai/src/embed.rs
index 55e340b66..12eaa2bd0 100644
--- a/engine/crates/xerj-ai/src/embed.rs
+++ b/engine/crates/xerj-ai/src/embed.rs
@@ -139,6 +139,13 @@ pub struct EmbeddingProxy {
impl EmbeddingProxy {
/// Create a new proxy with the given config.
pub fn new(config: EmbeddingProxyConfig) -> Result<Self> {
+ let endpoint = reqwest::Url::parse(&config.endpoint)
+ .map_err(|e| XerjError::embedding(format!("invalid embedding endpoint: {e}")))?;
+ if !matches!(endpoint.scheme(), "http" | "https") {
+ return Err(XerjError::embedding(
+ "embedding endpoint must use http or https",
+ ));
+ }
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(config.timeout_secs))
.build()
@@ -313,6 +320,14 @@ mod tests {
assert_eq!(cfg.api_key.as_deref(), Some("sk-test"));
}
+ #[test]
+ fn proxy_rejects_invalid_or_non_http_endpoints_at_construction() {
+ for endpoint in ["not a url", "file:///private/model"] {
+ let cfg = EmbeddingProxyConfig::new(endpoint, "model");
+ assert!(EmbeddingProxy::new(cfg).is_err(), "{endpoint} was accepted");
+ }
+ }
+
#[tokio::test]
async fn embed_empty_batch_returns_empty() {
let cfg = EmbeddingProxyConfig::new("http://localhost", "model");
diff --git a/engine/crates/xerj-ai/src/embedder.rs b/engine/crates/xerj-ai/src/embedder.rs
index 247881d83..812909c17 100644
--- a/engine/crates/xerj-ai/src/embedder.rs
+++ b/engine/crates/xerj-ai/src/embedder.rs
@@ -283,12 +283,12 @@ impl Embedder {
}
#[cfg(feature = "onnx-experimental")]
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+#[derive(Clone)]
pub struct OnnxConfig {
- pub model_path: std::path::PathBuf,
- pub tokenizer_path: std::path::PathBuf,
- /// Content fingerprints are part of the shared-session cache key and are
- /// rechecked immediately before load, preventing same-path asset swaps.
+ /// Immutable byte snapshots are shared by identity reporting and lazy
+ /// loading. A later same-path replacement cannot change the loaded space.
+ pub model_bytes: std::sync::Arc<[u8]>,
+ pub tokenizer_bytes: std::sync::Arc<[u8]>,
pub model_sha256: String,
pub tokenizer_sha256: String,
pub intra_threads: usize,
@@ -300,6 +300,18 @@ pub struct OnnxConfig {
pub max_inflight_input_bytes: usize,
}
+#[cfg(feature = "onnx-experimental")]
+impl std::fmt::Debug for OnnxConfig {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("OnnxConfig")
+ .field("model_sha256", &self.model_sha256)
+ .field("tokenizer_sha256", &self.tokenizer_sha256)
+ .field("intra_threads", &self.intra_threads)
+ .field("session_pool_size", &self.session_pool_size)
+ .finish_non_exhaustive()
+ }
+}
+
#[cfg(feature = "onnx-experimental")]
#[derive(Debug, thiserror::Error)]
#[error("{reason}")]
@@ -313,17 +325,52 @@ pub struct OnnxHandle {
shared: std::sync::Arc<OnnxShared>,
}
+#[cfg(feature = "onnx-experimental")]
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+struct OnnxCacheKey {
+ model_sha256: String,
+ tokenizer_sha256: String,
+ intra_threads: usize,
+ session_pool_size: usize,
+ microbatch: crate::onnx::MicrobatchConfig,
+ max_inflight_calls: usize,
+ max_input_bytes_per_call: usize,
+ max_inflight_input_bytes: usize,
+}
+
+#[cfg(feature = "onnx-experimental")]
+impl From<&OnnxConfig> for OnnxCacheKey {
+ fn from(cfg: &OnnxConfig) -> Self {
+ use sha2::{Digest, Sha256};
+
+ Self {
+ // OnnxConfig is public, so its descriptive hash fields are not a
+ // construction invariant. Session sharing must be keyed from the
+ // bytes the runtime will actually load.
+ model_sha256: format!("{:x}", Sha256::digest(&cfg.model_bytes)),
+ tokenizer_sha256: format!("{:x}", Sha256::digest(&cfg.tokenizer_bytes)),
+ intra_threads: cfg.intra_threads,
+ session_pool_size: cfg.session_pool_size,
+ microbatch: cfg.microbatch,
+ max_inflight_calls: cfg.max_inflight_calls,
+ max_input_bytes_per_call: cfg.max_input_bytes_per_call,
+ max_inflight_input_bytes: cfg.max_inflight_input_bytes,
+ }
+ }
+}
+
#[cfg(feature = "onnx-experimental")]
impl OnnxHandle {
fn new(cfg: OnnxConfig) -> Self {
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock, Weak};
- static CELLS: OnceLock<Mutex<HashMap<OnnxConfig, Weak<OnnxShared>>>> = OnceLock::new();
+ static CELLS: OnceLock<Mutex<HashMap<OnnxCacheKey, Weak<OnnxShared>>>> = OnceLock::new();
+ let key = OnnxCacheKey::from(&cfg);
let mut cells = CELLS
.get_or_init(|| Mutex::new(HashMap::new()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
- if let Some(shared) = cells.get(&cfg).and_then(Weak::upgrade) {
+ if let Some(shared) = cells.get(&key).and_then(Weak::upgrade) {
return Self { cfg, shared };
}
cells.retain(|_, shared| shared.strong_count() > 0);
@@ -334,7 +381,7 @@ impl OnnxHandle {
cfg.max_inflight_input_bytes.max(1),
)),
});
- cells.insert(cfg.clone(), std::sync::Arc::downgrade(&shared));
+ cells.insert(key, std::sync::Arc::downgrade(&shared));
Self { cfg, shared }
}
@@ -343,30 +390,9 @@ impl OnnxHandle {
self.shared
.init
.get_or_spawn("xerj-onnx-init", move || {
- let model_bytes = std::fs::read(&cfg.model_path).map_err(|e| {
- anyhow!("read ONNX model {}: {e}", cfg.model_path.display())
- })?;
- let tokenizer_bytes = std::fs::read(&cfg.tokenizer_path).map_err(|e| {
- anyhow!("read ONNX tokenizer {}: {e}", cfg.tokenizer_path.display())
- })?;
- let actual_model = sha256_bytes(&model_bytes);
- let actual_tokenizer = sha256_bytes(&tokenizer_bytes);
- if actual_model != cfg.model_sha256 || actual_tokenizer != cfg.tokenizer_sha256
- {
- return Err(anyhow!(
- "ONNX assets changed after configuration; refusing to load a \
- different vector space from the same path (model expected {}, \
- actual {}; tokenizer expected {}, actual {}). Restart with the \
- intended assets or reindex under a new prefix",
- cfg.model_sha256,
- actual_model,
- cfg.tokenizer_sha256,
- actual_tokenizer
- ));
- }
let embedder = crate::onnx::OnnxPool::load_bytes(
- &model_bytes,
- &tokenizer_bytes,
+ &cfg.model_bytes,
+ &cfg.tokenizer_bytes,
cfg.intra_threads,
cfg.session_pool_size,
)?;
@@ -474,26 +500,17 @@ impl OnnxHandle {
}
}
-#[cfg(feature = "onnx-experimental")]
-fn sha256_bytes(bytes: &[u8]) -> String {
- use sha2::{Digest, Sha256};
- let mut hasher = Sha256::new();
- hasher.update(bytes);
- format!("{:x}", hasher.finalize())
-}
-
#[cfg(all(test, feature = "onnx-experimental"))]
mod onnx_handle_tests {
use super::{CancellationSafeInit, OnnxConfig, OnnxHandle};
use crate::onnx::MicrobatchConfig;
- use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
fn cfg(model_sha256: &str) -> OnnxConfig {
OnnxConfig {
- model_path: PathBuf::from("/tmp/model.onnx"),
- tokenizer_path: PathBuf::from("/tmp/tokenizer.json"),
+ model_bytes: Arc::from(format!("model bytes for {model_sha256}").into_bytes()),
+ tokenizer_bytes: Arc::from(b"tokenizer bytes".as_slice()),
model_sha256: model_sha256.into(),
tokenizer_sha256: "tokenizer-hash".into(),
intra_threads: 4,
@@ -506,12 +523,48 @@ mod onnx_handle_tests {
}
#[test]
- fn same_paths_with_different_content_hashes_never_share_session_cell() {
- let first = OnnxHandle::new(cfg("model-a"));
- let second = OnnxHandle::new(cfg("model-b"));
+ fn different_actual_bytes_never_share_session_cell() {
+ let first = cfg("model-a");
+ let mut second = cfg("model-b");
+ second.model_bytes = Arc::from(b"different model bytes".as_slice());
+ let first = OnnxHandle::new(first);
+ let second = OnnxHandle::new(second);
assert!(!Arc::ptr_eq(&first.shared, &second.shared));
}
+ #[test]
+ fn identical_bytes_share_even_when_descriptive_hash_fields_differ() {
+ let first_cfg = cfg("descriptive-a");
+ let mut second_cfg = cfg("descriptive-b");
+ second_cfg.model_bytes = first_cfg.model_bytes.clone();
+ second_cfg.tokenizer_bytes = first_cfg.tokenizer_bytes.clone();
+ let first = OnnxHandle::new(first_cfg);
+ let second = OnnxHandle::new(second_cfg);
+ assert!(Arc::ptr_eq(&first.shared, &second.shared));
+ }
+
+ #[test]
+ fn forged_equal_hash_fields_cannot_share_sessions_for_different_bytes() {
+ let first = cfg("forged");
+ let mut second = cfg("forged");
+ second.model_bytes = Arc::from(b"different model bytes".as_slice());
+ second.tokenizer_bytes = Arc::from(b"different tokenizer bytes".as_slice());
+ let first = OnnxHandle::new(first);
+ let second = OnnxHandle::new(second);
+ assert!(!Arc::ptr_eq(&first.shared, &second.shared));
+ }
+
+ #[test]
+ fn session_registry_key_does_not_retain_asset_bytes_after_handle_drop() {
+ let config = cfg("releasable-assets");
+ let weak_model = Arc::downgrade(&config.model_bytes);
+ let weak_tokenizer = Arc::downgrade(&config.tokenizer_bytes);
+ let handle = OnnxHandle::new(config);
+ drop(handle);
+ assert!(weak_model.upgrade().is_none());
+ assert!(weak_tokenizer.upgrade().is_none());
+ }
+
#[test]
fn global_admission_caps_calls_and_releases_permits() {
let handle = OnnxHandle::new(cfg("admission-calls"));
diff --git a/engine/crates/xerj-ai/src/local.rs b/engine/crates/xerj-ai/src/local.rs
index 9ac1836a9..7dbf02a0f 100644
--- a/engine/crates/xerj-ai/src/local.rs
+++ b/engine/crates/xerj-ai/src/local.rs
@@ -122,6 +122,69 @@ mod tests {
assert_eq!(a, b, "embedding must be deterministic");
}
+ /// Deliberately independent of [`fnv1a64`] so that changing the embedder's
+ /// hash cannot silently move the checksum with it.
+ fn djb2_over_bits(v: &[f32]) -> u64 {
+ let mut h: u64 = 5381;
+ for x in v {
+ for byte in x.to_le_bytes() {
+ h = h.wrapping_mul(33) ^ byte as u64;
+ }
+ }
+ h
+ }
+
+ /// GOLDEN VECTOR — do not "fix" this by re-recording the constants.
+ ///
+ /// `lexical` is the default backend and the one most deployments run, and
+ /// `GET /v1/embedding/identity` reports it as `resumable: true`, which
+ /// tells `xerj autoindex` it may resume an existing semantic run straight
+ /// across a restart. That identity is derived from the literal string
+ /// `lexical-feature-hash.v1;dimensions=384`, so nothing in it is computed
+ /// from this function: changing the tokenizer, the trigram padding, the
+ /// feature weights, the hash seed, or the normalisation would move every
+ /// vector into a new space while `identity_sha256` stayed byte-identical,
+ /// and autoindex would resume across the change and mix the two spaces —
+ /// exactly the failure the identity endpoint exists to prevent.
+ ///
+ /// If this test fails, the lexical vector space changed. The fix is to
+ /// bump `lexical-feature-hash.v1` to `.v2` in
+ /// `xerj-engine/src/index.rs::embedding_execution_identity` (which changes
+ /// `identity_sha256` and so makes resumes across the change fail closed),
+ /// and only then re-record the constants below.
+ #[test]
+ fn lexical_vector_space_is_frozen() {
+ let v = local_embed(
+ "quarterly revenue increased in the london office",
+ DEFAULT_DIMS,
+ );
+ assert_eq!(v.len(), DEFAULT_DIMS);
+
+ // Whole-vector fingerprint: catches any change to any bucket.
+ assert_eq!(
+ djb2_over_bits(&v),
+ 14262738046721457820,
+ "the lexical vector space changed; see this test's doc comment"
+ );
+
+ // A few exact buckets, so a failure says *what* moved rather than just
+ // that a checksum differs. Bit-exact: these are IEEE-754 f32 results
+ // of a fixed sequence of adds and one divide, so they are reproducible
+ // across platforms.
+ for (index, expected) in [
+ (0usize, 0.09425097f32),
+ (43, 0.09425097),
+ (120, -0.09425097),
+ (123, 0.26928848),
+ ] {
+ assert_eq!(
+ v[index], expected,
+ "bucket {index} moved: {} != {expected}",
+ v[index]
+ );
+ }
+ }
+
#[test]
fn l2_normalised() {
let v = local_embed("hello world of vectors", DEFAULT_DIMS);
diff --git a/engine/crates/xerj-api/Cargo.toml b/engine/crates/xerj-api/Cargo.toml
index 42176a3d1..3d06c3ea3 100644
--- a/engine/crates/xerj-api/Cargo.toml
+++ b/engine/crates/xerj-api/Cargo.toml
@@ -5,6 +5,9 @@ edition.workspace = true
license.workspace = true
authors.workspace = true
+[features]
+onnx-experimental = ["xerj-engine/onnx-experimental"]
+
[dependencies]
xerj-common = { path = "../xerj-common" }
xerj-engine = { path = "../xerj-engine" }
diff --git a/engine/crates/xerj-api/src/authz.rs b/engine/crates/xerj-api/src/authz.rs
index a43fdd5f3..b6a3df603 100644
--- a/engine/crates/xerj-api/src/authz.rs
+++ b/engine/crates/xerj-api/src/authz.rs
@@ -2417,6 +2417,10 @@ mod tests {
assert!(check(p, Method::GET, "/_mapping", ""), "mapping");
assert!(check(p, Method::GET, "/_cat/indices", ""), "cat");
assert!(check(p, Method::GET, "/v1/health", ""), "native health");
+ assert!(
+ check(p, Method::GET, "/v1/embedding/identity", ""),
+ "embedding identity"
+ );
assert!(
check(p, Method::GET, "/_resolve/index/logs-*", ""),
"resolve"
diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rs
index 5306e3106..824dc13b3 100644
--- a/engine/crates/xerj-api/src/native.rs
+++ b/engine/crates/xerj-api/src/native.rs
@@ -655,6 +655,47 @@ pub async fn health(State(state): State<AppState>) -> impl IntoResponse {
Json(resp).into_response()
}
+/// Machine-readable, privacy-safe embedding identity for resumable ingestion.
+pub async fn embedding_execution_identity(State(state): State<AppState>) -> impl IntoResponse {
+ let started = Instant::now();
+ let request_id = Uuid::new_v4().to_string();
+ let engine = state.engine.clone();
+ let result = tokio::task::spawn_blocking(move || engine.embedding_execution_identity()).await;
+ match result {
+ Ok(Ok(identity)) => Json(NativeResponse::new(
+ identity,
+ started.elapsed().as_millis() as u64,
+ &request_id,
+ ))
+ .into_response(),
+ Ok(Err(error)) => {
+ tracing::warn!(error = %error, "embedding execution identity is unavailable");
+ native_error(
+ xerj_common::XerjError::embedding(
+ "embedding execution identity is unavailable; verify the configured embedding \
+ backend and local assets in the server logs. Asset-read failures are cached \
+ for this server lifetime; restart the server after fixing the assets",
+ ),
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response()
+ }
+ Err(error) => {
+ tracing::error!(error = %error, "embedding identity worker failed");
+ native_error(
+ xerj_common::XerjError::embedding(
+ "embedding execution identity worker failed; inspect the server logs and \
+ restart the server before retrying",
+ ),
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response()
+ }
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// k8s probes — v0.8 8-P4
//
diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rs
index 68f523f81..a91c4b7bf 100644
--- a/engine/crates/xerj-api/src/router.rs
+++ b/engine/crates/xerj-api/src/router.rs
@@ -104,6 +104,10 @@ pub fn build_native_router(state: AppState) -> Router {
.route("/v1/health", get(native::health))
.route("/v1/health/ready", get(native::readiness))
.route("/v1/cluster/health", get(native::cluster_health))
+ .route(
+ "/v1/embedding/identity",
+ get(native::embedding_execution_identity),
+ )
// k8s probes — v0.8 8-P4 — see comment in `native.rs::liveness`.
.route("/health/live", get(native::liveness))
.route("/health/ready", get(native::readiness))
@@ -225,6 +229,13 @@ pub fn build_es_compat_router(state: AppState) -> Router {
// firewalled off) can still scrape metrics. Same handler as the native
// router; gated by the optional read-only `auth.metrics_token`.
.route("/v1/metrics", get(native::metrics))
+ // Autoindex defaults to the ES-compatible listener, so the native
+ // embedding identity contract must be available here as well as on
+ // the dedicated native listener.
+ .route(
+ "/v1/embedding/identity",
+ get(native::embedding_execution_identity),
+ )
// ── Cluster-level ──────────────────────────────────────────────────
.route("/", get(es_compat::es_info))
.route("/_cluster/health", get(es_compat::cluster_health))
@@ -1045,11 +1056,19 @@ mod tests {
}
fn test_state(distribution: &str) -> AppState {
+ test_state_with(distribution, |_| {})
+ }
+
+ fn test_state_with(
+ distribution: &str,
+ configure: impl FnOnce(&mut xerj_common::config::Config),
+ ) -> AppState {
let dir = tempfile::tempdir().expect("tempdir").keep();
let mut config = xerj_common::config::Config::default();
config.server.data_dir = dir.to_string_lossy().into_owned();
config.storage.wal_sync = xerj_common::config::WalSync::Async;
config.compat.distribution = distribution.to_string();
+ configure(&mut config);
let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
AppState::new(config, engine, metrics)
@@ -1106,4 +1125,82 @@ mod tests {
Some("Elasticsearch")
);
}
+
+ #[tokio::test]
+ async fn embedding_identity_is_sanitized_machine_readable_native_data() {
+ for app in [
+ build_native_router(test_state("elasticsearch")),
+ build_es_compat_router(test_state("elasticsearch")),
+ ] {
+ let response = app
+ .oneshot(
+ Request::builder()
+ .uri("/v1/embedding/identity")
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.status(), axum::http::StatusCode::OK);
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
+ let data = value["data"].as_object().unwrap();
+ assert_eq!(data["backend"], "lexical");
+ assert_eq!(data["dimensions"], 384);
+ assert_eq!(data["resumable"], true);
+ assert_eq!(data["identity_sha256"].as_str().unwrap().len(), 64);
+ let encoded = String::from_utf8(bytes.to_vec()).unwrap();
+ for secret_field in ["api_key", "endpoint", "model_path", "tokenizer_path"] {
+ assert!(!encoded.contains(secret_field), "{secret_field} leaked");
+ }
+ }
+ }
+
+ #[cfg(feature = "onnx-experimental")]
+ #[tokio::test]
+ async fn embedding_identity_asset_failures_do_not_leak_local_paths() {
+ const MODEL: &str = "/private/customer-a/models/missing-model.onnx";
+ const TOKENIZER: &str = "/private/customer-a/models/missing-tokenizer.json";
+ for app in [
+ build_native_router(test_state_with("elasticsearch", |config| {
+ config.embedding.mode = "onnx-experimental".into();
+ config.embedding.onnx_model_path = MODEL.into();
+ config.embedding.onnx_tokenizer_path = TOKENIZER.into();
+ })),
+ build_es_compat_router(test_state_with("elasticsearch", |config| {
+ config.embedding.mode = "onnx-experimental".into();
+ config.embedding.onnx_model_path = MODEL.into();
+ config.embedding.onnx_tokenizer_path = TOKENIZER.into();
+ })),
+ ] {
+ let response = app
+ .oneshot(
+ Request::builder()
+ .uri("/v1/embedding/identity")
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert!(!response.status().is_success());
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let encoded = String::from_utf8(bytes.to_vec()).unwrap();
+ assert!(encoded.contains("verify the configured embedding backend"));
+ assert!(encoded.contains("restart the server after fixing the assets"));
+ for private in [
+ MODEL,
+ TOKENIZER,
+ "missing-model.onnx",
+ "missing-tokenizer.json",
+ "No such file",
+ "os error",
+ ] {
+ assert!(!encoded.contains(private), "{private} leaked: {encoded}");
+ }
+ }
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index a71d83f18..43ad3daa9 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -2,6 +2,7 @@
//! backoff on 429/5xx/transport errors; parses per-item bulk errors.
use anyhow::{anyhow, Context, Result};
+use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::time::Duration;
@@ -24,6 +25,21 @@ pub struct BulkOutcome {
pub first_server_error: Option<String>,
}
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct EmbeddingExecutionIdentity {
+ pub version: u32,
+ pub backend: String,
+ pub identity_sha256: String,
+ /// Absent for backends whose vector width the server does not pin
+ /// (`neural`, `proxy`). An absent width must not be read as 384.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub dimensions: Option<usize>,
+ pub semantic_contract: String,
+ pub resumable: bool,
+ #[serde(default)]
+ pub non_resumable_reason: Option<String>,
+}
+
/// Ensure an autoindex index-create body pins the index to a single WAL shard.
///
/// autoindex creates one index per inferred dataset — hundreds for a large
@@ -106,6 +122,52 @@ impl Es {
Ok(resp.json().unwrap_or(Value::Null))
}
+ pub fn embedding_execution_identity(&self) -> Result<EmbeddingExecutionIdentity> {
+ let response = self
+ .req(reqwest::Method::GET, "/v1/embedding/identity")
+ .send()
+ .context("GET /v1/embedding/identity")?;
+ let status = response.status();
+ if !status.is_success() {
+ anyhow::bail!(
+ "GET /v1/embedding/identity failed: HTTP {status}; semantic autoindex requires \
+ a XERJ server that exposes a resumable embedding identity"
+ );
+ }
+ let value: Value = response
+ .json()
+ .context("parse embedding identity response")?;
+ let identity: EmbeddingExecutionIdentity = serde_json::from_value(
+ value
+ .get("data")
+ .cloned()
+ .ok_or_else(|| anyhow!("embedding identity response has no data object"))?,
+ )
+ .context("parse embedding identity")?;
+ if identity.version != 1
+ || identity.identity_sha256.len() != 64
+ || !identity
+ .identity_sha256
+ .bytes()
+ .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
+ || identity.dimensions == Some(0)
+ // `lexical` and `onnx-experimental` are the backends whose width
+ // the server does pin, so an absent width from one of them means
+ // the response did not come from a server that pins it. Only the
+ // explicitly unpinned backends may omit it.
+ || (matches!(identity.backend.as_str(), "lexical" | "onnx-experimental")
+ && identity.dimensions.is_none())
+ || identity.semantic_contract != "semantic_text-derived-vector.v1"
+ || !matches!(
+ identity.backend.as_str(),
+ "lexical" | "neural" | "proxy" | "onnx-experimental"
+ )
+ {
+ anyhow::bail!("server returned an unsupported embedding execution identity");
+ }
+ Ok(identity)
+ }
+
/// GET an arbitrary path and return only the HTTP status code.
///
/// One attempt, no retry/backoff: `xerj brain` uses this to probe
@@ -485,6 +547,75 @@ mod tests {
stream.write_all(body).unwrap();
}
+ #[test]
+ fn embedding_identity_uses_native_endpoint_and_parses_sanitized_data() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let server = std::thread::spawn(move || {
+ let (mut stream, _) = listener.accept().unwrap();
+ let request = read_request(&mut stream);
+ assert!(
+ String::from_utf8_lossy(&request)
+ .starts_with("GET /v1/embedding/identity HTTP/1.1"),
+ "{}",
+ String::from_utf8_lossy(&request)
+ );
+ respond_json(
+ &mut stream,
+ br#"{"data":{"version":1,"backend":"lexical","identity_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","dimensions":384,"semantic_contract":"semantic_text-derived-vector.v1","resumable":true},"took_ms":0,"request_id":"test"}"#,
+ );
+ });
+ let es = Es::new(&format!("http://{address}"), None).unwrap();
+ let identity = es.embedding_execution_identity().unwrap();
+ assert_eq!(identity.backend, "lexical");
+ assert!(identity.resumable);
+ assert_eq!(identity.dimensions, Some(384));
+ server.join().unwrap();
+ }
+
+ /// `neural` and `proxy` omit the width entirely, so the client must accept
+ /// a response without a `dimensions` key rather than failing to parse it.
+ /// It still rejects an explicit zero, which no backend should ever send,
+ /// and it rejects an omitted width from `lexical`/`onnx-experimental`,
+ /// whose widths a real XERJ server always reports.
+ #[test]
+ fn embedding_identity_accepts_an_omitted_width_and_rejects_a_zero_one() {
+ for (body, expect_ok) in [
+ (
+ br#"{"data":{"version":1,"backend":"proxy","identity_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","semantic_contract":"semantic_text-derived-vector.v1","resumable":false,"non_resumable_reason":"remote"},"took_ms":0,"request_id":"test"}"#.to_vec(),
+ true,
+ ),
+ (
+ br#"{"data":{"version":1,"backend":"lexical","identity_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","dimensions":0,"semantic_contract":"semantic_text-derived-vector.v1","resumable":true},"took_ms":0,"request_id":"test"}"#.to_vec(),
+ false,
+ ),
+ (
+ br#"{"data":{"version":1,"backend":"lexical","identity_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","semantic_contract":"semantic_text-derived-vector.v1","resumable":true},"took_ms":0,"request_id":"test"}"#.to_vec(),
+ false,
+ ),
+ (
+ br#"{"data":{"version":1,"backend":"onnx-experimental","identity_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","semantic_contract":"semantic_text-derived-vector.v1","resumable":true},"took_ms":0,"request_id":"test"}"#.to_vec(),
+ false,
+ ),
+ ] {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let server = std::thread::spawn(move || {
+ let (mut stream, _) = listener.accept().unwrap();
+ let _ = read_request(&mut stream);
+ respond_json(&mut stream, &body);
+ });
+ let es = Es::new(&format!("http://{address}"), None).unwrap();
+ let result = es.embedding_execution_identity();
+ assert_eq!(result.is_ok(), expect_ok, "{result:?}");
+ if let Ok(identity) = result {
+ assert_eq!(identity.dimensions, None);
+ assert!(!identity.resumable);
+ }
+ server.join().unwrap();
+ }
+ }
+
#[test]
fn retry_path_recovers_after_one_bulk_timeout() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index c456e037b..c7d18289f 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -24,6 +24,7 @@ struct MockState {
failed_once: bool,
delete_calls: usize,
stop: bool,
+ embedding_identity_sha256: String,
}
struct MockEndpoint {
@@ -39,6 +40,7 @@ impl MockEndpoint {
let url = format!("http://{}", listener.local_addr().unwrap());
let state = Arc::new(Mutex::new(MockState {
fail_data_bulk,
+ embedding_identity_sha256: "a".repeat(64),
..MockState::default()
}));
let server_state = Arc::clone(&state);
@@ -99,7 +101,17 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {
let method = parts.next().unwrap_or("");
let path = parts.next().unwrap_or("");
- let response = if method == "POST" && path == "/_bulk" {
+ let response = if method == "GET" && path == "/v1/embedding/identity" {
+ let identity = state.lock().unwrap().embedding_identity_sha256.clone();
+ json!({"data": {
+ "version": 1,
+ "backend": "lexical",
+ "identity_sha256": identity,
+ "dimensions": 384,
+ "semantic_contract": "semantic_text-derived-vector.v1",
+ "resumable": true
+ }, "took_ms": 0, "request_id": "test"})
+ } else if method == "POST" && path == "/_bulk" {
bulk_response(&body, state)
} else if method == "POST" && path.contains("/_delete_by_query") {
let query: Value = serde_json::from_slice(&body).unwrap();
@@ -225,6 +237,35 @@ fn event_count(state_dir: &Path, kind: &str) -> usize {
.count()
}
+#[test]
+fn semantic_resume_rejects_embedding_identity_drift_before_another_bulk() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("report.txt"),
+ "Quarterly operating income improved materially after stronger subscription renewals. \
+ Management expects durable demand across the next reporting period.",
+ )
+ .unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ config.no_semantic = false;
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ assert_eq!(event_count(state_dir.path(), "embedding_identity"), 1);
+ let bulks_before = endpoint.state.lock().unwrap().data_bulk_number;
+ endpoint.state.lock().unwrap().embedding_identity_sha256 = "b".repeat(64);
+ let error = run_index(config).unwrap_err().to_string();
+ assert!(error.contains("refusing to mix vector spaces"), "{error}");
+ assert!(error.contains("--fresh"), "{error}");
+ assert_eq!(
+ endpoint.state.lock().unwrap().data_bulk_number,
+ bulks_before,
+ "identity drift must fail before another data bulk"
+ );
+}
+
#[test]
fn fresh_publication_skips_delete_and_noop_resume_does_not_append_plan() {
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 64b803afa..cf2e089b4 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -765,6 +765,21 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
return Ok((0, None));
}
+ if plan
+ .datasets
+ .iter()
+ .any(|dataset| dataset.semantic_field.is_some())
+ {
+ let identity = es
+ .embedding_execution_identity()
+ .context("semantic autoindex could not pin the server embedding execution identity")?;
+ journal.pin_embedding_identity(
+ &identity.identity_sha256,
+ identity.resumable,
+ identity.non_resumable_reason.as_deref(),
+ )?;
+ }
+
// ── create indices with explicit mappings ────────────────────────────
for d in &plan.datasets {
es.ensure_index(&d.index, &build_mapping(&d.specs))
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index da59f5eb7..5df8fdd60 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -79,6 +79,105 @@ mod tests {
assert!(text.contains("\"kind\":\"resume\""));
assert!(text.contains("\"bulk_timeout_secs\":900"));
}
+
+ #[test]
+ fn semantic_identity_is_durable_and_drift_fails_closed() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut first = Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap();
+ first
+ .pin_embedding_identity(&"a".repeat(64), true, None)
+ .unwrap();
+ drop(first);
+
+ let mut resumed = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ resumed
+ .pin_embedding_identity(&"a".repeat(64), true, None)
+ .unwrap();
+ let error = resumed
+ .pin_embedding_identity(&"b".repeat(64), true, None)
+ .unwrap_err()
+ .to_string();
+ assert!(error.contains("refusing to mix vector spaces"), "{error}");
+ assert!(error.contains("Restore the original"), "{error}");
+ assert!(error.contains("--fresh"), "{error}");
+ assert!(error.contains("--prefix"), "{error}");
+ assert!(error.contains("delete and recreate"), "{error}");
+ }
+
+ #[test]
+ fn unpinned_backend_allows_fresh_run_but_not_resume() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut journal = Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap();
+ journal
+ .pin_embedding_identity(&"a".repeat(64), false, Some("remote alias can drift"))
+ .unwrap();
+ drop(journal);
+ let mut resumed = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ let error = resumed
+ .pin_embedding_identity(&"a".repeat(64), false, Some("remote alias can drift"))
+ .unwrap_err()
+ .to_string();
+ assert!(error.contains("remote alias can drift"), "{error}");
+ assert!(error.contains("--fresh"), "{error}");
+ assert!(error.contains("--prefix"), "{error}");
+ assert!(error.contains("delete and recreate"), "{error}");
+ }
+
+ #[test]
+ fn completed_legacy_semantic_journal_requires_fresh_rebuild() {
+ let dir = tempfile::tempdir().unwrap();
+ std::fs::write(
+ dir.path().join("journal.ndjson"),
+ concat!(
+ "{\"v\":1,\"kind\":\"run\",\"root\":\"root\",\"url\":\"url\",",
+ "\"prefix\":\"prefix\",\"run_id\":\"legacy\"}\n",
+ "{\"kind\":\"file_done\",\"file_key\":\"f\",\"path\":\"report.txt\",",
+ "\"records\":1,\"junk\":0,\"bytes\":10}\n"
+ ),
+ )
+ .unwrap();
+ let mut journal = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ let error = journal
+ .pin_embedding_identity(&"a".repeat(64), true, None)
+ .unwrap_err()
+ .to_string();
+ assert!(
+ error.contains("predates embedding identity pinning"),
+ "{error}"
+ );
+ assert!(error.contains("--fresh"), "{error}");
+ assert!(error.contains("--prefix"), "{error}");
+ assert!(error.contains("delete and recreate"), "{error}");
+ }
+
+ #[test]
+ fn malformed_or_conflicting_identity_records_fail_during_replay() {
+ for tail in [
+ "{\"v\":1,\"kind\":\"embedding_identity\",\"identity_sha256\":\"bad\",\"resumable\":true}\n",
+ concat!(
+ "{\"v\":1,\"kind\":\"embedding_identity\",\"identity_sha256\":\"",
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "\",\"resumable\":true}\n",
+ "{\"v\":1,\"kind\":\"embedding_identity\",\"identity_sha256\":\"",
+ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+ "\",\"resumable\":true}\n"
+ ),
+ ] {
+ let dir = tempfile::tempdir().unwrap();
+ std::fs::write(
+ dir.path().join("journal.ndjson"),
+ format!(
+ "{{\"v\":1,\"kind\":\"run\",\"root\":\"root\",\"url\":\"url\",\
+ \"prefix\":\"prefix\",\"run_id\":\"legacy\"}}\n{tail}"
+ ),
+ )
+ .unwrap();
+ assert!(
+ Journal::open(dir.path(), "root", "url", "prefix", 300, false).is_err(),
+ "{tail}"
+ );
+ }
+ }
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -160,6 +259,8 @@ pub struct Journal {
/// publication can never make resume skip a zero/partial generation.
pub pending_replacements: HashMap<String, String>,
pub plan: Option<Plan>,
+ pub embedding_identity_sha256: Option<String>,
+ pub embedding_identity_resumable: Option<bool>,
}
pub fn default_state_dir(root: &str, url: &str, prefix: &str) -> PathBuf {
@@ -218,6 +319,8 @@ impl Journal {
let mut done = HashMap::new();
let mut pending_replacements: HashMap<String, String> = HashMap::new();
let mut plan = None;
+ let mut embedding_identity_sha256 = None;
+ let mut embedding_identity_resumable = None;
let mut run_id = None;
let mut resumed = false;
if jpath.exists() {
@@ -268,6 +371,49 @@ impl Journal {
}
}
}
+ Some("embedding_identity") => {
+ let digest = v
+ .get("identity_sha256")
+ .and_then(Value::as_str)
+ .filter(|digest| {
+ digest.len() == 64
+ && digest.bytes().all(|byte| {
+ byte.is_ascii_hexdigit()
+ && !byte.is_ascii_uppercase()
+ })
+ });
+ let resumable = v.get("resumable").and_then(Value::as_bool);
+ if v.get("v").and_then(Value::as_u64) != Some(1)
+ || digest.is_none()
+ || resumable.is_none()
+ || v.as_object().is_none_or(|object| {
+ object.len() != 4
+ || !["v", "kind", "identity_sha256", "resumable"]
+ .iter()
+ .all(|key| object.contains_key(*key))
+ })
+ {
+ anyhow::bail!(
+ "journal at {} contains a malformed embedding identity; \
+ restore it from backup or re-run with --fresh",
+ jpath.display()
+ );
+ }
+ if embedding_identity_sha256
+ .as_deref()
+ .is_some_and(|existing| existing != digest.unwrap())
+ || embedding_identity_resumable
+ .is_some_and(|existing| existing != resumable.unwrap())
+ {
+ anyhow::bail!(
+ "journal at {} contains conflicting embedding identities; \
+ restore it from backup or re-run with --fresh",
+ jpath.display()
+ );
+ }
+ embedding_identity_sha256 = digest.map(str::to_owned);
+ embedding_identity_resumable = resumable;
+ }
Some("file_done") => {
if let Ok(fd) = serde_json::from_value::<FileDone>(v.clone()) {
let commits_pending = pending_replacements
@@ -345,6 +491,8 @@ impl Journal {
done,
pending_replacements,
plan,
+ embedding_identity_sha256,
+ embedding_identity_resumable,
};
if is_new {
j.append_transaction(
@@ -367,6 +515,61 @@ impl Journal {
Ok(j)
}
+ /// Pin the server-side vector-space identity before any semantic write.
+ pub fn pin_embedding_identity(
+ &mut self,
+ identity_sha256: &str,
+ resumable: bool,
+ non_resumable_reason: Option<&str>,
+ ) -> Result<()> {
+ if identity_sha256.len() != 64
+ || !identity_sha256
+ .bytes()
+ .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
+ {
+ anyhow::bail!("server returned an invalid embedding identity digest");
+ }
+ if self.resumed && (!resumable || self.embedding_identity_resumable == Some(false)) {
+ anyhow::bail!(
+ "server embedding backend cannot safely resume semantic indexing: {}. \
+ Restore the exact original embedding identity, or rebuild all vectors with \
+ --fresh and a new --prefix. Before reusing the old prefix, delete and recreate \
+ its prior autoindex indices",
+ non_resumable_reason.unwrap_or("embedding identity is not immutable")
+ );
+ }
+ if let Some(existing) = &self.embedding_identity_sha256 {
+ if existing != identity_sha256 {
+ anyhow::bail!(
+ "embedding execution identity changed since this autoindex journal was \
+ created; refusing to mix vector spaces. Restore the original embedding \
+ identity, or rebuild all vectors with --fresh and a new --prefix. Before \
+ reusing the old prefix, delete and recreate its prior autoindex indices"
+ );
+ }
+ return Ok(());
+ }
+ if self.resumed && !self.done.is_empty() {
+ anyhow::bail!(
+ "this semantic autoindex journal predates embedding identity pinning and cannot \
+ be resumed safely. Rebuild all vectors with --fresh and a new --prefix. Before \
+ reusing the old prefix, delete and recreate its prior autoindex indices"
+ );
+ }
+ self.append_transaction(
+ &serde_json::json!({
+ "v": 1,
+ "kind": "embedding_identity",
+ "identity_sha256": identity_sha256,
+ "resumable": resumable,
+ }),
+ "embedding_identity",
+ )?;
+ self.embedding_identity_sha256 = Some(identity_sha256.to_owned());
+ self.embedding_identity_resumable = Some(resumable);
+ Ok(())
+ }
+
fn append_transaction(&mut self, v: &Value, what: &str) -> Result<()> {
let mut line = serde_json::to_string(v)?;
line.push('\n');
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index a7466843e..06f8616e7 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -894,6 +894,20 @@ pub struct EmbeddingConfig {
pub onnx_max_input_bytes_per_call: usize,
/// Aggregate UTF-8 bytes admitted globally per shared ONNX model/session.
pub onnx_max_inflight_input_bytes: usize,
+ /// Engine-lifetime immutable ONNX assets. Cloned configurations share this
+ /// cell, so identity reporting and lazy loading consume the same bytes.
+ /// It is runtime state, never configuration input or serialized output.
+ #[serde(skip)]
+ pub runtime_onnx_assets:
+ std::sync::Arc<std::sync::OnceLock<Result<EmbeddingAssetSnapshot, String>>>,
+}
+
+#[derive(Debug, Clone)]
+pub struct EmbeddingAssetSnapshot {
+ pub model_bytes: std::sync::Arc<[u8]>,
+ pub tokenizer_bytes: std::sync::Arc<[u8]>,
+ pub model_sha256: String,
+ pub tokenizer_sha256: String,
}
impl Default for EmbeddingConfig {
@@ -920,6 +934,7 @@ impl Default for EmbeddingConfig {
onnx_max_inflight_calls: 8,
onnx_max_input_bytes_per_call: 8 * 1024 * 1024,
onnx_max_inflight_input_bytes: 32 * 1024 * 1024,
+ runtime_onnx_assets: std::sync::Arc::new(std::sync::OnceLock::new()),
}
}
}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 39eccddc9..36750ea6e 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -15,6 +15,29 @@ use xerj_common::types::{IndexName, Schema};
use crate::index::{Index, IndexStats};
use crate::{EngineError, Result};
+/// Privacy-safe identity of the embedding execution contract exposed to
+/// remote ingestion clients. The digest is opaque: model paths, provider URLs,
+/// credentials, and model names never cross the API boundary.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct EmbeddingExecutionIdentity {
+ pub version: u32,
+ pub backend: String,
+ pub identity_sha256: String,
+ /// Vector width, but only for the backends where the server actually
+ /// pins it: `lexical` always emits [`xerj_ai::local::DEFAULT_DIMS`], and
+ /// `onnx-experimental` is constrained to 384 by `validate_onnx_dimensions`.
+ /// `neural` takes its width from the loaded model's `hidden_size` and
+ /// `proxy` from whatever the remote returns, so neither is known here and
+ /// the field is omitted rather than guessed — a reader must not treat an
+ /// absent width as 384.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub dimensions: Option<usize>,
+ pub semantic_contract: String,
+ pub resumable: bool,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub non_resumable_reason: Option<String>,
+}
+
// ── Clustering types ─────────────────────────────────────────────────────────
/// Node identity and cluster membership configuration.
@@ -366,7 +389,11 @@ impl Engine {
}
/// Create a new engine, opening any existing indices from disk.
- pub fn new(config: Config) -> Result<Self> {
+ pub fn new(mut config: Config) -> Result<Self> {
+ // Runtime asset ownership belongs to this Engine, not to the caller's
+ // clone lineage. This ensures a later Engine re-reads same-path ONNX
+ // assets while all Index config clones within one Engine share bytes.
+ config.embedding.runtime_onnx_assets = Arc::new(std::sync::OnceLock::new());
let data_dir = PathBuf::from(&config.server.data_dir);
std::fs::create_dir_all(&data_dir)?;
@@ -1494,6 +1521,12 @@ impl Engine {
&self.config
}
+ /// Return the effective embedding identity without exposing configuration
+ /// secrets or local asset paths.
+ pub fn embedding_execution_identity(&self) -> Result<EmbeddingExecutionIdentity> {
+ crate::index::embedding_execution_identity(&self.config.embedding)
+ }
+
/// Sum of every open index's live memtable footprint, in bytes. This is
/// the quantity the process-wide memtable budget (item 1) is checked
/// against — per-index back-pressure only ever sees one index's slice.
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index ecf495e38..19eb69ec7 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -23408,53 +23408,34 @@ fn validate_onnx_dimensions(fields: &[FieldConfig]) -> Result<()> {
}
#[cfg(feature = "onnx-experimental")]
-fn sha256_file(path: &Path) -> Result<String> {
+fn configured_onnx_assets(
+ cfg: &xerj_common::config::EmbeddingConfig,
+) -> Result<xerj_common::config::EmbeddingAssetSnapshot> {
use sha2::{Digest, Sha256};
- use std::collections::HashMap;
- use std::io::Read;
- use std::sync::{Mutex, OnceLock};
- use std::time::UNIX_EPOCH;
-
- static CACHE: OnceLock<Mutex<HashMap<(PathBuf, u64, u128), String>>> = OnceLock::new();
- let canonical = path.canonicalize().map_err(|e| {
- EngineError::Common(xerj_common::XerjError::embedding(format!(
- "cannot resolve embedding asset {}: {e}",
- path.display()
- )))
- })?;
- let metadata = std::fs::metadata(&canonical)?;
- let modified = metadata
- .modified()
- .ok()
- .and_then(|time| time.duration_since(UNIX_EPOCH).ok())
- .map(|duration| duration.as_nanos())
- .unwrap_or(0);
- let key = (canonical.clone(), metadata.len(), modified);
- let mut cache = CACHE
- .get_or_init(|| Mutex::new(HashMap::new()))
- .lock()
- .unwrap_or_else(|poisoned| poisoned.into_inner());
- if let Some(hash) = cache.get(&key).cloned() {
- return Ok(hash);
- }
- let mut file = std::fs::File::open(&canonical).map_err(|e| {
- EngineError::Common(xerj_common::XerjError::embedding(format!(
- "cannot fingerprint embedding asset {}: {e}",
- path.display()
- )))
- })?;
- let mut hasher = Sha256::new();
- let mut buf = [0_u8; 1024 * 1024];
- loop {
- let n = file.read(&mut buf)?;
- if n == 0 {
- break;
- }
- hasher.update(&buf[..n]);
- }
- let hash = format!("{:x}", hasher.finalize());
- cache.insert(key, hash.clone());
- Ok(hash)
+
+ cfg.runtime_onnx_assets
+ .get_or_init(|| {
+ let model = Path::new(&cfg.onnx_model_path)
+ .canonicalize()
+ .map_err(|e| format!("cannot resolve embedding model asset: {e}"))?;
+ let tokenizer = Path::new(&cfg.onnx_tokenizer_path)
+ .canonicalize()
+ .map_err(|e| format!("cannot resolve embedding tokenizer asset: {e}"))?;
+ let model_bytes: Arc<[u8]> = std::fs::read(&model)
+ .map(Into::into)
+ .map_err(|e| format!("cannot read embedding model asset: {e}"))?;
+ let tokenizer_bytes: Arc<[u8]> = std::fs::read(&tokenizer)
+ .map(Into::into)
+ .map_err(|e| format!("cannot read embedding tokenizer asset: {e}"))?;
+ Ok(xerj_common::config::EmbeddingAssetSnapshot {
+ model_sha256: format!("{:x}", Sha256::digest(&model_bytes)),
+ tokenizer_sha256: format!("{:x}", Sha256::digest(&tokenizer_bytes)),
+ model_bytes,
+ tokenizer_bytes,
+ })
+ })
+ .clone()
+ .map_err(|reason| EngineError::Common(xerj_common::XerjError::embedding(reason)))
}
fn configured_onnx_identity(
@@ -23469,15 +23450,158 @@ fn configured_onnx_identity(
onnx-experimental feature; no embedding identity marker was written",
)));
#[cfg(feature = "onnx-experimental")]
- Ok(Some(PersistedEmbeddingIdentity {
+ {
+ let assets = configured_onnx_assets(cfg)?;
+ Ok(Some(PersistedEmbeddingIdentity {
+ version: 1,
+ backend: "onnx-experimental".into(),
+ model_sha256: assets.model_sha256,
+ tokenizer_sha256: assets.tokenizer_sha256,
+ dimensions: 384,
+ pooling: "attention-mask-mean+l2-normalize".into(),
+ max_tokens: 512,
+ }))
+ }
+}
+
+pub(crate) fn embedding_execution_identity(
+ cfg: &xerj_common::config::EmbeddingConfig,
+) -> Result<crate::engine::EmbeddingExecutionIdentity> {
+ use sha2::{Digest, Sha256};
+
+ const CONTRACT: &str = "semantic_text-derived-vector.v1";
+ let requested = cfg.mode.trim().to_ascii_lowercase();
+ let mut backend = match requested.as_str() {
+ "neural" => {
+ #[cfg(feature = "neural")]
+ {
+ "neural"
+ }
+ #[cfg(not(feature = "neural"))]
+ {
+ "lexical"
+ }
+ }
+ "onnx-experimental" => "onnx-experimental",
+ "lexical" => "lexical",
+ "proxy" if cfg.default_endpoint.is_empty() => "lexical",
+ "proxy" => "proxy",
+ _ if cfg.default_endpoint.is_empty() => "lexical",
+ _ => "proxy",
+ };
+ if backend == "proxy" && !proxy_initializes(cfg) {
+ backend = "lexical";
+ }
+ // Only report a width for the backends whose width this server actually
+ // pins. `neural` reads it from the loaded model's `hidden_size` and
+ // `proxy` gets whatever the remote returns, so both are omitted rather
+ // than reported as 384.
+ let (material, resumable, reason, dimensions) = match backend {
+ "lexical" => (
+ format!(
+ "lexical-feature-hash.v1;dimensions={};algorithm-probe={}",
+ xerj_ai::local::DEFAULT_DIMS,
+ lexical_algorithm_probe_sha256()
+ ),
+ true,
+ None,
+ Some(xerj_ai::local::DEFAULT_DIMS),
+ ),
+ "onnx-experimental" => {
+ let identity = configured_onnx_identity(cfg)?.ok_or_else(|| {
+ EngineError::Common(xerj_common::XerjError::embedding(
+ "ONNX embedding identity is unavailable",
+ ))
+ })?;
+ let onnx_dimensions = identity.dimensions;
+ (
+ format!(
+ "onnx-experimental.v1;model={};tokenizer={};dimensions={};pooling={};max_tokens={}",
+ identity.model_sha256,
+ identity.tokenizer_sha256,
+ identity.dimensions,
+ identity.pooling,
+ identity.max_tokens
+ ),
+ true,
+ None,
+ Some(onnx_dimensions),
+ )
+ }
+ "neural" => (
+ format!("neural-unpinned.v1;configured-model={}", cfg.neural_model),
+ false,
+ Some(
+ "the neural model is not content-addressed; use a fresh autoindex state after changing it"
+ .to_string(),
+ ),
+ None,
+ ),
+ "proxy" => (
+ format!("proxy-unpinned.v1;configured-model={}", cfg.default_model),
+ false,
+ Some(
+ "the remote provider does not attest immutable model assets; semantic resume is unsafe"
+ .to_string(),
+ ),
+ None,
+ ),
+ _ => unreachable!(),
+ };
+ let canonical = format!("{CONTRACT};backend={backend};{material}");
+ Ok(crate::engine::EmbeddingExecutionIdentity {
version: 1,
- backend: "onnx-experimental".into(),
- model_sha256: sha256_file(Path::new(&cfg.onnx_model_path))?,
- tokenizer_sha256: sha256_file(Path::new(&cfg.onnx_tokenizer_path))?,
- dimensions: 384,
- pooling: "attention-mask-mean+l2-normalize".into(),
- max_tokens: 512,
- }))
+ backend: backend.to_string(),
+ identity_sha256: format!("{:x}", Sha256::digest(canonical.as_bytes())),
+ dimensions,
+ semantic_contract: CONTRACT.to_string(),
+ resumable,
+ non_resumable_reason: reason,
+ })
+}
+
+/// Fingerprint representative output from the zero-config embedder.
+///
+/// The lexical backend has no model asset to hash, so its identity must be
+/// bound to executable behaviour instead. Encoding each `f32` as little-endian
+/// bytes makes the material stable across host endianness. If tokenisation,
+/// feature weights, hashing, normalisation, or the default width changes, this
+/// digest (and therefore the resume identity) changes automatically.
+fn lexical_algorithm_probe_sha256() -> String {
+ use sha2::{Digest, Sha256};
+
+ const PROBES: &[&str] = &[
+ "",
+ "The quick brown fox jumps over the lazy dog.",
+ "Revenue grew 12.5% in Q4 2025; naïve café 東京.",
+ "snake_case/path-like:value + punctuation!",
+ ];
+ let mut digest = Sha256::new();
+ for probe in PROBES {
+ digest.update((probe.len() as u64).to_le_bytes());
+ digest.update(probe.as_bytes());
+ for value in xerj_ai::local::local_embed(probe, xerj_ai::local::DEFAULT_DIMS) {
+ digest.update(value.to_le_bytes());
+ }
+ }
+ format!("{:x}", digest.finalize())
+}
+
+fn proxy_config(
+ cfg: &xerj_common::config::EmbeddingConfig,
+) -> xerj_ai::embed::EmbeddingProxyConfig {
+ xerj_ai::embed::EmbeddingProxyConfig {
+ endpoint: cfg.default_endpoint.clone(),
+ api_key: std::env::var("XERJ_EMBEDDING_API_KEY").ok(),
+ model: cfg.default_model.clone(),
+ timeout_secs: cfg.timeout_ms / 1000,
+ max_concurrent: 4,
+ max_retries: 3,
+ }
+}
+
+fn proxy_initializes(cfg: &xerj_common::config::EmbeddingConfig) -> bool {
+ xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)).is_ok()
}
/// Fail closed rather than mixing vectors produced by different model,
@@ -23604,6 +23728,303 @@ mod embedding_identity_tests {
schema
}
+ #[test]
+ fn lexical_execution_identity_is_stable_and_sanitized() {
+ let cfg = xerj_common::config::EmbeddingConfig::default();
+ let identity = embedding_execution_identity(&cfg).unwrap();
+ assert_eq!(identity.backend, "lexical");
+ assert!(identity.resumable);
+ // `Embedder::Lexical` always embeds at DEFAULT_DIMS, so this width is
+ // the one the server will really produce.
+ assert_eq!(identity.dimensions, Some(xerj_ai::local::DEFAULT_DIMS));
+ // The lexical identity has no model asset behind it, so it is bound to
+ // the embedder's actual output instead. If this digest moves, every
+ // lexical vector moved with it and the resume identity below must move
+ // too — see `lexical_algorithm_probe_sha256`.
+ assert_eq!(
+ lexical_algorithm_probe_sha256(),
+ "e6240f3d323831d621fd69a62b39ad70b7efba72e58e949362fdb7f5d53eea53"
+ );
+ assert_eq!(
+ identity.identity_sha256,
+ "58a7576ccc1497ee471418bb6e8918c7e5a837161792992e1e213a2dcb2ebdbe"
+ );
+ let encoded = serde_json::to_string(&identity).unwrap();
+ assert!(!encoded.contains("endpoint"));
+ assert!(!encoded.contains("path"));
+ }
+
+ #[test]
+ fn configured_backends_report_the_effective_runtime_backend() {
+ let neural = xerj_common::config::EmbeddingConfig {
+ mode: "neural".into(),
+ ..Default::default()
+ };
+ let neural_identity = embedding_execution_identity(&neural).unwrap();
+ #[cfg(feature = "neural")]
+ {
+ assert_eq!(neural_identity.backend, "neural");
+ assert!(!neural_identity.resumable);
+ // The neural width is the loaded model's `hidden_size`, which is
+ // not known until the model loads — reporting 384 here would be a
+ // false statement for any encoder that is not 384-wide.
+ assert_eq!(neural_identity.dimensions, None);
+ let encoded = serde_json::to_string(&neural_identity).unwrap();
+ assert!(!encoded.contains("dimensions"), "{encoded}");
+ }
+ #[cfg(not(feature = "neural"))]
+ {
+ assert_eq!(neural_identity.backend, "lexical");
+ assert!(neural_identity.resumable);
+ assert_eq!(
+ neural_identity.dimensions,
+ Some(xerj_ai::local::DEFAULT_DIMS)
+ );
+ }
+ let proxy = xerj_common::config::EmbeddingConfig {
+ mode: "proxy".into(),
+ default_endpoint: "https://secret.example/v1".into(),
+ default_model: "private-alias".into(),
+ ..Default::default()
+ };
+ let identity = embedding_execution_identity(&proxy).unwrap();
+ assert_eq!(identity.backend, "proxy");
+ assert!(!identity.resumable);
+ // A remote provider returns whatever width its model has; this server
+ // never sees it, so it is omitted rather than asserted.
+ assert_eq!(identity.dimensions, None);
+ let encoded = serde_json::to_string(&identity).unwrap();
+ assert!(!encoded.contains("secret.example"));
+ assert!(!encoded.contains("private-alias"));
+ assert!(!encoded.contains("dimensions"), "{encoded}");
+
+ for endpoint in ["", "not a url", "file:///private/model"] {
+ let fallback = xerj_common::config::EmbeddingConfig {
+ mode: "proxy".into(),
+ default_endpoint: endpoint.into(),
+ ..Default::default()
+ };
+ let identity = embedding_execution_identity(&fallback).unwrap();
+ assert_eq!(identity.backend, "lexical", "{endpoint}");
+ assert!(identity.resumable, "{endpoint}");
+ // The fallback really did become lexical, so the pinned lexical
+ // width is the truthful answer here.
+ assert_eq!(
+ identity.dimensions,
+ Some(xerj_ai::local::DEFAULT_DIMS),
+ "{endpoint}"
+ );
+ }
+ }
+
+ #[cfg(feature = "onnx-experimental")]
+ #[test]
+ fn onnx_execution_identity_changes_with_asset_content() {
+ let dir = tempfile::tempdir().unwrap();
+ let first = embedding_execution_identity(&onnx_config(dir.path(), "a")).unwrap();
+ let second = embedding_execution_identity(&onnx_config(dir.path(), "b")).unwrap();
+ assert!(first.resumable);
+ // ONNX is the one non-lexical backend whose width the server does pin:
+ // `validate_onnx_dimensions` refuses any mapping that is not 384.
+ assert_eq!(first.dimensions, Some(384));
+ assert_ne!(first.identity_sha256, second.identity_sha256);
+ }
+
+ #[cfg(feature = "onnx-experimental")]
+ #[test]
+ fn onnx_identity_and_lazy_loader_share_one_immutable_byte_snapshot() {
+ use std::fs::{File, FileTimes, OpenOptions};
+
+ let dir = tempfile::tempdir().unwrap();
+ let cfg = onnx_config(dir.path(), "stable-path");
+ let model_path = Path::new(&cfg.onnx_model_path);
+ let tokenizer_path = Path::new(&cfg.onnx_tokenizer_path);
+ let model_modified = model_path.metadata().unwrap().modified().unwrap();
+ let tokenizer_modified = tokenizer_path.metadata().unwrap().modified().unwrap();
+
+ let identity_before = embedding_execution_identity(&cfg).unwrap();
+ let snapshot_before = configured_onnx_assets(&cfg).unwrap();
+ let replacement_model = vec![b'x'; snapshot_before.model_bytes.len()];
+ let replacement_tokenizer = vec![b'y'; snapshot_before.tokenizer_bytes.len()];
+ std::fs::write(model_path, &replacement_model).unwrap();
+ std::fs::write(tokenizer_path, &replacement_tokenizer).unwrap();
+ OpenOptions::new()
+ .write(true)
+ .open(model_path)
+ .unwrap()
+ .set_times(FileTimes::new().set_modified(model_modified))
+ .unwrap();
+ File::options()
+ .write(true)
+ .open(tokenizer_path)
+ .unwrap()
+ .set_times(FileTimes::new().set_modified(tokenizer_modified))
+ .unwrap();
+
+ let snapshot_for_loader = configured_onnx_assets(&cfg).unwrap();
+ let identity_after = embedding_execution_identity(&cfg).unwrap();
+ assert_eq!(identity_before, identity_after);
+ assert_eq!(
+ snapshot_for_loader.model_bytes.as_ref(),
+ snapshot_before.model_bytes.as_ref()
+ );
+ assert_eq!(
+ snapshot_for_loader.tokenizer_bytes.as_ref(),
+ snapshot_before.tokenizer_bytes.as_ref()
+ );
+ assert_ne!(snapshot_for_loader.model_bytes.as_ref(), replacement_model);
+ assert_ne!(
+ snapshot_for_loader.tokenizer_bytes.as_ref(),
+ replacement_tokenizer
+ );
+ }
+
+ #[cfg(feature = "onnx-experimental")]
+ #[test]
+ fn onnx_snapshot_is_scoped_to_one_runtime_config_and_released_with_it() {
+ let dir = tempfile::tempdir().unwrap();
+ let first_cfg = onnx_config(dir.path(), "runtime-scope");
+ let first = configured_onnx_assets(&first_cfg).unwrap();
+ let weak_model = Arc::downgrade(&first.model_bytes);
+ let first_identity = embedding_execution_identity(&first_cfg).unwrap();
+
+ let replacement_model = vec![b'z'; first.model_bytes.len()];
+ std::fs::write(&first_cfg.onnx_model_path, &replacement_model).unwrap();
+ drop(first);
+ drop(first_cfg);
+ assert!(
+ weak_model.upgrade().is_none(),
+ "dropping the runtime config must release its immutable asset snapshot"
+ );
+
+ let second_cfg = xerj_common::config::EmbeddingConfig {
+ mode: "onnx-experimental".into(),
+ onnx_model_path: dir
+ .path()
+ .join("model-runtime-scope.onnx")
+ .to_string_lossy()
+ .into_owned(),
+ onnx_tokenizer_path: dir
+ .path()
+ .join("tokenizer-runtime-scope.json")
+ .to_string_lossy()
+ .into_owned(),
+ ..Default::default()
+ };
+ let second = configured_onnx_assets(&second_cfg).unwrap();
+ let second_identity = embedding_execution_identity(&second_cfg).unwrap();
+ assert_eq!(second.model_bytes.as_ref(), replacement_model);
+ assert_ne!(
+ first_identity.identity_sha256,
+ second_identity.identity_sha256
+ );
+ }
+
+ #[cfg(feature = "onnx-experimental")]
+ #[tokio::test]
+ async fn sequential_engines_from_one_caller_config_refresh_same_path_assets() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut caller_config = xerj_common::config::Config::default();
+ caller_config.server.data_dir = dir
+ .path()
+ .join("engine-data")
+ .to_string_lossy()
+ .into_owned();
+ caller_config.embedding = onnx_config(dir.path(), "engine-lifetime");
+
+ let first_engine = crate::Engine::new(caller_config.clone()).unwrap();
+ let first_identity = first_engine.embedding_execution_identity().unwrap();
+ let model_path = caller_config.embedding.onnx_model_path.clone();
+ let original_len = std::fs::metadata(&model_path).unwrap().len() as usize;
+ drop(first_engine);
+
+ std::fs::write(&model_path, vec![b'q'; original_len]).unwrap();
+ let second_engine = crate::Engine::new(caller_config).unwrap();
+ let second_identity = second_engine.embedding_execution_identity().unwrap();
+ assert_ne!(
+ first_identity.identity_sha256,
+ second_identity.identity_sha256
+ );
+ }
+
+ #[cfg(feature = "onnx-experimental")]
+ #[test]
+ fn concurrent_identity_and_loader_access_share_one_runtime_snapshot() {
+ let dir = tempfile::tempdir().unwrap();
+ let cfg = onnx_config(dir.path(), "concurrent");
+ let identity_cfg = cfg.clone();
+ let identity_thread =
+ std::thread::spawn(move || embedding_execution_identity(&identity_cfg).unwrap());
+ let loader_cfg = cfg.clone();
+ let loader_thread =
+ std::thread::spawn(move || configured_onnx_assets(&loader_cfg).unwrap());
+
+ let identity = identity_thread.join().unwrap();
+ let loader_snapshot = loader_thread.join().unwrap();
+ let owner_snapshot = configured_onnx_assets(&cfg).unwrap();
+ assert_eq!(
+ identity.identity_sha256,
+ embedding_execution_identity(&cfg).unwrap().identity_sha256
+ );
+ assert!(Arc::ptr_eq(
+ &loader_snapshot.model_bytes,
+ &owner_snapshot.model_bytes
+ ));
+ assert!(Arc::ptr_eq(
+ &loader_snapshot.tokenizer_bytes,
+ &owner_snapshot.tokenizer_bytes
+ ));
+ }
+
+ /// Run with matching production assets:
+ ///
+ /// `XERJ_ONNX_TEST_MODEL=/abs/model.onnx
+ /// XERJ_ONNX_TEST_TOKENIZER=/abs/tokenizer.json
+ /// cargo test -p xerj-engine --features onnx-experimental
+ /// concurrent_identity_and_real_lazy_pool_share_snapshot -- --ignored`
+ #[cfg(feature = "onnx-experimental")]
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ #[ignore = "requires explicit matching ONNX model and tokenizer assets"]
+ async fn concurrent_identity_and_real_lazy_pool_share_snapshot() {
+ let cfg = xerj_common::config::EmbeddingConfig {
+ mode: "onnx-experimental".into(),
+ onnx_model_path: std::env::var("XERJ_ONNX_TEST_MODEL")
+ .expect("XERJ_ONNX_TEST_MODEL is required"),
+ onnx_tokenizer_path: std::env::var("XERJ_ONNX_TEST_TOKENIZER")
+ .expect("XERJ_ONNX_TEST_TOKENIZER is required"),
+ ..Default::default()
+ };
+ let barrier = Arc::new(std::sync::Barrier::new(2));
+ let identity_cfg = cfg.clone();
+ let identity_barrier = barrier.clone();
+ let identity = tokio::task::spawn_blocking(move || {
+ identity_barrier.wait();
+ embedding_execution_identity(&identity_cfg).unwrap()
+ });
+ let embedder = make_embedder(&cfg).unwrap();
+ barrier.wait();
+ let vectors = embedder
+ .embed_batch(vec!["quarterly revenue increased".into()])
+ .await
+ .unwrap();
+ let identity = identity.await.unwrap();
+ assert_eq!(identity.backend, "onnx-experimental");
+ assert_eq!(vectors.len(), 1);
+ assert_eq!(Some(vectors[0].len()), identity.dimensions);
+ let snapshot = configured_onnx_assets(&cfg).unwrap();
+ assert_eq!(
+ identity.identity_sha256,
+ embedding_execution_identity(&cfg).unwrap().identity_sha256
+ );
+ assert_eq!(
+ snapshot.model_sha256,
+ format!(
+ "{:x}",
+ <sha2::Sha256 as sha2::Digest>::digest(&snapshot.model_bytes)
+ )
+ );
+ }
+
#[cfg(feature = "onnx-experimental")]
fn onnx_config(dir: &Path, suffix: &str) -> xerj_common::config::EmbeddingConfig {
let model = dir.join(format!("model-{suffix}.onnx"));
@@ -23761,24 +24182,18 @@ fn make_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> Result<xerj_ai::
warn!("embedding.mode=proxy but embedding.default_endpoint is empty — falling back to lexical");
return Ok(xerj_ai::Embedder::lexical());
}
- let proxy_cfg = xerj_ai::embed::EmbeddingProxyConfig {
- endpoint: cfg.default_endpoint.clone(),
- api_key: std::env::var("XERJ_EMBEDDING_API_KEY").ok(),
- model: cfg.default_model.clone(),
- timeout_secs: cfg.timeout_ms / 1000,
- max_concurrent: 4,
- max_retries: 3,
- };
- return Ok(match xerj_ai::embed::EmbeddingProxy::new(proxy_cfg) {
- Ok(p) => {
- info!(endpoint = %cfg.default_endpoint, "embedding backend: external proxy");
- xerj_ai::Embedder::proxy(p)
- }
- Err(e) => {
- warn!(error = %e, "embedding proxy init failed — falling back to lexical");
- xerj_ai::Embedder::lexical()
- }
- });
+ return Ok(
+ match xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)) {
+ Ok(p) => {
+ info!(endpoint = %cfg.default_endpoint, "embedding backend: external proxy");
+ xerj_ai::Embedder::proxy(p)
+ }
+ Err(e) => {
+ warn!(error = %e, "embedding proxy init failed — falling back to lexical");
+ xerj_ai::Embedder::lexical()
+ }
+ },
+ );
}
Ok(xerj_ai::Embedder::lexical())
@@ -23842,11 +24257,12 @@ fn make_onnx_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> Result<xerj
),
)));
}
+ let assets = configured_onnx_assets(cfg)?;
let onnx_cfg = xerj_ai::embedder::OnnxConfig {
- model_sha256: sha256_file(&model_path)?,
- tokenizer_sha256: sha256_file(&tokenizer_path)?,
- model_path,
- tokenizer_path,
+ model_sha256: assets.model_sha256,
+ tokenizer_sha256: assets.tokenizer_sha256,
+ model_bytes: assets.model_bytes,
+ tokenizer_bytes: assets.tokenizer_bytes,
intra_threads: cfg.onnx_intra_threads.max(1),
session_pool_size: cfg.onnx_session_pool_size,
microbatch: xerj_ai::onnx::MicrobatchConfig {
@@ -23859,8 +24275,8 @@ fn make_onnx_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> Result<xerj
max_inflight_input_bytes: cfg.onnx_max_inflight_input_bytes,
};
info!(
- model = %onnx_cfg.model_path.display(),
- tokenizer = %onnx_cfg.tokenizer_path.display(),
+ model_sha256 = %onnx_cfg.model_sha256,
+ tokenizer_sha256 = %onnx_cfg.tokenizer_sha256,
scheduling_window = cfg.onnx_scheduling_window,
session_pool_size = onnx_cfg.session_pool_size,
"embedding backend: experimental ONNX Runtime (loads on first use)"
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 6a51a9d28..1e07ea989 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -40,7 +40,7 @@ mod bounded_ingest_resources_tests;
// ── Re-exports ────────────────────────────────────────────────────────────────
-pub use engine::{Engine, HealthStatus, IndexInfo};
+pub use engine::{EmbeddingExecutionIdentity, Engine, HealthStatus, IndexInfo};
pub use index::{
detect_log_format, resolve_date_math, resolve_field_alias, EnrichTable, FieldEncodingInfo,
Index, IndexResponse, IndexStats, LogFormat,