Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): resume semantic runs that use a proxy embedder (#2307)

xerj-org47622fa19778
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 24ea8ad2d..3b7f6a59c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -59,6 +59,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Semantic `xerj autoindex` against `--embed-mode proxy` resumes after a
+ mid-run failure and does not re-embed documents already indexed, when the
+ backend, configured model, endpoint, and confirmed vector width are
+ unchanged. The proxy digest is still the model and endpoint (#533); the
+ width is learned with one probe (and remembered under the data directory)
+ and compared on resume, so a successful probe does not change the hash an
+ existing index was recorded under. A proxy whose endpoint has not confirmed
+ a width stays non-resumable, and the refusal says to make the endpoint
+ reachable and rerun — already-indexed documents are kept — or to rebuild
+ with a new `--state-dir` and `--prefix` only to discard the corpus. A
+ provider that swaps weights without changing the name, endpoint, or width
+ is still not detectable
+ ([#2306](https://github.com/xerj-org/xerj/issues/2306)).
+
- An existing index's persisted `index.default_pipeline` runs again after a
clean restart. `GET /_settings` still reported the binding, and the
pipeline definition was still there, but an ordinary document write no
diff --git a/engine/crates/xerj-api/tests/proxy_semantic_autoindex_resumes.rs b/engine/crates/xerj-api/tests/proxy_semantic_autoindex_resumes.rs
new file mode 100644
index 000000000..d98034c3f
--- /dev/null
+++ b/engine/crates/xerj-api/tests/proxy_semantic_autoindex_resumes.rs
@@ -0,0 +1,317 @@
+//! Issue #2306: a semantic autoindex run whose embedder is `--embed-mode proxy`
+//! must resume after a mid-run failure. Documents already embedded are not
+//! sent to the proxy again when the backend, model, endpoint, and vector
+//! width are unchanged.
+//!
+//! The mock answers OpenAI `/v1/embeddings` with 384-wide vectors (the width
+//! `semantic_text` uses when the mapping omits `dimensions`). The identity
+//! probe always succeeds. The first document embed succeeds and is indexed;
+//! every later document embed is refused with HTTP 400 until the rerun turns
+//! the refusal off. 400 is permanent, so the interrupted run fails without
+//! retry backoff. Which file is first does not matter.
+
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+use xerj_autoindex::cli::IndexCfg;
+
+const DIMS: usize = 384;
+const ALPHA: &str = "ALPHA_DOC_ONE";
+const BETA: &str = "BETA_DOC_TWO";
+/// Same text `confirm_proxy_embedding_width` sends. Kept here so the interrupt
+/// does not treat the identity probe as a document.
+const PROBE: &str = "xerj-proxy-identity-probe";
+
+fn prose(marker: &str) -> String {
+ format!(
+ "{marker} The quarterly revenue report describes how the operating team closed the books \
+ after a long review of customer contracts and deferred revenue. The narrative is ordinary \
+ prose so the indexer treats the file as a document and embeds the body once, instead of \
+ splitting it into line windows."
+ )
+}
+
+struct EmbedLog {
+ texts: Mutex<Vec<String>>,
+ /// Document embed requests that may still succeed while `reject` is set.
+ allow_document_requests: AtomicUsize,
+ reject: AtomicBool,
+}
+
+async fn mock_embedder() -> (String, Arc<EmbedLog>) {
+ let log = Arc::new(EmbedLog {
+ texts: Mutex::new(Vec::new()),
+ allow_document_requests: AtomicUsize::new(1),
+ reject: AtomicBool::new(true),
+ });
+ let shared = Arc::clone(&log);
+ let app = axum::Router::new().route(
+ "/v1/embeddings",
+ axum::routing::post(move |axum::Json(body): axum::Json<Value>| {
+ let shared = Arc::clone(&shared);
+ async move {
+ let inputs: Vec<String> = match body.get("input") {
+ Some(Value::Array(items)) => items
+ .iter()
+ .filter_map(|item| item.as_str().map(str::to_owned))
+ .collect(),
+ Some(Value::String(text)) => vec![text.clone()],
+ _ => Vec::new(),
+ };
+ shared.texts.lock().unwrap().extend(inputs.iter().cloned());
+ let document = inputs.iter().any(|text| !text.contains(PROBE));
+ let allowed = !document
+ || !shared.reject.load(Ordering::SeqCst)
+ || shared
+ .allow_document_requests
+ .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| {
+ left.checked_sub(1)
+ })
+ .is_ok();
+ if !allowed {
+ return (
+ StatusCode::BAD_REQUEST,
+ axum::Json(json!({"error": "interrupted"})),
+ );
+ }
+ let data: Vec<Value> = inputs
+ .iter()
+ .enumerate()
+ .map(|(index, _)| {
+ let mut embedding = vec![0.0_f64; DIMS];
+ embedding[0] = 1.0;
+ json!({"index": index, "embedding": embedding})
+ })
+ .collect();
+ (StatusCode::OK, axum::Json(json!({"data": data})))
+ }
+ }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("bind mock embedder");
+ let addr = listener.local_addr().expect("mock embedder addr");
+ tokio::spawn(async move {
+ let _ = axum::serve(listener, app).await;
+ });
+ (format!("http://{addr}/v1/embeddings"), log)
+}
+
+async fn proxy_node(endpoint: String) -> (axum::Router, String) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.auth.enabled = false;
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ config.embedding.mode = "proxy".into();
+ config.embedding.default_endpoint = endpoint;
+ config.embedding.default_model = "mock-embed".into();
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ std::mem::forget(dir);
+ let app = xerj_api::router::build_es_compat_router(state);
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("bind xerj");
+ let addr = listener.local_addr().expect("xerj addr");
+ let served = app.clone();
+ tokio::spawn(async move {
+ let _ = axum::serve(listener, served).await;
+ });
+ (app, format!("http://{addr}"))
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+fn index_cfg(root: &Path, state_dir: &Path, url: &str, prefix: &str, no_graph: bool) -> IndexCfg {
+ IndexCfg {
+ root: root.to_owned(),
+ endpoint_url: None,
+ stub_globs: Vec::new(),
+ url: url.to_owned(),
+ api_key: None,
+ api_key_file: None,
+ workers: 1,
+ scan_workers: 1,
+ pdf_workers: 1,
+ resource_notes: Vec::new(),
+ xerj_url_note: None,
+ pdf_timeout_secs: 30,
+ bulk_mb: 1,
+ bulk_timeout_secs: 30,
+ snapshot_max_bytes: 64 << 30,
+ prefix: prefix.to_owned(),
+ replace_prefix: false,
+ prefix_note: None,
+ state_dir: Some(state_dir.to_owned()),
+ fresh: false,
+ follow_symlinks: false,
+ follow_symlinks_outside_root: false,
+ ignore: xerj_autoindex::ignore_rules::IgnoreOptions::default(),
+ max_file_bytes: 1 << 30,
+ sample: 50,
+ no_semantic: false,
+ code_analyzer: Default::default(),
+ brain: Some(format!("brain-{prefix}")),
+ no_graph,
+ max_minutes: 0,
+ approve: None,
+ max_index_bytes: None,
+ budget_plan: None,
+ drop_indexed: false,
+ dry_run: false,
+ walk_only: false,
+ json: false,
+ quiet: true,
+ progress: xerj_autoindex::progress::ProgressMode::None,
+ progress_interval: None,
+ watch: false,
+ label: None,
+ debounce: Duration::from_millis(0),
+ exclude_rels: Vec::new(),
+ skip_corpus_data: false,
+ }
+}
+
+fn run_autoindex(
+ root: &Path,
+ state_dir: &Path,
+ url: &str,
+ prefix: &str,
+ no_graph: bool,
+) -> Result<i32, String> {
+ let cfg = index_cfg(root, state_dir, url, prefix, no_graph);
+ match xerj_autoindex::run_index_report(cfg) {
+ Ok((code, _)) => Ok(code),
+ Err(error) => Err(format!("{error:#}")),
+ }
+}
+
+async fn run_autoindex_blocking(
+ root: &Path,
+ state_dir: &Path,
+ url: &str,
+ prefix: &str,
+ no_graph: bool,
+) -> Result<i32, String> {
+ let root = root.to_owned();
+ let state_dir = state_dir.to_owned();
+ let url = url.to_owned();
+ let prefix = prefix.to_owned();
+ tokio::task::spawn_blocking(move || run_autoindex(&root, &state_dir, &url, &prefix, no_graph))
+ .await
+ .expect("autoindex task")
+}
+
+fn texts_containing(log: &[String], marker: &str) -> usize {
+ log.iter().filter(|text| text.contains(marker)).count()
+}
+
+async fn indexed_markers(app: &axum::Router, prefix: &str) -> (bool, bool) {
+ let (refresh_status, refresh_body) =
+ call(app, "POST", &format!("/{prefix}-*/_refresh"), Value::Null).await;
+ assert!(
+ refresh_status.is_success(),
+ "refresh {prefix}: {refresh_status} {refresh_body}"
+ );
+ let (status, body) = call(
+ app,
+ "POST",
+ &format!("/{prefix}-*/_search"),
+ json!({"size": 50, "query": {"match_all": {}}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "search {prefix}: {body}");
+ let rendered = body.to_string();
+ (rendered.contains(ALPHA), rendered.contains(BETA))
+}
+
+async fn interrupt_then_resume(no_graph: bool) {
+ let (endpoint, log) = mock_embedder().await;
+ let (app, url) = proxy_node(endpoint).await;
+ let corpus = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ std::fs::write(corpus.path().join("a-alpha.txt"), prose(ALPHA)).unwrap();
+ std::fs::write(corpus.path().join("b-beta.txt"), prose(BETA)).unwrap();
+ let prefix = if no_graph { "pxresumeng" } else { "pxresume" };
+
+ let interrupted = run_autoindex_blocking(corpus.path(), state.path(), &url, prefix, no_graph)
+ .await
+ .expect_err("the refused document must stop the run");
+ assert!(
+ interrupted.contains("embedding") || interrupted.contains("bulk"),
+ "the failure must name the embedding problem: {interrupted}"
+ );
+
+ let during = log.texts.lock().unwrap().clone();
+ assert!(
+ texts_containing(&during, ALPHA) >= 1 && texts_containing(&during, BETA) >= 1,
+ "the run must embed one document and reach the next before it stops: {during:?}"
+ );
+ let (alpha_indexed, beta_indexed) = indexed_markers(&app, prefix).await;
+ assert!(
+ alpha_indexed != beta_indexed,
+ "exactly one document is indexed before the refusal \
+ (alpha={alpha_indexed} beta={beta_indexed})"
+ );
+ let already = if alpha_indexed { ALPHA } else { BETA };
+ let pending = if alpha_indexed { BETA } else { ALPHA };
+
+ let cutoff = during.len();
+ log.reject.store(false, Ordering::SeqCst);
+ let code = run_autoindex_blocking(corpus.path(), state.path(), &url, prefix, no_graph)
+ .await
+ .unwrap_or_else(|error| panic!("resume failed: {error}"));
+ assert_eq!(code, 0, "resume must finish");
+
+ let resumed = log.texts.lock().unwrap().clone();
+ let again = &resumed[cutoff..];
+ assert_eq!(
+ texts_containing(again, already),
+ 0,
+ "already-indexed documents must not be re-embedded: {again:?}"
+ );
+ assert!(
+ texts_containing(again, pending) >= 1,
+ "the document that failed must be embedded on the rerun: {again:?}"
+ );
+ let (alpha_indexed, beta_indexed) = indexed_markers(&app, prefix).await;
+ assert!(
+ alpha_indexed && beta_indexed,
+ "both documents are searchable after resume (alpha={alpha_indexed} beta={beta_indexed})"
+ );
+}
+
+#[tokio::test]
+async fn proxy_semantic_autoindex_resumes_without_reembedding() {
+ interrupt_then_resume(false).await;
+}
+
+#[tokio::test]
+async fn proxy_semantic_autoindex_resumes_without_reembedding_no_graph() {
+ interrupt_then_resume(true).await;
+}
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index 6983f9642..52e6c85c9 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -1141,10 +1141,11 @@ pub struct EmbeddingExecutionIdentity {
pub version: u32,
pub backend: String,
pub identity_sha256: String,
- /// Absent for backends whose vector width the server does not pin
- /// (`proxy`, and a `neural` node that could not resolve its model assets;
- /// a resolved `neural` node reports its `config.json` `hidden_size` since
- /// #487). An absent width must not be read as 384.
+ /// Absent when the server has not pinned a vector width: a `proxy` whose
+ /// endpoint has not confirmed one (#2306), and a `neural` node that could
+ /// not resolve its model assets. A confirmed proxy reports the probed
+ /// width, and a resolved `neural` node reports its `config.json`
+ /// `hidden_size` (#487). An absent width must not be read as 384.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dimensions: Option<usize>,
pub semantic_contract: String,
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 17bbe24b9..ef4668bf1 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -1798,6 +1798,7 @@ fn begin_non_graph_generation(
journal.pin_embedding_identity(
&identity.identity_sha256,
identity.resumable,
+ identity.dimensions,
identity.non_resumable_reason.as_deref(),
)?;
identity
@@ -5080,13 +5081,15 @@ fn pending_for_phase_b(
.collect()
}
-/// #487: an unchanged re-index on a `neural`/`proxy` backend must not fail with
-/// a false "embedding execution identity changed" message. The identity CHANGED
-/// only if a comparable field differs from what the journal pinned; `resumable`
-/// is a standing property of the backend (false by construction for backends
-/// whose identity hashes the model NAME, not its bytes), so it is checked
-/// SEPARATELY with an accurate message — restoring the identity cannot fix a
-/// backend that simply cannot resume.
+/// An unchanged re-index must not fail with a false "embedding execution
+/// identity changed" message (#487). The identity changed only when a
+/// comparable field differs from what the journal pinned. `resumable` is
+/// checked on its own.
+///
+/// A proxy whose vector width is not confirmed reports `dimensions: None` and
+/// `resumable: false` (#2306). That missing width is the resume limit, not a
+/// different embedder. A confirmed width that disagrees with the pinned width
+/// is a real change and must not be mixed into the corpus.
fn ensure_embedding_execution_unchanged_and_resumable(
current: &crate::esclient::EmbeddingExecutionIdentity,
expected_sha: &str,
@@ -5094,11 +5097,22 @@ fn ensure_embedding_execution_unchanged_and_resumable(
expected_dimension: Option<usize>,
expected_semantic_contract: &str,
) -> anyhow::Result<()> {
+ let same_embedder = current.identity_sha256 == expected_sha
+ && current.backend == expected_backend
+ && current.semantic_contract == expected_semantic_contract;
+ let same_width = current.dimensions == expected_dimension;
+ if same_embedder && !same_width && !(current.dimensions.is_none() && !current.resumable) {
+ anyhow::bail!(
+ "embedding execution identity changed since this autoindex journal was created \
+ (vector width {} vs {}); refusing to mix vector spaces. No remote mutation was \
+ attempted. Restore the original identity, or rebuild with a new --state-dir and a \
+ new --prefix",
+ dimension_label(expected_dimension),
+ dimension_label(current.dimensions),
+ );
+ }
anyhow::ensure!(
- current.identity_sha256 == expected_sha
- && current.backend == expected_backend
- && current.dimensions == expected_dimension
- && current.semantic_contract == expected_semantic_contract,
+ same_embedder && (same_width || (current.dimensions.is_none() && !current.resumable)),
"embedding execution identity changed since this autoindex journal was created; \
refusing to mix vector spaces. No remote mutation was attempted. Restore the \
original identity, or rebuild with a new --state-dir and a new --prefix"
@@ -5107,8 +5121,10 @@ fn ensure_embedding_execution_unchanged_and_resumable(
current.resumable,
"the `{}` embedding backend cannot resume semantic indexing{}. The pinned identity is \
UNCHANGED — this is a backend limitation, not a change, so restoring the identity will \
- not help. Rebuild with a new --state-dir and a new --prefix, or content-address the \
- model identity so this backend can resume (#367).",
+ not help. Follow the reason above. Documents already indexed were kept. If the vector \
+ width was not confirmed, make the embedding endpoint reachable and rerun — \
+ already-embedded documents are reused once the same width is confirmed. Rebuild with a \
+ new --state-dir and a new --prefix only to discard this corpus and start over.",
current.backend,
current
.non_resumable_reason
@@ -5119,6 +5135,13 @@ fn ensure_embedding_execution_unchanged_and_resumable(
Ok(())
}
+fn dimension_label(width: Option<usize>) -> String {
+ match width {
+ Some(width) => width.to_string(),
+ None => "unreported".to_string(),
+ }
+}
+
fn select_resume_plan_keys(
files: &[walk::FileEntry],
content_keys: &[String],
@@ -5967,6 +5990,7 @@ fn pin_pending_embedding_identity(
journal.pin_embedding_identity(
&current.identity_sha256,
current.resumable,
+ current.dimensions,
current.non_resumable_reason.as_deref(),
)
}
@@ -8883,6 +8907,7 @@ fn run_index_report_inner(
journal.pin_embedding_identity(
&identity.identity_sha256,
identity.resumable,
+ identity.dimensions,
identity.non_resumable_reason.as_deref(),
)?;
// #1189: name the live backend in the run output. A server started
@@ -8893,6 +8918,15 @@ fn run_index_report_inner(
identity.backend,
if identity.resumable { "" } else { "non-" }
));
+ if !identity.resumable {
+ pr.note(&format!(
+ "semantic autoindex cannot resume this embedding backend: {}",
+ identity
+ .non_resumable_reason
+ .as_deref()
+ .unwrap_or("the server did not provide an immutable embedding identity")
+ ));
+ }
}
// ── create indices with explicit mappings ────────────────────────────
@@ -12782,6 +12816,47 @@ mod duplicate_integration_tests {
err2.contains("identity changed"),
"a real identity change must still report it: {err2}"
);
+ // A confirmed width that disagrees with the pinned width is a real
+ // change. An unconfirmed width on a non-resumable backend is not.
+ let wider = crate::esclient::EmbeddingExecutionIdentity {
+ dimensions: Some(768),
+ resumable: true,
+ non_resumable_reason: None,
+ ..current.clone()
+ };
+ let err3 = ensure_embedding_execution_unchanged_and_resumable(
+ &wider,
+ "6e51e5ce3e46",
+ "neural",
+ None,
+ "semantic_text-derived-vector.v1",
+ )
+ .unwrap_err()
+ .to_string();
+ assert!(
+ err3.contains("identity changed") && err3.contains("768"),
+ "a different vector width must be reported as a change: {err3}"
+ );
+ let unconfirmed = crate::esclient::EmbeddingExecutionIdentity {
+ dimensions: None,
+ resumable: false,
+ non_resumable_reason: Some("vector width was not confirmed".into()),
+ ..current
+ };
+ let err4 = ensure_embedding_execution_unchanged_and_resumable(
+ &unconfirmed,
+ "6e51e5ce3e46",
+ "neural",
+ Some(384),
+ "semantic_text-derived-vector.v1",
+ )
+ .unwrap_err()
+ .to_string();
+ assert!(
+ err4.contains("cannot resume") && !err4.contains("identity changed"),
+ "an unconfirmed width is a resume limit, not a changed identity: {err4}"
+ );
+ assert!(err4.contains("vector width was not confirmed"), "{err4}");
}
#[test]
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index b9f308834..4e132190d 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -283,16 +283,16 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
let mut first = Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap();
first
- .pin_embedding_identity(&"a".repeat(64), true, None)
+ .pin_embedding_identity(&"a".repeat(64), true, None, None)
.unwrap();
drop(first);
let mut resumed = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
resumed
- .pin_embedding_identity(&"a".repeat(64), true, None)
+ .pin_embedding_identity(&"a".repeat(64), true, None, None)
.unwrap();
let error = resumed
- .pin_embedding_identity(&"b".repeat(64), true, None)
+ .pin_embedding_identity(&"b".repeat(64), true, None, None)
.unwrap_err()
.to_string();
assert!(error.contains("refusing to mix vector spaces"), "{error}");
@@ -307,12 +307,12 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
let mut journal = Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap();
journal
- .pin_embedding_identity(&"a".repeat(64), false, Some("remote alias can drift"))
+ .pin_embedding_identity(&"a".repeat(64), false, None, Some("remote alias can drift"))
.unwrap();
drop(journal);
let mut resumed = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
let error = resumed
- .pin_embedding_identity(&"a".repeat(64), false, Some("remote alias can drift"))
+ .pin_embedding_identity(&"a".repeat(64), false, None, Some("remote alias can drift"))
.unwrap_err()
.to_string();
assert!(error.contains("remote alias can drift"), "{error}");
@@ -321,6 +321,62 @@ mod tests {
assert!(error.contains("delete and recreate"), "{error}");
}
+ /// #2306: a journal pinned while the proxy width was still unconfirmed
+ /// (`resumable: false`, no width) resumes once the same digest is reported
+ /// as resumable. A later width that disagrees with a pinned width does not.
+ #[test]
+ fn confirmed_width_resumes_a_journal_pinned_before_confirmation() {
+ let dir = tempfile::tempdir().unwrap();
+ let digest = "a".repeat(64);
+ let mut journal = Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap();
+ journal
+ .pin_embedding_identity(&digest, false, None, Some("width was not confirmed"))
+ .unwrap();
+ drop(journal);
+
+ let mut resumed = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ resumed
+ .pin_embedding_identity(&digest, true, Some(384), None)
+ .expect("same digest becomes resumable once the width is confirmed");
+ // The pre-confirmation record did not carry a width, and resume does
+ // not append a second identity line (a flipped resumable flag would
+ // fail replay). A width is therefore not yet pinned on this journal.
+ assert_eq!(resumed.embedding_identity_dimensions, None);
+ }
+
+ #[test]
+ fn a_pinned_width_is_durable_and_a_missing_width_is_not_drift() {
+ let dir = tempfile::tempdir().unwrap();
+ let digest = "c".repeat(64);
+ let mut first = Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap();
+ first
+ .pin_embedding_identity(&digest, true, Some(384), None)
+ .unwrap();
+ drop(first);
+ let text = std::fs::read_to_string(dir.path().join("journal.ndjson")).unwrap();
+ assert!(text.contains("\"dimensions\":384"), "{text}");
+
+ let mut resumed = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ assert_eq!(resumed.embedding_identity_dimensions, Some(384));
+ let lost = resumed
+ .pin_embedding_identity(&digest, true, None, None)
+ .unwrap_err()
+ .to_string();
+ assert!(lost.contains("no longer reported"), "{lost}");
+ assert!(lost.contains("--fresh"), "{lost}");
+ let drifted = resumed
+ .pin_embedding_identity(&digest, true, Some(768), None)
+ .unwrap_err()
+ .to_string();
+ assert!(
+ drifted.contains("refusing to mix vector spaces"),
+ "{drifted}"
+ );
+ assert!(drifted.contains("384"), "{drifted}");
+ assert!(drifted.contains("768"), "{drifted}");
+ assert!(drifted.contains("--prefix"), "{drifted}");
+ }
+
#[test]
fn completed_legacy_semantic_journal_requires_fresh_rebuild() {
let dir = tempfile::tempdir().unwrap();
@@ -336,7 +392,7 @@ mod tests {
.unwrap();
let mut journal = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
let error = journal
- .pin_embedding_identity(&"a".repeat(64), true, None)
+ .pin_embedding_identity(&"a".repeat(64), true, None, None)
.unwrap_err()
.to_string();
assert!(
@@ -352,6 +408,11 @@ mod tests {
fn malformed_or_conflicting_identity_records_fail_during_replay() {
for tail in [
"{\"v\":1,\"kind\":\"embedding_identity\",\"identity_sha256\":\"bad\",\"resumable\":true}\n",
+ concat!(
+ "{\"v\":1,\"kind\":\"embedding_identity\",\"identity_sha256\":\"",
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "\",\"resumable\":true,\"dimensions\":0}\n"
+ ),
concat!(
"{\"v\":1,\"kind\":\"embedding_identity\",\"identity_sha256\":\"",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
@@ -688,6 +749,11 @@ pub struct Journal {
pub plan: Option<Plan>,
pub embedding_identity_sha256: Option<String>,
pub embedding_identity_resumable: Option<bool>,
+ /// Vector width pinned with the identity, when the record carried one.
+ ///
+ /// Older journals omit it. A resume that now reports a width is not drift
+ /// for that reason alone (#2306); two recorded widths that disagree are.
+ pub embedding_identity_dimensions: Option<usize>,
/// Newest corpus reconciliation transaction without a durable
/// `sync_commit`. Its desired plan is deliberately not authoritative yet.
pub committed_manifest: Option<crate::sync::CommittedManifest>,
@@ -1081,6 +1147,19 @@ pub struct JournalPreflight {
pub no_graph_genesis: bool,
}
+fn embedding_resume_refused(reason: &str) -> String {
+ format!(
+ "server embedding backend cannot safely resume semantic indexing: {reason}. \
+ No remote mutation was attempted. Documents already indexed were kept. \
+ If the vector width was not confirmed, make the embedding endpoint reachable \
+ and rerun the same command — already-embedded documents are reused once the \
+ same width is confirmed. Restore the exact original embedding identity, or \
+ rebuild all vectors with --fresh and a new --prefix only when the backend, \
+ model, endpoint, or vector width changed. Before reusing the old prefix, \
+ delete and recreate its prior autoindex indices"
+ )
+}
+
impl Journal {
/// `fresh` is not a formality here. `open_after_preflight` removes the
/// journal before replaying it when `--fresh` is set, so nothing inside a
@@ -1216,6 +1295,10 @@ impl Journal {
let mut plan = None;
let mut embedding_identity_sha256 = None;
let mut embedding_identity_resumable = None;
+ // `None` = no identity record yet. `Some(None)` = a record that omitted
+ // the width. `Some(Some(n))` = a recorded width. Absence and a width
+ // in the same journal disagree; two different widths disagree.
+ let mut embedding_identity_dimensions_seen: Option<Option<usize>> = None;
let mut committed_manifest: Option<crate::sync::CommittedManifest> = None;
let mut pending_sync: Option<crate::sync::PendingSync> = None;
let mut legacy_migration_reasons = Vec::new();
@@ -1307,15 +1390,42 @@ impl Journal {
})
});
let resumable = v.get("resumable").and_then(Value::as_bool);
+ let has_dimensions = v
+ .as_object()
+ .is_some_and(|object| object.contains_key("dimensions"));
+ let keys_ok = v.as_object().is_some_and(|object| {
+ let base = ["v", "kind", "identity_sha256", "resumable"]
+ .iter()
+ .all(|key| object.contains_key(*key));
+ if has_dimensions {
+ object.len() == 5 && base
+ } else {
+ object.len() == 4 && base
+ }
+ });
+ let recorded_width = if has_dimensions {
+ match v.get("dimensions").and_then(Value::as_u64) {
+ Some(width)
+ if width > 0 && usize::try_from(width).is_ok() =>
+ {
+ Some(width as usize)
+ }
+ _ => {
+ anyhow::bail!(
+ "journal at {} contains a malformed embedding \
+ identity; restore it from backup or re-run with \
+ --fresh",
+ jpath.display()
+ );
+ }
+ }
+ } else {
+ None
+ };
if v.get("v").and_then(Value::as_u64) != Some(1)
|| digest.is_none()
|| resumable.is_none()
- || v.as_object().is_none_or(|object| {
- object.len() != 4
- || !["v", "kind", "identity_sha256", "resumable"]
- .iter()
- .all(|key| object.contains_key(*key))
- })
+ || !keys_ok
{
anyhow::bail!(
"journal at {} contains a malformed embedding identity; \
@@ -1328,6 +1438,8 @@ impl Journal {
.is_some_and(|existing| existing != digest.unwrap())
|| embedding_identity_resumable
.is_some_and(|existing| existing != resumable.unwrap())
+ || embedding_identity_dimensions_seen
+ .is_some_and(|existing| existing != recorded_width)
{
anyhow::bail!(
"journal at {} contains conflicting embedding identities; \
@@ -1337,6 +1449,7 @@ impl Journal {
}
embedding_identity_sha256 = digest.map(str::to_owned);
embedding_identity_resumable = resumable;
+ embedding_identity_dimensions_seen = Some(recorded_width);
}
Some(kind) if crate::sync::is_sync_record_kind(kind) => {
// A durable record that no longer re-validates
@@ -1450,6 +1563,7 @@ impl Journal {
plan,
embedding_identity_sha256,
embedding_identity_resumable,
+ embedding_identity_dimensions: embedding_identity_dimensions_seen.flatten(),
committed_manifest,
pending_sync,
legacy_migration_reasons,
@@ -1477,10 +1591,15 @@ impl Journal {
}
/// Pin the server-side vector-space identity before any semantic write.
+ ///
+ /// `dimensions` is the confirmed vector width when the server has one.
+ /// The key is omitted from the journal record when it is absent, so a
+ /// journal written before widths were pinned still replays.
pub fn pin_embedding_identity(
&mut self,
identity_sha256: &str,
resumable: bool,
+ dimensions: Option<usize>,
non_resumable_reason: Option<&str>,
) -> Result<()> {
if identity_sha256.len() != 64
@@ -1490,15 +1609,6 @@ impl Journal {
{
anyhow::bail!("server returned an invalid embedding identity digest");
}
- if self.resumed && (!resumable || self.embedding_identity_resumable == Some(false)) {
- anyhow::bail!(
- "server embedding backend cannot safely resume semantic indexing: {}. \
- Restore the exact original embedding identity, or rebuild all vectors with \
- --fresh and a new --prefix. Before reusing the old prefix, delete and recreate \
- its prior autoindex indices",
- non_resumable_reason.unwrap_or("embedding identity is not immutable")
- );
- }
if let Some(existing) = &self.embedding_identity_sha256 {
if existing != identity_sha256 {
anyhow::bail!(
@@ -1508,8 +1618,53 @@ impl Journal {
reusing the old prefix, delete and recreate its prior autoindex indices"
);
}
+ match (self.embedding_identity_dimensions, dimensions) {
+ (Some(stored), Some(current)) if stored != current => {
+ anyhow::bail!(
+ "embedding execution identity changed since this autoindex journal was \
+ created (vector width {stored} vs {current}); refusing to mix vector \
+ spaces. Restore the original embedding identity, or rebuild all vectors \
+ with --fresh and a new --prefix. Before reusing the old prefix, delete \
+ and recreate its prior autoindex indices"
+ );
+ }
+ (Some(stored), None) => {
+ anyhow::bail!(
+ "embedding execution identity changed since this autoindex journal was \
+ created (vector width {stored} is no longer reported); refusing to mix \
+ vector spaces. Restore the original embedding identity, or rebuild all \
+ vectors with --fresh and a new --prefix. Before reusing the old prefix, \
+ delete and recreate its prior autoindex indices"
+ );
+ }
+ _ => {}
+ }
+ if !resumable {
+ // Same digest, still not safe to reuse vectors. A proxy whose
+ // width was never confirmed lands here; the server's reason
+ // says how to get a resume instead of a rebuild.
+ anyhow::bail!(
+ "{}",
+ embedding_resume_refused(
+ non_resumable_reason.unwrap_or("embedding identity is not immutable")
+ )
+ );
+ }
+ // Same digest, and the server can resume. Do not append another
+ // identity record: replay treats a flipped `resumable` flag as
+ // corruption. That is what lets a journal pinned before a proxy
+ // width could be confirmed (#2306) continue once this process
+ // confirms the same model and endpoint.
return Ok(());
}
+ if self.resumed && !resumable {
+ anyhow::bail!(
+ "{}",
+ embedding_resume_refused(
+ non_resumable_reason.unwrap_or("embedding identity is not immutable")
+ )
+ );
+ }
if self.resumed && !self.done.is_empty() {
anyhow::bail!(
"this semantic autoindex journal predates embedding identity pinning and cannot \
@@ -1517,17 +1672,19 @@ impl Journal {
reusing the old prefix, delete and recreate its prior autoindex indices"
);
}
- self.append_transaction(
- &serde_json::json!({
- "v": 1,
- "kind": "embedding_identity",
- "identity_sha256": identity_sha256,
- "resumable": resumable,
- }),
- "embedding_identity",
- )?;
+ let mut record = serde_json::json!({
+ "v": 1,
+ "kind": "embedding_identity",
+ "identity_sha256": identity_sha256,
+ "resumable": resumable,
+ });
+ if let Some(width) = dimensions {
+ record["dimensions"] = serde_json::json!(width);
+ }
+ self.append_transaction(&record, "embedding_identity")?;
self.embedding_identity_sha256 = Some(identity_sha256.to_owned());
self.embedding_identity_resumable = Some(resumable);
+ self.embedding_identity_dimensions = dimensions;
Ok(())
}
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index d549314d3..4cfcb02aa 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -1671,6 +1671,15 @@ pub struct EmbeddingConfig {
#[serde(skip)]
pub runtime_onnx_assets:
std::sync::Arc<std::sync::OnceLock<Result<EmbeddingAssetSnapshot, String>>>,
+ /// Vector width last confirmed for this proxy endpoint and model (#2306).
+ ///
+ /// Filled by the engine, not by configuration. Cloned configs share the
+ /// cell. Empty until a probe (or the on-disk fallback) succeeds; identity
+ /// reporting then treats the proxy as resumable and publishes the width.
+ /// Serde skips it: it is not a setting and must not round-trip through a
+ /// config file.
+ #[serde(skip)]
+ pub runtime_proxy_width: std::sync::Arc<std::sync::RwLock<Option<ConfirmedProxyWidth>>>,
/// Neural ingest probe, shared by cloned configs (the server preloads
/// from a clone). `None` inside the cell means the probe ran and did not
/// produce a rate; an empty cell means it has not run. Not a setting.
@@ -1678,6 +1687,18 @@ pub struct EmbeddingConfig {
pub neural_ingest_probe: std::sync::Arc<std::sync::OnceLock<Option<NeuralIngestProbe>>>,
}
+/// Vector width a proxy embedder confirmed for one endpoint and model.
+///
+/// The digest of a proxy identity stays the configured model and endpoint
+/// (#533). This width is the other half of that identity: autoindex compares
+/// it on resume and refuses to mix a different width into an existing corpus.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ConfirmedProxyWidth {
+ pub endpoint: String,
+ pub model: String,
+ pub dimensions: usize,
+}
+
#[derive(Debug, Clone)]
pub struct EmbeddingAssetSnapshot {
pub model_bytes: std::sync::Arc<[u8]>,
@@ -1710,6 +1731,7 @@ impl Default for EmbeddingConfig {
onnx_max_input_bytes_per_call: 8 * 1024 * 1024,
onnx_max_inflight_input_bytes: 32 * 1024 * 1024,
runtime_onnx_assets: std::sync::Arc::new(std::sync::OnceLock::new()),
+ runtime_proxy_width: std::sync::Arc::new(std::sync::RwLock::new(None)),
neural_ingest_probe: std::sync::Arc::new(std::sync::OnceLock::new()),
}
}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 506488586..fb8694083 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -176,8 +176,9 @@ pub struct EmbeddingExecutionIdentity {
/// while content-addressing it (#487) — the same number
/// `NeuralEmbedder::dims` reports, so it is known without loading the
/// model. It stays `None` for a `neural` node whose assets cannot be
- /// resolved at all, and for `proxy`, whose width is whatever the remote
- /// returns. A reader must not treat an absent width as 384.
+ /// resolved at all, and for a `proxy` whose endpoint has not confirmed a
+ /// width yet (#2306). A confirmed proxy reports the probed width. A reader
+ /// must not treat an absent width as 384.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dimensions: Option<usize>,
pub semantic_contract: String,
@@ -938,6 +939,9 @@ impl Engine {
// clone lineage. This ensures a later Engine re-reads same-path ONNX
// assets while all Index config clones within one Engine share bytes.
config.embedding.runtime_onnx_assets = Arc::new(std::sync::OnceLock::new());
+ // Same ownership rule for the proxy width cache: this engine probes
+ // into its own cell, and every index cloned from `config` shares it.
+ config.embedding.runtime_proxy_width = Arc::new(std::sync::RwLock::new(None));
let data_dir = PathBuf::from(&config.server.data_dir);
std::fs::create_dir_all(&data_dir)?;
@@ -4233,6 +4237,11 @@ impl Engine {
/// Return the effective embedding identity without exposing configuration
/// secrets or local asset paths.
pub fn embedding_execution_identity(&self) -> Result<EmbeddingExecutionIdentity> {
+ // #2306: a proxy resume needs a confirmed vector width. The probe runs
+ // here, on the identity request autoindex makes before it writes, and
+ // not on index open — open compares the model+endpoint digest, which
+ // does not wait on the endpoint.
+ crate::index::confirm_proxy_embedding_width(&self.config.embedding, &self.data_dir);
crate::index::embedding_execution_identity(&self.config.embedding)
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 7ca0d1535..f9b1d8d2a 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -39231,6 +39231,181 @@ fn configured_neural_identity(
unreachable!("the neural backend is not compiled into this build")
}
+/// Probe text sent once per process to learn a proxy model's vector width.
+///
+/// Kept distinct from document text so a resume test can tell the probe apart
+/// from a re-embedded document.
+const PROXY_WIDTH_PROBE_TEXT: &str = "xerj-proxy-identity-probe";
+
+/// On-disk copy of the last confirmed proxy width, under the data directory.
+///
+/// A restart whose endpoint is briefly down reuses this instead of reporting
+/// a different identity and refusing to resume. It is not an index directory
+/// (`Engine::new` only opens subdirectories that contain a `wal`).
+const PROXY_WIDTH_CACHE_FILE: &str = "proxy-embedding-probe.json";
+
+fn proxy_width_unconfirmed_reason() -> String {
+ "the proxy embedding endpoint has not confirmed a vector width, so semantic \
+ resume cannot reuse already-embedded documents. The embedder identity is the \
+ backend, the configured model, the endpoint, and the vector width; without a \
+ confirmed width a rerun cannot tell this model from a different one. Make the \
+ embedding endpoint reachable and rerun — documents already indexed are kept and \
+ are not re-embedded once the same width is confirmed. Rebuild with a new \
+ --state-dir and a new --prefix only to discard this corpus and start over"
+ .to_string()
+}
+
+fn confirmed_proxy_width(cfg: &xerj_common::config::EmbeddingConfig) -> Option<usize> {
+ let guard = cfg
+ .runtime_proxy_width
+ .read()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ guard.as_ref().and_then(|confirmed| {
+ (confirmed.endpoint == cfg.default_endpoint
+ && confirmed.model == cfg.default_model
+ && confirmed.dimensions > 0)
+ .then_some(confirmed.dimensions)
+ })
+}
+
+fn remember_proxy_width(cfg: &xerj_common::config::EmbeddingConfig, dimensions: usize) {
+ let mut guard = cfg
+ .runtime_proxy_width
+ .write()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ *guard = Some(xerj_common::config::ConfirmedProxyWidth {
+ endpoint: cfg.default_endpoint.clone(),
+ model: cfg.default_model.clone(),
+ dimensions,
+ });
+}
+
+fn proxy_mode_selected(cfg: &xerj_common::config::EmbeddingConfig) -> bool {
+ let requested = cfg.mode.trim().to_ascii_lowercase();
+ match requested.as_str() {
+ "proxy" => !cfg.default_endpoint.is_empty(),
+ "neural" | "lexical" | "onnx-experimental" => false,
+ _ => !cfg.default_endpoint.is_empty(),
+ }
+}
+
+#[derive(serde::Serialize, serde::Deserialize)]
+struct ProxyWidthCacheFile {
+ endpoint: String,
+ model: String,
+ dimensions: usize,
+}
+
+fn load_proxy_width_cache(
+ data_dir: &Path,
+ cfg: &xerj_common::config::EmbeddingConfig,
+) -> Option<usize> {
+ let bytes = std::fs::read(data_dir.join(PROXY_WIDTH_CACHE_FILE)).ok()?;
+ let cached: ProxyWidthCacheFile = serde_json::from_slice(&bytes).ok()?;
+ (cached.endpoint == cfg.default_endpoint
+ && cached.model == cfg.default_model
+ && cached.dimensions > 0)
+ .then_some(cached.dimensions)
+}
+
+fn store_proxy_width_cache(
+ data_dir: &Path,
+ cfg: &xerj_common::config::EmbeddingConfig,
+ dimensions: usize,
+) {
+ if let Err(error) = std::fs::create_dir_all(data_dir) {
+ warn!(
+ error = %error,
+ "could not create the data directory for the proxy width cache"
+ );
+ return;
+ }
+ let bytes = match serde_json::to_vec(&ProxyWidthCacheFile {
+ endpoint: cfg.default_endpoint.clone(),
+ model: cfg.default_model.clone(),
+ dimensions,
+ }) {
+ Ok(bytes) => bytes,
+ Err(error) => {
+ warn!(error = %error, "could not encode the proxy width cache");
+ return;
+ }
+ };
+ if let Err(error) = write_file_atomic(&data_dir.join(PROXY_WIDTH_CACHE_FILE), &bytes) {
+ warn!(error = %error, "could not store the confirmed proxy vector width");
+ }
+}
+
+fn probe_proxy_width(
+ cfg: &xerj_common::config::EmbeddingConfig,
+) -> std::result::Result<usize, String> {
+ let proxy =
+ xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)).map_err(|e| e.to_string())?;
+ let probed = proxy.embed(PROXY_WIDTH_PROBE_TEXT.to_string());
+ let vector = if let Ok(handle) = tokio::runtime::Handle::try_current() {
+ tokio::task::block_in_place(|| handle.block_on(probed))
+ } else {
+ let runtime = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .map_err(|e| e.to_string())?;
+ runtime.block_on(probed)
+ }
+ .map_err(|e| e.to_string())?;
+ if vector.is_empty() {
+ return Err("embedding endpoint returned an empty vector".to_string());
+ }
+ Ok(vector.len())
+}
+
+/// Confirm the proxy model's vector width once per process (#2306).
+///
+/// A successful probe is remembered in memory and under `data_dir`, and the
+/// identity that follows reports `resumable: true` with that width. A failed
+/// probe reuses the on-disk width for the same endpoint and model so a restart
+/// while the endpoint is down does not look like a different embedder. With
+/// neither, the identity stays non-resumable and names what to do.
+///
+/// No-op unless this config actually resolves to the proxy backend. Called from
+/// the identity request, not from index open.
+pub(crate) fn confirm_proxy_embedding_width(
+ cfg: &xerj_common::config::EmbeddingConfig,
+ data_dir: &Path,
+) {
+ if !proxy_mode_selected(cfg) || confirmed_proxy_width(cfg).is_some() {
+ return;
+ }
+ // An endpoint this process cannot even construct a client for is the
+ // identity function's error (explicit proxy) or its lexical fallback
+ // (auto). Probing it would only repeat that failure.
+ if xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)).is_err() {
+ return;
+ }
+ match probe_proxy_width(cfg) {
+ Ok(dimensions) => {
+ remember_proxy_width(cfg, dimensions);
+ store_proxy_width_cache(data_dir, cfg, dimensions);
+ }
+ Err(error) => {
+ if let Some(dimensions) = load_proxy_width_cache(data_dir, cfg) {
+ warn!(
+ error = %error,
+ dimensions,
+ "proxy width probe failed; reusing the last confirmed width for this \
+ endpoint and model"
+ );
+ remember_proxy_width(cfg, dimensions);
+ } else {
+ warn!(
+ error = %error,
+ "proxy embedding width could not be confirmed; semantic autoindex cannot \
+ resume until the endpoint answers a probe"
+ );
+ }
+ }
+ }
+}
+
pub(crate) fn embedding_execution_identity(
cfg: &xerj_common::config::EmbeddingConfig,
) -> Result<crate::engine::EmbeddingExecutionIdentity> {
@@ -39278,9 +39453,10 @@ pub(crate) fn embedding_execution_identity(
}
// Only report a width for the backends whose width this server actually
// pins. `neural` reads it from the resolved model's `config.json`
- // (`hidden_size`, the same number `NeuralEmbedder::dims` reports) and
- // `proxy` gets whatever the remote returns, so only the latter is omitted
- // rather than reported as 384.
+ // (`hidden_size`, the same number `NeuralEmbedder::dims` reports). An
+ // unconfirmed `proxy` omits the width rather than reporting 384; once a
+ // probe (or the on-disk fallback) has confirmed one, that width is
+ // reported (#2306). It is not part of the digest.
let (material, resumable, reason, dimensions) = match backend {
"lexical" => (
format!(
@@ -39314,23 +39490,34 @@ pub(crate) fn embedding_execution_identity(
)
}
"neural" => configured_neural_identity(cfg),
- "proxy" => (
+ "proxy" => {
// #533: the endpoint is part of the embedder identity — the same
// model name behind a different `default_endpoint` is a DIFFERENT
// embedder, and omitting it let the endpoint change silently past the
// #434 guard (a missed detection). `api_key` is deliberately left out
// (a rotated key over the same endpoint+model is the same embedder).
- format!(
+ //
+ // #2306: the digest stays that model+endpoint string. Folding the
+ // probed width into it would change the hash the moment a probe
+ // succeeds and refuse to open every index recorded before the probe
+ // (or recorded while the endpoint was down). The width travels in
+ // `dimensions` instead, and autoindex compares it on resume. A
+ // provider that swaps weights without changing the name, the
+ // endpoint, or the width is not detectable from here.
+ let material = format!(
"proxy-unpinned.v1;configured-model={};endpoint={}",
cfg.default_model, cfg.default_endpoint
- ),
- false,
- Some(
- "the remote provider does not attest immutable model assets; semantic resume is unsafe"
- .to_string(),
- ),
- None,
- ),
+ );
+ match confirmed_proxy_width(cfg) {
+ Some(width) => (material, true, None, Some(width)),
+ None => (
+ material,
+ false,
+ Some(proxy_width_unconfirmed_reason()),
+ None,
+ ),
+ }
+ }
_ => unreachable!(),
};
let canonical = format!("{CONTRACT};backend={backend};{material}");
@@ -39762,9 +39949,17 @@ mod embedding_identity_tests {
let identity = embedding_execution_identity(&proxy).unwrap();
assert_eq!(identity.backend, "proxy");
assert!(!identity.resumable);
- // A remote provider returns whatever width its model has; this server
- // never sees it, so it is omitted rather than asserted.
+ // No probe has confirmed a width, so it is omitted rather than asserted.
assert_eq!(identity.dimensions, None);
+ let reason = identity.non_resumable_reason.clone().unwrap();
+ assert!(
+ reason.contains("vector width") && reason.contains("reachable"),
+ "an unconfirmed proxy must say why resume is refused and how to retry: {reason}"
+ );
+ assert!(
+ !reason.contains("secret.example") && !reason.contains("private-alias"),
+ "the endpoint and model must not reach a remote client: {reason}"
+ );
let encoded = serde_json::to_string(&identity).unwrap();
assert!(!encoded.contains("secret.example"));
assert!(!encoded.contains("private-alias"));
@@ -39939,6 +40134,147 @@ mod embedding_identity_tests {
assert!(format!("{err}").contains("holds vectors produced by embedding backend"));
}
+ /// #2306: confirming a proxy width makes the backend resumable and publishes
+ /// that width. The digest stays the model+endpoint string, so a probe
+ /// cannot change the hash an existing index was recorded under.
+ #[test]
+ fn confirmed_proxy_width_makes_semantic_resume_possible() {
+ let cfg = xerj_common::config::EmbeddingConfig {
+ mode: "proxy".into(),
+ default_endpoint: "https://secret.example/v1".into(),
+ default_model: "private-alias".into(),
+ ..Default::default()
+ };
+ let unconfirmed = embedding_execution_identity(&cfg).unwrap();
+ assert!(!unconfirmed.resumable);
+ assert_eq!(unconfirmed.dimensions, None);
+
+ let confirmed_cfg = cfg.clone();
+ *confirmed_cfg.runtime_proxy_width.write().unwrap() =
+ Some(xerj_common::config::ConfirmedProxyWidth {
+ endpoint: confirmed_cfg.default_endpoint.clone(),
+ model: confirmed_cfg.default_model.clone(),
+ dimensions: 384,
+ });
+ let confirmed = embedding_execution_identity(&confirmed_cfg).unwrap();
+ assert!(confirmed.resumable);
+ assert_eq!(confirmed.dimensions, Some(384));
+ assert!(confirmed.non_resumable_reason.is_none());
+ assert_eq!(
+ confirmed.identity_sha256, unconfirmed.identity_sha256,
+ "the probed width must not move the digest"
+ );
+ let encoded = serde_json::to_string(&confirmed).unwrap();
+ assert!(!encoded.contains("secret.example"));
+ assert!(!encoded.contains("private-alias"));
+
+ *confirmed_cfg.runtime_proxy_width.write().unwrap() =
+ Some(xerj_common::config::ConfirmedProxyWidth {
+ endpoint: confirmed_cfg.default_endpoint.clone(),
+ model: confirmed_cfg.default_model.clone(),
+ dimensions: 768,
+ });
+ let wider = embedding_execution_identity(&confirmed_cfg).unwrap();
+ assert_eq!(wider.dimensions, Some(768));
+ assert_eq!(wider.identity_sha256, unconfirmed.identity_sha256);
+
+ // A cached width for a different endpoint is not this embedder.
+ *confirmed_cfg.runtime_proxy_width.write().unwrap() =
+ Some(xerj_common::config::ConfirmedProxyWidth {
+ endpoint: "https://other.example/v1".into(),
+ model: confirmed_cfg.default_model.clone(),
+ dimensions: 384,
+ });
+ let mismatched = embedding_execution_identity(&confirmed_cfg).unwrap();
+ assert!(!mismatched.resumable);
+ assert_eq!(mismatched.dimensions, None);
+ }
+
+ /// #2306: one successful probe confirms the width and stores it. A later
+ /// probe that cannot reach the endpoint reuses that store for the same
+ /// model. A store for a different model is ignored.
+ #[test]
+ fn proxy_width_probe_confirms_dimensions_and_falls_back_to_the_last_confirmation() {
+ let endpoint = serve_proxy_embeddings(8, 200);
+ let dir = tempfile::tempdir().unwrap();
+ let cfg = xerj_common::config::EmbeddingConfig {
+ mode: "proxy".into(),
+ default_endpoint: endpoint,
+ default_model: "mock-embed".into(),
+ timeout_ms: 2_000,
+ ..Default::default()
+ };
+ confirm_proxy_embedding_width(&cfg, dir.path());
+ let identity = embedding_execution_identity(&cfg).unwrap();
+ assert!(identity.resumable, "{identity:?}");
+ assert_eq!(identity.dimensions, Some(8));
+ assert!(dir.path().join(PROXY_WIDTH_CACHE_FILE).is_file());
+ let encoded = serde_json::to_string(&identity).unwrap();
+ assert!(!encoded.contains("127.0.0.1"));
+ assert!(!encoded.contains("mock-embed"));
+
+ let down = xerj_common::config::EmbeddingConfig {
+ mode: "proxy".into(),
+ default_endpoint: "http://127.0.0.1:1/v1/embeddings".into(),
+ default_model: "m".into(),
+ timeout_ms: 200,
+ ..Default::default()
+ };
+ let cache_dir = tempfile::tempdir().unwrap();
+ std::fs::write(
+ cache_dir.path().join(PROXY_WIDTH_CACHE_FILE),
+ serde_json::json!({
+ "endpoint": "http://127.0.0.1:1/v1/embeddings",
+ "model": "m",
+ "dimensions": 32
+ })
+ .to_string(),
+ )
+ .unwrap();
+ confirm_proxy_embedding_width(&down, cache_dir.path());
+ let reused = embedding_execution_identity(&down).unwrap();
+ assert!(reused.resumable, "{reused:?}");
+ assert_eq!(reused.dimensions, Some(32));
+ let reused_encoded = serde_json::to_string(&reused).unwrap();
+ assert!(!reused_encoded.contains("127.0.0.1"));
+ assert!(!reused_encoded.contains("\"m\""));
+
+ let other_model = xerj_common::config::EmbeddingConfig {
+ default_model: "other".into(),
+ ..down.clone()
+ };
+ confirm_proxy_embedding_width(&other_model, cache_dir.path());
+ let ignored = embedding_execution_identity(&other_model).unwrap();
+ assert!(!ignored.resumable);
+ assert_eq!(ignored.dimensions, None);
+ }
+
+ /// One HTTP embedding response, then the socket closes. `status` is 200 or 400.
+ fn serve_proxy_embeddings(width: usize, status: u16) -> String {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let addr = listener.local_addr().unwrap();
+ std::thread::spawn(move || {
+ let _ = listener.set_nonblocking(false);
+ if let Ok((mut sock, _)) = listener.accept() {
+ let mut buf = [0u8; 8192];
+ let _ = std::io::Read::read(&mut sock, &mut buf);
+ let embedding: Vec<f64> = (0..width).map(|i| (i as f64) * 0.01).collect();
+ let body = if status == 200 {
+ serde_json::json!({"data":[{"index":0,"embedding":embedding}]}).to_string()
+ } else {
+ "{\"error\":\"refused\"}".to_string()
+ };
+ let reason = if status == 200 { "OK" } else { "Bad Request" };
+ let resp = format!(
+ "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ );
+ let _ = std::io::Write::write_all(&mut sock, resp.as_bytes());
+ }
+ });
+ format!("http://{addr}/v1/embeddings")
+ }
+
/// #533 migration: a record persisted at version 1 (before the endpoint was
/// folded in) must be grandfathered — not refused — on upgrade, then
/// re-recorded at the current version so a LATER change IS detected.