AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): resume semantic runs that use a proxy embedder (#2307)
xerj-org47622fa19778
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 24ea8ad2d..3b7f6a59c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -59,6 +59,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Semantic `xerj autoindex` against `--embed-mode proxy` resumes after a+ mid-run failure and does not re-embed documents already indexed, when the+ backend, configured model, endpoint, and confirmed vector width are+ unchanged. The proxy digest is still the model and endpoint (#533); the+ width is learned with one probe (and remembered under the data directory)+ and compared on resume, so a successful probe does not change the hash an+ existing index was recorded under. A proxy whose endpoint has not confirmed+ a width stays non-resumable, and the refusal says to make the endpoint+ reachable and rerun — already-indexed documents are kept — or to rebuild+ with a new `--state-dir` and `--prefix` only to discard the corpus. A+ provider that swaps weights without changing the name, endpoint, or width+ is still not detectable+ ([#2306](https://github.com/xerj-org/xerj/issues/2306)).+- An existing index's persisted `index.default_pipeline` runs again after aclean restart. `GET /_settings` still reported the binding, and thepipeline definition was still there, but an ordinary document write nodiff --git a/engine/crates/xerj-api/tests/proxy_semantic_autoindex_resumes.rs b/engine/crates/xerj-api/tests/proxy_semantic_autoindex_resumes.rsnew file mode 100644index 000000000..d98034c3f--- /dev/null+++ b/engine/crates/xerj-api/tests/proxy_semantic_autoindex_resumes.rs@@ -0,0 +1,317 @@+//! Issue #2306: a semantic autoindex run whose embedder is `--embed-mode proxy`+//! must resume after a mid-run failure. Documents already embedded are not+//! sent to the proxy again when the backend, model, endpoint, and vector+//! width are unchanged.+//!+//! The mock answers OpenAI `/v1/embeddings` with 384-wide vectors (the width+//! `semantic_text` uses when the mapping omits `dimensions`). The identity+//! probe always succeeds. The first document embed succeeds and is indexed;+//! every later document embed is refused with HTTP 400 until the rerun turns+//! the refusal off. 400 is permanent, so the interrupted run fails without+//! retry backoff. Which file is first does not matter.++use std::path::Path;+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+use std::sync::{Arc, Mutex};+use std::time::Duration;++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;+use xerj_autoindex::cli::IndexCfg;++const DIMS: usize = 384;+const ALPHA: &str = "ALPHA_DOC_ONE";+const BETA: &str = "BETA_DOC_TWO";+/// Same text `confirm_proxy_embedding_width` sends. Kept here so the interrupt+/// does not treat the identity probe as a document.+const PROBE: &str = "xerj-proxy-identity-probe";++fn prose(marker: &str) -> String {+ format!(+ "{marker} The quarterly revenue report describes how the operating team closed the books \+ after a long review of customer contracts and deferred revenue. The narrative is ordinary \+ prose so the indexer treats the file as a document and embeds the body once, instead of \+ splitting it into line windows."+ )+}++struct EmbedLog {+ texts: Mutex<Vec<String>>,+ /// Document embed requests that may still succeed while `reject` is set.+ allow_document_requests: AtomicUsize,+ reject: AtomicBool,+}++async fn mock_embedder() -> (String, Arc<EmbedLog>) {+ let log = Arc::new(EmbedLog {+ texts: Mutex::new(Vec::new()),+ allow_document_requests: AtomicUsize::new(1),+ reject: AtomicBool::new(true),+ });+ let shared = Arc::clone(&log);+ let app = axum::Router::new().route(+ "/v1/embeddings",+ axum::routing::post(move |axum::Json(body): axum::Json<Value>| {+ let shared = Arc::clone(&shared);+ async move {+ let inputs: Vec<String> = match body.get("input") {+ Some(Value::Array(items)) => items+ .iter()+ .filter_map(|item| item.as_str().map(str::to_owned))+ .collect(),+ Some(Value::String(text)) => vec![text.clone()],+ _ => Vec::new(),+ };+ shared.texts.lock().unwrap().extend(inputs.iter().cloned());+ let document = inputs.iter().any(|text| !text.contains(PROBE));+ let allowed = !document+ || !shared.reject.load(Ordering::SeqCst)+ || shared+ .allow_document_requests+ .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| {+ left.checked_sub(1)+ })+ .is_ok();+ if !allowed {+ return (+ StatusCode::BAD_REQUEST,+ axum::Json(json!({"error": "interrupted"})),+ );+ }+ let data: Vec<Value> = inputs+ .iter()+ .enumerate()+ .map(|(index, _)| {+ let mut embedding = vec![0.0_f64; DIMS];+ embedding[0] = 1.0;+ json!({"index": index, "embedding": embedding})+ })+ .collect();+ (StatusCode::OK, axum::Json(json!({"data": data})))+ }+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind mock embedder");+ let addr = listener.local_addr().expect("mock embedder addr");+ tokio::spawn(async move {+ let _ = axum::serve(listener, app).await;+ });+ (format!("http://{addr}/v1/embeddings"), log)+}++async fn proxy_node(endpoint: String) -> (axum::Router, String) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.auth.enabled = false;+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ config.embedding.mode = "proxy".into();+ config.embedding.default_endpoint = endpoint;+ config.embedding.default_model = "mock-embed".into();+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ std::mem::forget(dir);+ let app = xerj_api::router::build_es_compat_router(state);+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind xerj");+ let addr = listener.local_addr().expect("xerj addr");+ let served = app.clone();+ tokio::spawn(async move {+ let _ = axum::serve(listener, served).await;+ });+ (app, format!("http://{addr}"))+}++async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+}++fn index_cfg(root: &Path, state_dir: &Path, url: &str, prefix: &str, no_graph: bool) -> IndexCfg {+ IndexCfg {+ root: root.to_owned(),+ endpoint_url: None,+ stub_globs: Vec::new(),+ url: url.to_owned(),+ api_key: None,+ api_key_file: None,+ workers: 1,+ scan_workers: 1,+ pdf_workers: 1,+ resource_notes: Vec::new(),+ xerj_url_note: None,+ pdf_timeout_secs: 30,+ bulk_mb: 1,+ bulk_timeout_secs: 30,+ snapshot_max_bytes: 64 << 30,+ prefix: prefix.to_owned(),+ replace_prefix: false,+ prefix_note: None,+ state_dir: Some(state_dir.to_owned()),+ fresh: false,+ follow_symlinks: false,+ follow_symlinks_outside_root: false,+ ignore: xerj_autoindex::ignore_rules::IgnoreOptions::default(),+ max_file_bytes: 1 << 30,+ sample: 50,+ no_semantic: false,+ code_analyzer: Default::default(),+ brain: Some(format!("brain-{prefix}")),+ no_graph,+ max_minutes: 0,+ approve: None,+ max_index_bytes: None,+ budget_plan: None,+ drop_indexed: false,+ dry_run: false,+ walk_only: false,+ json: false,+ quiet: true,+ progress: xerj_autoindex::progress::ProgressMode::None,+ progress_interval: None,+ watch: false,+ label: None,+ debounce: Duration::from_millis(0),+ exclude_rels: Vec::new(),+ skip_corpus_data: false,+ }+}++fn run_autoindex(+ root: &Path,+ state_dir: &Path,+ url: &str,+ prefix: &str,+ no_graph: bool,+) -> Result<i32, String> {+ let cfg = index_cfg(root, state_dir, url, prefix, no_graph);+ match xerj_autoindex::run_index_report(cfg) {+ Ok((code, _)) => Ok(code),+ Err(error) => Err(format!("{error:#}")),+ }+}++async fn run_autoindex_blocking(+ root: &Path,+ state_dir: &Path,+ url: &str,+ prefix: &str,+ no_graph: bool,+) -> Result<i32, String> {+ let root = root.to_owned();+ let state_dir = state_dir.to_owned();+ let url = url.to_owned();+ let prefix = prefix.to_owned();+ tokio::task::spawn_blocking(move || run_autoindex(&root, &state_dir, &url, &prefix, no_graph))+ .await+ .expect("autoindex task")+}++fn texts_containing(log: &[String], marker: &str) -> usize {+ log.iter().filter(|text| text.contains(marker)).count()+}++async fn indexed_markers(app: &axum::Router, prefix: &str) -> (bool, bool) {+ let (refresh_status, refresh_body) =+ call(app, "POST", &format!("/{prefix}-*/_refresh"), Value::Null).await;+ assert!(+ refresh_status.is_success(),+ "refresh {prefix}: {refresh_status} {refresh_body}"+ );+ let (status, body) = call(+ app,+ "POST",+ &format!("/{prefix}-*/_search"),+ json!({"size": 50, "query": {"match_all": {}}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "search {prefix}: {body}");+ let rendered = body.to_string();+ (rendered.contains(ALPHA), rendered.contains(BETA))+}++async fn interrupt_then_resume(no_graph: bool) {+ let (endpoint, log) = mock_embedder().await;+ let (app, url) = proxy_node(endpoint).await;+ let corpus = tempfile::tempdir().unwrap();+ let state = tempfile::tempdir().unwrap();+ std::fs::write(corpus.path().join("a-alpha.txt"), prose(ALPHA)).unwrap();+ std::fs::write(corpus.path().join("b-beta.txt"), prose(BETA)).unwrap();+ let prefix = if no_graph { "pxresumeng" } else { "pxresume" };++ let interrupted = run_autoindex_blocking(corpus.path(), state.path(), &url, prefix, no_graph)+ .await+ .expect_err("the refused document must stop the run");+ assert!(+ interrupted.contains("embedding") || interrupted.contains("bulk"),+ "the failure must name the embedding problem: {interrupted}"+ );++ let during = log.texts.lock().unwrap().clone();+ assert!(+ texts_containing(&during, ALPHA) >= 1 && texts_containing(&during, BETA) >= 1,+ "the run must embed one document and reach the next before it stops: {during:?}"+ );+ let (alpha_indexed, beta_indexed) = indexed_markers(&app, prefix).await;+ assert!(+ alpha_indexed != beta_indexed,+ "exactly one document is indexed before the refusal \+ (alpha={alpha_indexed} beta={beta_indexed})"+ );+ let already = if alpha_indexed { ALPHA } else { BETA };+ let pending = if alpha_indexed { BETA } else { ALPHA };++ let cutoff = during.len();+ log.reject.store(false, Ordering::SeqCst);+ let code = run_autoindex_blocking(corpus.path(), state.path(), &url, prefix, no_graph)+ .await+ .unwrap_or_else(|error| panic!("resume failed: {error}"));+ assert_eq!(code, 0, "resume must finish");++ let resumed = log.texts.lock().unwrap().clone();+ let again = &resumed[cutoff..];+ assert_eq!(+ texts_containing(again, already),+ 0,+ "already-indexed documents must not be re-embedded: {again:?}"+ );+ assert!(+ texts_containing(again, pending) >= 1,+ "the document that failed must be embedded on the rerun: {again:?}"+ );+ let (alpha_indexed, beta_indexed) = indexed_markers(&app, prefix).await;+ assert!(+ alpha_indexed && beta_indexed,+ "both documents are searchable after resume (alpha={alpha_indexed} beta={beta_indexed})"+ );+}++#[tokio::test]+async fn proxy_semantic_autoindex_resumes_without_reembedding() {+ interrupt_then_resume(false).await;+}++#[tokio::test]+async fn proxy_semantic_autoindex_resumes_without_reembedding_no_graph() {+ interrupt_then_resume(true).await;+}diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 6983f9642..52e6c85c9 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -1141,10 +1141,11 @@ pub struct EmbeddingExecutionIdentity {pub version: u32,pub backend: String,pub identity_sha256: String,- /// Absent for backends whose vector width the server does not pin- /// (`proxy`, and a `neural` node that could not resolve its model assets;- /// a resolved `neural` node reports its `config.json` `hidden_size` since- /// #487). An absent width must not be read as 384.+ /// Absent when the server has not pinned a vector width: a `proxy` whose+ /// endpoint has not confirmed one (#2306), and a `neural` node that could+ /// not resolve its model assets. A confirmed proxy reports the probed+ /// width, and a resolved `neural` node reports its `config.json`+ /// `hidden_size` (#487). An absent width must not be read as 384.#[serde(default, skip_serializing_if = "Option::is_none")]pub dimensions: Option<usize>,pub semantic_contract: String,diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 17bbe24b9..ef4668bf1 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -1798,6 +1798,7 @@ fn begin_non_graph_generation(journal.pin_embedding_identity(&identity.identity_sha256,identity.resumable,+ identity.dimensions,identity.non_resumable_reason.as_deref(),)?;identity@@ -5080,13 +5081,15 @@ fn pending_for_phase_b(.collect()}-/// #487: an unchanged re-index on a `neural`/`proxy` backend must not fail with-/// a false "embedding execution identity changed" message. The identity CHANGED-/// only if a comparable field differs from what the journal pinned; `resumable`-/// is a standing property of the backend (false by construction for backends-/// whose identity hashes the model NAME, not its bytes), so it is checked-/// SEPARATELY with an accurate message — restoring the identity cannot fix a-/// backend that simply cannot resume.+/// An unchanged re-index must not fail with a false "embedding execution+/// identity changed" message (#487). The identity changed only when a+/// comparable field differs from what the journal pinned. `resumable` is+/// checked on its own.+///+/// A proxy whose vector width is not confirmed reports `dimensions: None` and+/// `resumable: false` (#2306). That missing width is the resume limit, not a+/// different embedder. A confirmed width that disagrees with the pinned width+/// is a real change and must not be mixed into the corpus.fn ensure_embedding_execution_unchanged_and_resumable(current: &crate::esclient::EmbeddingExecutionIdentity,expected_sha: &str,@@ -5094,11 +5097,22 @@ fn ensure_embedding_execution_unchanged_and_resumable(expected_dimension: Option<usize>,expected_semantic_contract: &str,) -> anyhow::Result<()> {+ let same_embedder = current.identity_sha256 == expected_sha+ && current.backend == expected_backend+ && current.semantic_contract == expected_semantic_contract;+ let same_width = current.dimensions == expected_dimension;+ if same_embedder && !same_width && !(current.dimensions.is_none() && !current.resumable) {+ anyhow::bail!(+ "embedding execution identity changed since this autoindex journal was created \+ (vector width {} vs {}); refusing to mix vector spaces. No remote mutation was \+ attempted. Restore the original identity, or rebuild with a new --state-dir and a \+ new --prefix",+ dimension_label(expected_dimension),+ dimension_label(current.dimensions),+ );+ }anyhow::ensure!(- current.identity_sha256 == expected_sha- && current.backend == expected_backend- && current.dimensions == expected_dimension- && current.semantic_contract == expected_semantic_contract,+ same_embedder && (same_width || (current.dimensions.is_none() && !current.resumable)),"embedding execution identity changed since this autoindex journal was created; \refusing to mix vector spaces. No remote mutation was attempted. Restore the \original identity, or rebuild with a new --state-dir and a new --prefix"@@ -5107,8 +5121,10 @@ fn ensure_embedding_execution_unchanged_and_resumable(current.resumable,"the `{}` embedding backend cannot resume semantic indexing{}. The pinned identity is \UNCHANGED — this is a backend limitation, not a change, so restoring the identity will \- not help. Rebuild with a new --state-dir and a new --prefix, or content-address the \- model identity so this backend can resume (#367).",+ not help. Follow the reason above. Documents already indexed were kept. If the vector \+ width was not confirmed, make the embedding endpoint reachable and rerun — \+ already-embedded documents are reused once the same width is confirmed. Rebuild with a \+ new --state-dir and a new --prefix only to discard this corpus and start over.",current.backend,current.non_resumable_reason@@ -5119,6 +5135,13 @@ fn ensure_embedding_execution_unchanged_and_resumable(Ok(())}+fn dimension_label(width: Option<usize>) -> String {+ match width {+ Some(width) => width.to_string(),+ None => "unreported".to_string(),+ }+}+fn select_resume_plan_keys(files: &[walk::FileEntry],content_keys: &[String],@@ -5967,6 +5990,7 @@ fn pin_pending_embedding_identity(journal.pin_embedding_identity(¤t.identity_sha256,current.resumable,+ current.dimensions,current.non_resumable_reason.as_deref(),)}@@ -8883,6 +8907,7 @@ fn run_index_report_inner(journal.pin_embedding_identity(&identity.identity_sha256,identity.resumable,+ identity.dimensions,identity.non_resumable_reason.as_deref(),)?;// #1189: name the live backend in the run output. A server started@@ -8893,6 +8918,15 @@ fn run_index_report_inner(identity.backend,if identity.resumable { "" } else { "non-" }));+ if !identity.resumable {+ pr.note(&format!(+ "semantic autoindex cannot resume this embedding backend: {}",+ identity+ .non_resumable_reason+ .as_deref()+ .unwrap_or("the server did not provide an immutable embedding identity")+ ));+ }}// ── create indices with explicit mappings ────────────────────────────@@ -12782,6 +12816,47 @@ mod duplicate_integration_tests {err2.contains("identity changed"),"a real identity change must still report it: {err2}");+ // A confirmed width that disagrees with the pinned width is a real+ // change. An unconfirmed width on a non-resumable backend is not.+ let wider = crate::esclient::EmbeddingExecutionIdentity {+ dimensions: Some(768),+ resumable: true,+ non_resumable_reason: None,+ ..current.clone()+ };+ let err3 = ensure_embedding_execution_unchanged_and_resumable(+ &wider,+ "6e51e5ce3e46",+ "neural",+ None,+ "semantic_text-derived-vector.v1",+ )+ .unwrap_err()+ .to_string();+ assert!(+ err3.contains("identity changed") && err3.contains("768"),+ "a different vector width must be reported as a change: {err3}"+ );+ let unconfirmed = crate::esclient::EmbeddingExecutionIdentity {+ dimensions: None,+ resumable: false,+ non_resumable_reason: Some("vector width was not confirmed".into()),+ ..current+ };+ let err4 = ensure_embedding_execution_unchanged_and_resumable(+ &unconfirmed,+ "6e51e5ce3e46",+ "neural",+ Some(384),+ "semantic_text-derived-vector.v1",+ )+ .unwrap_err()+ .to_string();+ assert!(+ err4.contains("cannot resume") && !err4.contains("identity changed"),+ "an unconfirmed width is a resume limit, not a changed identity: {err4}"+ );+ assert!(err4.contains("vector width was not confirmed"), "{err4}");}#[test]diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rsindex b9f308834..4e132190d 100644--- a/engine/crates/xerj-autoindex/src/state.rs+++ b/engine/crates/xerj-autoindex/src/state.rs@@ -283,16 +283,16 @@ mod tests {let dir = tempfile::tempdir().unwrap();let mut first = Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap();first- .pin_embedding_identity(&"a".repeat(64), true, None)+ .pin_embedding_identity(&"a".repeat(64), true, None, None).unwrap();drop(first);let mut resumed = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();resumed- .pin_embedding_identity(&"a".repeat(64), true, None)+ .pin_embedding_identity(&"a".repeat(64), true, None, None).unwrap();let error = resumed- .pin_embedding_identity(&"b".repeat(64), true, None)+ .pin_embedding_identity(&"b".repeat(64), true, None, None).unwrap_err().to_string();assert!(error.contains("refusing to mix vector spaces"), "{error}");@@ -307,12 +307,12 @@ mod tests {let dir = tempfile::tempdir().unwrap();let mut journal = Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap();journal- .pin_embedding_identity(&"a".repeat(64), false, Some("remote alias can drift"))+ .pin_embedding_identity(&"a".repeat(64), false, None, Some("remote alias can drift")).unwrap();drop(journal);let mut resumed = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();let error = resumed- .pin_embedding_identity(&"a".repeat(64), false, Some("remote alias can drift"))+ .pin_embedding_identity(&"a".repeat(64), false, None, Some("remote alias can drift")).unwrap_err().to_string();assert!(error.contains("remote alias can drift"), "{error}");@@ -321,6 +321,62 @@ mod tests {assert!(error.contains("delete and recreate"), "{error}");}+ /// #2306: a journal pinned while the proxy width was still unconfirmed+ /// (`resumable: false`, no width) resumes once the same digest is reported+ /// as resumable. A later width that disagrees with a pinned width does not.+ #[test]+ fn confirmed_width_resumes_a_journal_pinned_before_confirmation() {+ let dir = tempfile::tempdir().unwrap();+ let digest = "a".repeat(64);+ let mut journal = Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap();+ journal+ .pin_embedding_identity(&digest, false, None, Some("width was not confirmed"))+ .unwrap();+ drop(journal);++ let mut resumed = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();+ resumed+ .pin_embedding_identity(&digest, true, Some(384), None)+ .expect("same digest becomes resumable once the width is confirmed");+ // The pre-confirmation record did not carry a width, and resume does+ // not append a second identity line (a flipped resumable flag would+ // fail replay). A width is therefore not yet pinned on this journal.+ assert_eq!(resumed.embedding_identity_dimensions, None);+ }++ #[test]+ fn a_pinned_width_is_durable_and_a_missing_width_is_not_drift() {+ let dir = tempfile::tempdir().unwrap();+ let digest = "c".repeat(64);+ let mut first = Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap();+ first+ .pin_embedding_identity(&digest, true, Some(384), None)+ .unwrap();+ drop(first);+ let text = std::fs::read_to_string(dir.path().join("journal.ndjson")).unwrap();+ assert!(text.contains("\"dimensions\":384"), "{text}");++ let mut resumed = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();+ assert_eq!(resumed.embedding_identity_dimensions, Some(384));+ let lost = resumed+ .pin_embedding_identity(&digest, true, None, None)+ .unwrap_err()+ .to_string();+ assert!(lost.contains("no longer reported"), "{lost}");+ assert!(lost.contains("--fresh"), "{lost}");+ let drifted = resumed+ .pin_embedding_identity(&digest, true, Some(768), None)+ .unwrap_err()+ .to_string();+ assert!(+ drifted.contains("refusing to mix vector spaces"),+ "{drifted}"+ );+ assert!(drifted.contains("384"), "{drifted}");+ assert!(drifted.contains("768"), "{drifted}");+ assert!(drifted.contains("--prefix"), "{drifted}");+ }+#[test]fn completed_legacy_semantic_journal_requires_fresh_rebuild() {let dir = tempfile::tempdir().unwrap();@@ -336,7 +392,7 @@ mod tests {.unwrap();let mut journal = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();let error = journal- .pin_embedding_identity(&"a".repeat(64), true, None)+ .pin_embedding_identity(&"a".repeat(64), true, None, None).unwrap_err().to_string();assert!(@@ -352,6 +408,11 @@ mod tests {fn malformed_or_conflicting_identity_records_fail_during_replay() {for tail in ["{\"v\":1,\"kind\":\"embedding_identity\",\"identity_sha256\":\"bad\",\"resumable\":true}\n",+ concat!(+ "{\"v\":1,\"kind\":\"embedding_identity\",\"identity_sha256\":\"",+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "\",\"resumable\":true,\"dimensions\":0}\n"+ ),concat!("{\"v\":1,\"kind\":\"embedding_identity\",\"identity_sha256\":\"","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",@@ -688,6 +749,11 @@ pub struct Journal {pub plan: Option<Plan>,pub embedding_identity_sha256: Option<String>,pub embedding_identity_resumable: Option<bool>,+ /// Vector width pinned with the identity, when the record carried one.+ ///+ /// Older journals omit it. A resume that now reports a width is not drift+ /// for that reason alone (#2306); two recorded widths that disagree are.+ pub embedding_identity_dimensions: Option<usize>,/// Newest corpus reconciliation transaction without a durable/// `sync_commit`. Its desired plan is deliberately not authoritative yet.pub committed_manifest: Option<crate::sync::CommittedManifest>,@@ -1081,6 +1147,19 @@ pub struct JournalPreflight {pub no_graph_genesis: bool,}+fn embedding_resume_refused(reason: &str) -> String {+ format!(+ "server embedding backend cannot safely resume semantic indexing: {reason}. \+ No remote mutation was attempted. Documents already indexed were kept. \+ If the vector width was not confirmed, make the embedding endpoint reachable \+ and rerun the same command — already-embedded documents are reused once the \+ same width is confirmed. Restore the exact original embedding identity, or \+ rebuild all vectors with --fresh and a new --prefix only when the backend, \+ model, endpoint, or vector width changed. Before reusing the old prefix, \+ delete and recreate its prior autoindex indices"+ )+}+impl Journal {/// `fresh` is not a formality here. `open_after_preflight` removes the/// journal before replaying it when `--fresh` is set, so nothing inside a@@ -1216,6 +1295,10 @@ impl Journal {let mut plan = None;let mut embedding_identity_sha256 = None;let mut embedding_identity_resumable = None;+ // `None` = no identity record yet. `Some(None)` = a record that omitted+ // the width. `Some(Some(n))` = a recorded width. Absence and a width+ // in the same journal disagree; two different widths disagree.+ let mut embedding_identity_dimensions_seen: Option<Option<usize>> = None;let mut committed_manifest: Option<crate::sync::CommittedManifest> = None;let mut pending_sync: Option<crate::sync::PendingSync> = None;let mut legacy_migration_reasons = Vec::new();@@ -1307,15 +1390,42 @@ impl Journal {})});let resumable = v.get("resumable").and_then(Value::as_bool);+ let has_dimensions = v+ .as_object()+ .is_some_and(|object| object.contains_key("dimensions"));+ let keys_ok = v.as_object().is_some_and(|object| {+ let base = ["v", "kind", "identity_sha256", "resumable"]+ .iter()+ .all(|key| object.contains_key(*key));+ if has_dimensions {+ object.len() == 5 && base+ } else {+ object.len() == 4 && base+ }+ });+ let recorded_width = if has_dimensions {+ match v.get("dimensions").and_then(Value::as_u64) {+ Some(width)+ if width > 0 && usize::try_from(width).is_ok() =>+ {+ Some(width as usize)+ }+ _ => {+ anyhow::bail!(+ "journal at {} contains a malformed embedding \+ identity; restore it from backup or re-run with \+ --fresh",+ jpath.display()+ );+ }+ }+ } else {+ None+ };if v.get("v").and_then(Value::as_u64) != Some(1)|| digest.is_none()|| resumable.is_none()- || v.as_object().is_none_or(|object| {- object.len() != 4- || !["v", "kind", "identity_sha256", "resumable"]- .iter()- .all(|key| object.contains_key(*key))- })+ || !keys_ok{anyhow::bail!("journal at {} contains a malformed embedding identity; \@@ -1328,6 +1438,8 @@ impl Journal {.is_some_and(|existing| existing != digest.unwrap())|| embedding_identity_resumable.is_some_and(|existing| existing != resumable.unwrap())+ || embedding_identity_dimensions_seen+ .is_some_and(|existing| existing != recorded_width){anyhow::bail!("journal at {} contains conflicting embedding identities; \@@ -1337,6 +1449,7 @@ impl Journal {}embedding_identity_sha256 = digest.map(str::to_owned);embedding_identity_resumable = resumable;+ embedding_identity_dimensions_seen = Some(recorded_width);}Some(kind) if crate::sync::is_sync_record_kind(kind) => {// A durable record that no longer re-validates@@ -1450,6 +1563,7 @@ impl Journal {plan,embedding_identity_sha256,embedding_identity_resumable,+ embedding_identity_dimensions: embedding_identity_dimensions_seen.flatten(),committed_manifest,pending_sync,legacy_migration_reasons,@@ -1477,10 +1591,15 @@ impl Journal {}/// Pin the server-side vector-space identity before any semantic write.+ ///+ /// `dimensions` is the confirmed vector width when the server has one.+ /// The key is omitted from the journal record when it is absent, so a+ /// journal written before widths were pinned still replays.pub fn pin_embedding_identity(&mut self,identity_sha256: &str,resumable: bool,+ dimensions: Option<usize>,non_resumable_reason: Option<&str>,) -> Result<()> {if identity_sha256.len() != 64@@ -1490,15 +1609,6 @@ impl Journal {{anyhow::bail!("server returned an invalid embedding identity digest");}- if self.resumed && (!resumable || self.embedding_identity_resumable == Some(false)) {- anyhow::bail!(- "server embedding backend cannot safely resume semantic indexing: {}. \- Restore the exact original embedding identity, or rebuild all vectors with \- --fresh and a new --prefix. Before reusing the old prefix, delete and recreate \- its prior autoindex indices",- non_resumable_reason.unwrap_or("embedding identity is not immutable")- );- }if let Some(existing) = &self.embedding_identity_sha256 {if existing != identity_sha256 {anyhow::bail!(@@ -1508,8 +1618,53 @@ impl Journal {reusing the old prefix, delete and recreate its prior autoindex indices");}+ match (self.embedding_identity_dimensions, dimensions) {+ (Some(stored), Some(current)) if stored != current => {+ anyhow::bail!(+ "embedding execution identity changed since this autoindex journal was \+ created (vector width {stored} vs {current}); refusing to mix vector \+ spaces. Restore the original embedding identity, or rebuild all vectors \+ with --fresh and a new --prefix. Before reusing the old prefix, delete \+ and recreate its prior autoindex indices"+ );+ }+ (Some(stored), None) => {+ anyhow::bail!(+ "embedding execution identity changed since this autoindex journal was \+ created (vector width {stored} is no longer reported); refusing to mix \+ vector spaces. Restore the original embedding identity, or rebuild all \+ vectors with --fresh and a new --prefix. Before reusing the old prefix, \+ delete and recreate its prior autoindex indices"+ );+ }+ _ => {}+ }+ if !resumable {+ // Same digest, still not safe to reuse vectors. A proxy whose+ // width was never confirmed lands here; the server's reason+ // says how to get a resume instead of a rebuild.+ anyhow::bail!(+ "{}",+ embedding_resume_refused(+ non_resumable_reason.unwrap_or("embedding identity is not immutable")+ )+ );+ }+ // Same digest, and the server can resume. Do not append another+ // identity record: replay treats a flipped `resumable` flag as+ // corruption. That is what lets a journal pinned before a proxy+ // width could be confirmed (#2306) continue once this process+ // confirms the same model and endpoint.return Ok(());}+ if self.resumed && !resumable {+ anyhow::bail!(+ "{}",+ embedding_resume_refused(+ non_resumable_reason.unwrap_or("embedding identity is not immutable")+ )+ );+ }if self.resumed && !self.done.is_empty() {anyhow::bail!("this semantic autoindex journal predates embedding identity pinning and cannot \@@ -1517,17 +1672,19 @@ impl Journal {reusing the old prefix, delete and recreate its prior autoindex indices");}- self.append_transaction(- &serde_json::json!({- "v": 1,- "kind": "embedding_identity",- "identity_sha256": identity_sha256,- "resumable": resumable,- }),- "embedding_identity",- )?;+ let mut record = serde_json::json!({+ "v": 1,+ "kind": "embedding_identity",+ "identity_sha256": identity_sha256,+ "resumable": resumable,+ });+ if let Some(width) = dimensions {+ record["dimensions"] = serde_json::json!(width);+ }+ self.append_transaction(&record, "embedding_identity")?;self.embedding_identity_sha256 = Some(identity_sha256.to_owned());self.embedding_identity_resumable = Some(resumable);+ self.embedding_identity_dimensions = dimensions;Ok(())}diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rsindex d549314d3..4cfcb02aa 100644--- a/engine/crates/xerj-common/src/config.rs+++ b/engine/crates/xerj-common/src/config.rs@@ -1671,6 +1671,15 @@ pub struct EmbeddingConfig {#[serde(skip)]pub runtime_onnx_assets:std::sync::Arc<std::sync::OnceLock<Result<EmbeddingAssetSnapshot, String>>>,+ /// Vector width last confirmed for this proxy endpoint and model (#2306).+ ///+ /// Filled by the engine, not by configuration. Cloned configs share the+ /// cell. Empty until a probe (or the on-disk fallback) succeeds; identity+ /// reporting then treats the proxy as resumable and publishes the width.+ /// Serde skips it: it is not a setting and must not round-trip through a+ /// config file.+ #[serde(skip)]+ pub runtime_proxy_width: std::sync::Arc<std::sync::RwLock<Option<ConfirmedProxyWidth>>>,/// Neural ingest probe, shared by cloned configs (the server preloads/// from a clone). `None` inside the cell means the probe ran and did not/// produce a rate; an empty cell means it has not run. Not a setting.@@ -1678,6 +1687,18 @@ pub struct EmbeddingConfig {pub neural_ingest_probe: std::sync::Arc<std::sync::OnceLock<Option<NeuralIngestProbe>>>,}+/// Vector width a proxy embedder confirmed for one endpoint and model.+///+/// The digest of a proxy identity stays the configured model and endpoint+/// (#533). This width is the other half of that identity: autoindex compares+/// it on resume and refuses to mix a different width into an existing corpus.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct ConfirmedProxyWidth {+ pub endpoint: String,+ pub model: String,+ pub dimensions: usize,+}+#[derive(Debug, Clone)]pub struct EmbeddingAssetSnapshot {pub model_bytes: std::sync::Arc<[u8]>,@@ -1710,6 +1731,7 @@ impl Default for EmbeddingConfig {onnx_max_input_bytes_per_call: 8 * 1024 * 1024,onnx_max_inflight_input_bytes: 32 * 1024 * 1024,runtime_onnx_assets: std::sync::Arc::new(std::sync::OnceLock::new()),+ runtime_proxy_width: std::sync::Arc::new(std::sync::RwLock::new(None)),neural_ingest_probe: std::sync::Arc::new(std::sync::OnceLock::new()),}}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 506488586..fb8694083 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -176,8 +176,9 @@ pub struct EmbeddingExecutionIdentity {/// while content-addressing it (#487) — the same number/// `NeuralEmbedder::dims` reports, so it is known without loading the/// model. It stays `None` for a `neural` node whose assets cannot be- /// resolved at all, and for `proxy`, whose width is whatever the remote- /// returns. A reader must not treat an absent width as 384.+ /// resolved at all, and for a `proxy` whose endpoint has not confirmed a+ /// width yet (#2306). A confirmed proxy reports the probed width. A reader+ /// must not treat an absent width as 384.#[serde(default, skip_serializing_if = "Option::is_none")]pub dimensions: Option<usize>,pub semantic_contract: String,@@ -938,6 +939,9 @@ impl Engine {// clone lineage. This ensures a later Engine re-reads same-path ONNX// assets while all Index config clones within one Engine share bytes.config.embedding.runtime_onnx_assets = Arc::new(std::sync::OnceLock::new());+ // Same ownership rule for the proxy width cache: this engine probes+ // into its own cell, and every index cloned from `config` shares it.+ config.embedding.runtime_proxy_width = Arc::new(std::sync::RwLock::new(None));let data_dir = PathBuf::from(&config.server.data_dir);std::fs::create_dir_all(&data_dir)?;@@ -4233,6 +4237,11 @@ impl Engine {/// Return the effective embedding identity without exposing configuration/// secrets or local asset paths.pub fn embedding_execution_identity(&self) -> Result<EmbeddingExecutionIdentity> {+ // #2306: a proxy resume needs a confirmed vector width. The probe runs+ // here, on the identity request autoindex makes before it writes, and+ // not on index open — open compares the model+endpoint digest, which+ // does not wait on the endpoint.+ crate::index::confirm_proxy_embedding_width(&self.config.embedding, &self.data_dir);crate::index::embedding_execution_identity(&self.config.embedding)}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 7ca0d1535..f9b1d8d2a 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -39231,6 +39231,181 @@ fn configured_neural_identity(unreachable!("the neural backend is not compiled into this build")}+/// Probe text sent once per process to learn a proxy model's vector width.+///+/// Kept distinct from document text so a resume test can tell the probe apart+/// from a re-embedded document.+const PROXY_WIDTH_PROBE_TEXT: &str = "xerj-proxy-identity-probe";++/// On-disk copy of the last confirmed proxy width, under the data directory.+///+/// A restart whose endpoint is briefly down reuses this instead of reporting+/// a different identity and refusing to resume. It is not an index directory+/// (`Engine::new` only opens subdirectories that contain a `wal`).+const PROXY_WIDTH_CACHE_FILE: &str = "proxy-embedding-probe.json";++fn proxy_width_unconfirmed_reason() -> String {+ "the proxy embedding endpoint has not confirmed a vector width, so semantic \+ resume cannot reuse already-embedded documents. The embedder identity is the \+ backend, the configured model, the endpoint, and the vector width; without a \+ confirmed width a rerun cannot tell this model from a different one. Make the \+ embedding endpoint reachable and rerun — documents already indexed are kept and \+ are not re-embedded once the same width is confirmed. Rebuild with a new \+ --state-dir and a new --prefix only to discard this corpus and start over"+ .to_string()+}++fn confirmed_proxy_width(cfg: &xerj_common::config::EmbeddingConfig) -> Option<usize> {+ let guard = cfg+ .runtime_proxy_width+ .read()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ guard.as_ref().and_then(|confirmed| {+ (confirmed.endpoint == cfg.default_endpoint+ && confirmed.model == cfg.default_model+ && confirmed.dimensions > 0)+ .then_some(confirmed.dimensions)+ })+}++fn remember_proxy_width(cfg: &xerj_common::config::EmbeddingConfig, dimensions: usize) {+ let mut guard = cfg+ .runtime_proxy_width+ .write()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ *guard = Some(xerj_common::config::ConfirmedProxyWidth {+ endpoint: cfg.default_endpoint.clone(),+ model: cfg.default_model.clone(),+ dimensions,+ });+}++fn proxy_mode_selected(cfg: &xerj_common::config::EmbeddingConfig) -> bool {+ let requested = cfg.mode.trim().to_ascii_lowercase();+ match requested.as_str() {+ "proxy" => !cfg.default_endpoint.is_empty(),+ "neural" | "lexical" | "onnx-experimental" => false,+ _ => !cfg.default_endpoint.is_empty(),+ }+}++#[derive(serde::Serialize, serde::Deserialize)]+struct ProxyWidthCacheFile {+ endpoint: String,+ model: String,+ dimensions: usize,+}++fn load_proxy_width_cache(+ data_dir: &Path,+ cfg: &xerj_common::config::EmbeddingConfig,+) -> Option<usize> {+ let bytes = std::fs::read(data_dir.join(PROXY_WIDTH_CACHE_FILE)).ok()?;+ let cached: ProxyWidthCacheFile = serde_json::from_slice(&bytes).ok()?;+ (cached.endpoint == cfg.default_endpoint+ && cached.model == cfg.default_model+ && cached.dimensions > 0)+ .then_some(cached.dimensions)+}++fn store_proxy_width_cache(+ data_dir: &Path,+ cfg: &xerj_common::config::EmbeddingConfig,+ dimensions: usize,+) {+ if let Err(error) = std::fs::create_dir_all(data_dir) {+ warn!(+ error = %error,+ "could not create the data directory for the proxy width cache"+ );+ return;+ }+ let bytes = match serde_json::to_vec(&ProxyWidthCacheFile {+ endpoint: cfg.default_endpoint.clone(),+ model: cfg.default_model.clone(),+ dimensions,+ }) {+ Ok(bytes) => bytes,+ Err(error) => {+ warn!(error = %error, "could not encode the proxy width cache");+ return;+ }+ };+ if let Err(error) = write_file_atomic(&data_dir.join(PROXY_WIDTH_CACHE_FILE), &bytes) {+ warn!(error = %error, "could not store the confirmed proxy vector width");+ }+}++fn probe_proxy_width(+ cfg: &xerj_common::config::EmbeddingConfig,+) -> std::result::Result<usize, String> {+ let proxy =+ xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)).map_err(|e| e.to_string())?;+ let probed = proxy.embed(PROXY_WIDTH_PROBE_TEXT.to_string());+ let vector = if let Ok(handle) = tokio::runtime::Handle::try_current() {+ tokio::task::block_in_place(|| handle.block_on(probed))+ } else {+ let runtime = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .map_err(|e| e.to_string())?;+ runtime.block_on(probed)+ }+ .map_err(|e| e.to_string())?;+ if vector.is_empty() {+ return Err("embedding endpoint returned an empty vector".to_string());+ }+ Ok(vector.len())+}++/// Confirm the proxy model's vector width once per process (#2306).+///+/// A successful probe is remembered in memory and under `data_dir`, and the+/// identity that follows reports `resumable: true` with that width. A failed+/// probe reuses the on-disk width for the same endpoint and model so a restart+/// while the endpoint is down does not look like a different embedder. With+/// neither, the identity stays non-resumable and names what to do.+///+/// No-op unless this config actually resolves to the proxy backend. Called from+/// the identity request, not from index open.+pub(crate) fn confirm_proxy_embedding_width(+ cfg: &xerj_common::config::EmbeddingConfig,+ data_dir: &Path,+) {+ if !proxy_mode_selected(cfg) || confirmed_proxy_width(cfg).is_some() {+ return;+ }+ // An endpoint this process cannot even construct a client for is the+ // identity function's error (explicit proxy) or its lexical fallback+ // (auto). Probing it would only repeat that failure.+ if xerj_ai::embed::EmbeddingProxy::new(proxy_config(cfg)).is_err() {+ return;+ }+ match probe_proxy_width(cfg) {+ Ok(dimensions) => {+ remember_proxy_width(cfg, dimensions);+ store_proxy_width_cache(data_dir, cfg, dimensions);+ }+ Err(error) => {+ if let Some(dimensions) = load_proxy_width_cache(data_dir, cfg) {+ warn!(+ error = %error,+ dimensions,+ "proxy width probe failed; reusing the last confirmed width for this \+ endpoint and model"+ );+ remember_proxy_width(cfg, dimensions);+ } else {+ warn!(+ error = %error,+ "proxy embedding width could not be confirmed; semantic autoindex cannot \+ resume until the endpoint answers a probe"+ );+ }+ }+ }+}+pub(crate) fn embedding_execution_identity(cfg: &xerj_common::config::EmbeddingConfig,) -> Result<crate::engine::EmbeddingExecutionIdentity> {@@ -39278,9 +39453,10 @@ pub(crate) fn embedding_execution_identity(}// Only report a width for the backends whose width this server actually// pins. `neural` reads it from the resolved model's `config.json`- // (`hidden_size`, the same number `NeuralEmbedder::dims` reports) and- // `proxy` gets whatever the remote returns, so only the latter is omitted- // rather than reported as 384.+ // (`hidden_size`, the same number `NeuralEmbedder::dims` reports). An+ // unconfirmed `proxy` omits the width rather than reporting 384; once a+ // probe (or the on-disk fallback) has confirmed one, that width is+ // reported (#2306). It is not part of the digest.let (material, resumable, reason, dimensions) = match backend {"lexical" => (format!(@@ -39314,23 +39490,34 @@ pub(crate) fn embedding_execution_identity()}"neural" => configured_neural_identity(cfg),- "proxy" => (+ "proxy" => {// #533: the endpoint is part of the embedder identity — the same// model name behind a different `default_endpoint` is a DIFFERENT// embedder, and omitting it let the endpoint change silently past the// #434 guard (a missed detection). `api_key` is deliberately left out// (a rotated key over the same endpoint+model is the same embedder).- format!(+ //+ // #2306: the digest stays that model+endpoint string. Folding the+ // probed width into it would change the hash the moment a probe+ // succeeds and refuse to open every index recorded before the probe+ // (or recorded while the endpoint was down). The width travels in+ // `dimensions` instead, and autoindex compares it on resume. A+ // provider that swaps weights without changing the name, the+ // endpoint, or the width is not detectable from here.+ let material = format!("proxy-unpinned.v1;configured-model={};endpoint={}",cfg.default_model, cfg.default_endpoint- ),- false,- Some(- "the remote provider does not attest immutable model assets; semantic resume is unsafe"- .to_string(),- ),- None,- ),+ );+ match confirmed_proxy_width(cfg) {+ Some(width) => (material, true, None, Some(width)),+ None => (+ material,+ false,+ Some(proxy_width_unconfirmed_reason()),+ None,+ ),+ }+ }_ => unreachable!(),};let canonical = format!("{CONTRACT};backend={backend};{material}");@@ -39762,9 +39949,17 @@ mod embedding_identity_tests {let identity = embedding_execution_identity(&proxy).unwrap();assert_eq!(identity.backend, "proxy");assert!(!identity.resumable);- // A remote provider returns whatever width its model has; this server- // never sees it, so it is omitted rather than asserted.+ // No probe has confirmed a width, so it is omitted rather than asserted.assert_eq!(identity.dimensions, None);+ let reason = identity.non_resumable_reason.clone().unwrap();+ assert!(+ reason.contains("vector width") && reason.contains("reachable"),+ "an unconfirmed proxy must say why resume is refused and how to retry: {reason}"+ );+ assert!(+ !reason.contains("secret.example") && !reason.contains("private-alias"),+ "the endpoint and model must not reach a remote client: {reason}"+ );let encoded = serde_json::to_string(&identity).unwrap();assert!(!encoded.contains("secret.example"));assert!(!encoded.contains("private-alias"));@@ -39939,6 +40134,147 @@ mod embedding_identity_tests {assert!(format!("{err}").contains("holds vectors produced by embedding backend"));}+ /// #2306: confirming a proxy width makes the backend resumable and publishes+ /// that width. The digest stays the model+endpoint string, so a probe+ /// cannot change the hash an existing index was recorded under.+ #[test]+ fn confirmed_proxy_width_makes_semantic_resume_possible() {+ let cfg = xerj_common::config::EmbeddingConfig {+ mode: "proxy".into(),+ default_endpoint: "https://secret.example/v1".into(),+ default_model: "private-alias".into(),+ ..Default::default()+ };+ let unconfirmed = embedding_execution_identity(&cfg).unwrap();+ assert!(!unconfirmed.resumable);+ assert_eq!(unconfirmed.dimensions, None);++ let confirmed_cfg = cfg.clone();+ *confirmed_cfg.runtime_proxy_width.write().unwrap() =+ Some(xerj_common::config::ConfirmedProxyWidth {+ endpoint: confirmed_cfg.default_endpoint.clone(),+ model: confirmed_cfg.default_model.clone(),+ dimensions: 384,+ });+ let confirmed = embedding_execution_identity(&confirmed_cfg).unwrap();+ assert!(confirmed.resumable);+ assert_eq!(confirmed.dimensions, Some(384));+ assert!(confirmed.non_resumable_reason.is_none());+ assert_eq!(+ confirmed.identity_sha256, unconfirmed.identity_sha256,+ "the probed width must not move the digest"+ );+ let encoded = serde_json::to_string(&confirmed).unwrap();+ assert!(!encoded.contains("secret.example"));+ assert!(!encoded.contains("private-alias"));++ *confirmed_cfg.runtime_proxy_width.write().unwrap() =+ Some(xerj_common::config::ConfirmedProxyWidth {+ endpoint: confirmed_cfg.default_endpoint.clone(),+ model: confirmed_cfg.default_model.clone(),+ dimensions: 768,+ });+ let wider = embedding_execution_identity(&confirmed_cfg).unwrap();+ assert_eq!(wider.dimensions, Some(768));+ assert_eq!(wider.identity_sha256, unconfirmed.identity_sha256);++ // A cached width for a different endpoint is not this embedder.+ *confirmed_cfg.runtime_proxy_width.write().unwrap() =+ Some(xerj_common::config::ConfirmedProxyWidth {+ endpoint: "https://other.example/v1".into(),+ model: confirmed_cfg.default_model.clone(),+ dimensions: 384,+ });+ let mismatched = embedding_execution_identity(&confirmed_cfg).unwrap();+ assert!(!mismatched.resumable);+ assert_eq!(mismatched.dimensions, None);+ }++ /// #2306: one successful probe confirms the width and stores it. A later+ /// probe that cannot reach the endpoint reuses that store for the same+ /// model. A store for a different model is ignored.+ #[test]+ fn proxy_width_probe_confirms_dimensions_and_falls_back_to_the_last_confirmation() {+ let endpoint = serve_proxy_embeddings(8, 200);+ let dir = tempfile::tempdir().unwrap();+ let cfg = xerj_common::config::EmbeddingConfig {+ mode: "proxy".into(),+ default_endpoint: endpoint,+ default_model: "mock-embed".into(),+ timeout_ms: 2_000,+ ..Default::default()+ };+ confirm_proxy_embedding_width(&cfg, dir.path());+ let identity = embedding_execution_identity(&cfg).unwrap();+ assert!(identity.resumable, "{identity:?}");+ assert_eq!(identity.dimensions, Some(8));+ assert!(dir.path().join(PROXY_WIDTH_CACHE_FILE).is_file());+ let encoded = serde_json::to_string(&identity).unwrap();+ assert!(!encoded.contains("127.0.0.1"));+ assert!(!encoded.contains("mock-embed"));++ let down = xerj_common::config::EmbeddingConfig {+ mode: "proxy".into(),+ default_endpoint: "http://127.0.0.1:1/v1/embeddings".into(),+ default_model: "m".into(),+ timeout_ms: 200,+ ..Default::default()+ };+ let cache_dir = tempfile::tempdir().unwrap();+ std::fs::write(+ cache_dir.path().join(PROXY_WIDTH_CACHE_FILE),+ serde_json::json!({+ "endpoint": "http://127.0.0.1:1/v1/embeddings",+ "model": "m",+ "dimensions": 32+ })+ .to_string(),+ )+ .unwrap();+ confirm_proxy_embedding_width(&down, cache_dir.path());+ let reused = embedding_execution_identity(&down).unwrap();+ assert!(reused.resumable, "{reused:?}");+ assert_eq!(reused.dimensions, Some(32));+ let reused_encoded = serde_json::to_string(&reused).unwrap();+ assert!(!reused_encoded.contains("127.0.0.1"));+ assert!(!reused_encoded.contains("\"m\""));++ let other_model = xerj_common::config::EmbeddingConfig {+ default_model: "other".into(),+ ..down.clone()+ };+ confirm_proxy_embedding_width(&other_model, cache_dir.path());+ let ignored = embedding_execution_identity(&other_model).unwrap();+ assert!(!ignored.resumable);+ assert_eq!(ignored.dimensions, None);+ }++ /// One HTTP embedding response, then the socket closes. `status` is 200 or 400.+ fn serve_proxy_embeddings(width: usize, status: u16) -> String {+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let addr = listener.local_addr().unwrap();+ std::thread::spawn(move || {+ let _ = listener.set_nonblocking(false);+ if let Ok((mut sock, _)) = listener.accept() {+ let mut buf = [0u8; 8192];+ let _ = std::io::Read::read(&mut sock, &mut buf);+ let embedding: Vec<f64> = (0..width).map(|i| (i as f64) * 0.01).collect();+ let body = if status == 200 {+ serde_json::json!({"data":[{"index":0,"embedding":embedding}]}).to_string()+ } else {+ "{\"error\":\"refused\"}".to_string()+ };+ let reason = if status == 200 { "OK" } else { "Bad Request" };+ let resp = format!(+ "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len()+ );+ let _ = std::io::Write::write_all(&mut sock, resp.as_bytes());+ }+ });+ format!("http://{addr}/v1/embeddings")+ }+/// #533 migration: a record persisted at version 1 (before the endpoint was/// folded in) must be grandfathered — not refused — on upgrade, then/// re-recorded at the current version so a LATER change IS detected.