Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(embedding_host): delegate embedding verification to tinyinference

Steven Enamakel099471bd4dd8
diff --git a/crates/openhuman-core/src/inference/embedding_host/rpc/embed.rs b/crates/openhuman-core/src/inference/embedding_host/rpc/embed.rs
index 143ebc2447..b76cc7cb56 100644
--- a/crates/openhuman-core/src/inference/embedding_host/rpc/embed.rs
+++ b/crates/openhuman-core/src/inference/embedding_host/rpc/embed.rs
@@ -4,8 +4,10 @@ use crate::config::Config;
use crate::core::Outcome;
use super::{resolve_api_key, LOG_PREFIX};
-use crate::inference::embedding_host::factory::create_embedding_provider_with_config;
-use tinyinference_embeddings::probe::probe_custom_embeddings;
+use crate::inference::embedding_host::factory::{
+ credential_resolver, managed_factory, settings, create_embedding_provider_with_config,
+};
+use tinyinference_embeddings::verify::test_embedding_connection;
use tinyinference_providers::route::split_custom_provider;
/// Generates embeddings for the given input texts using the currently
@@ -73,8 +75,6 @@ pub async fn test_connection(
let model = model.unwrap_or(&config.memory.embedding_model);
let dims = dims.unwrap_or(config.memory.embedding_dimensions);
- let api_key = resolve_api_key(config, slug);
-
let (provider_slug, custom_endpoint) = split_custom_provider(slug);
let provider_tag = if custom_endpoint.is_some() {
@@ -90,25 +90,29 @@ pub async fn test_connection(
"{LOG_PREFIX} test_connection starting"
);
- let result = if let Some(endpoint) = custom_endpoint.as_deref() {
- probe_custom_embeddings(endpoint, &api_key, model, dims)
- .await
- .map_err(|error| error.to_string())
- } else {
- let embedder = create_embedding_provider_with_config(
- config,
- provider_tag,
- model,
- dims,
- &api_key,
- None,
- )
- .map_err(|e| e.to_string())?;
- embedder
- .embed(&["connection test".to_string()])
- .await
- .map_err(|e| e.to_string())
- };
+ let selected = settings(
+ if custom_endpoint.is_some() {
+ &format!("custom:{}", custom_endpoint.as_deref().unwrap_or_default())
+ } else {
+ provider_tag
+ },
+ model,
+ dims,
+ tinyinference_local::ollama::ollama_base_url_from_override(
+ config.local_ai.base_url.as_deref(),
+ ),
+ );
+ let credentials = credential_resolver(config);
+ let managed = managed_factory(Some(config.clone()), None);
+ let result = test_embedding_connection(
+ &selected,
+ &credentials,
+ &managed,
+ custom_endpoint.as_deref(),
+ )
+ .await
+ .map(|result| result.actual_dims)
+ .map_err(|error| error.to_string());
match result {
Ok(vectors) => {
diff --git a/crates/openhuman-core/src/inference/embedding_host/rpc/settings.rs b/crates/openhuman-core/src/inference/embedding_host/rpc/settings.rs
index f50d8d45b4..3021d621e6 100644
--- a/crates/openhuman-core/src/inference/embedding_host/rpc/settings.rs
+++ b/crates/openhuman-core/src/inference/embedding_host/rpc/settings.rs
@@ -200,143 +200,53 @@ pub async fn update_settings(
}
_ => new_provider.clone(),
};
- // `new_dims`/`new_sig` are recomputed after the Custom
- // verification probe auto-detects the endpoint's real vector length
- // (issue #4056), so they must be mutable.
+ // Custom verification can auto-detect the endpoint's vector width.
let mut new_dims = dimensions.unwrap_or(config.memory.embedding_dimensions);
let mut new_sig = format_embedding_signature(&effective_provider, &new_model, new_dims);
let mut sig_changed = new_sig != old_sig;
- // Setup-time verification gate (TAURI-RUST-5JR / 4P4): a Custom
- // (OpenAI-compatible) embeddings endpoint — e.g. LM Studio — must prove it
- // can actually embed *before* we accept it. We run one live test embed and
- // only persist the config if it succeeds; any failure (no `/embeddings`
- // route, no model loaded, timeout, 5xx, empty/zero-dim vector) rejects the
- // save so a config that can't embed is never stored (and we never wipe
- // memory for one). Verifying at setup is the fix — we deliberately do NOT
- // try to classify-and-suppress the resulting embed flood in code; any
- // residual flood (e.g. the user unloads the model *after* a good save) is
- // handled on the Sentry side.
- //
// Only custom endpoints are probed: named catalog providers are
- // embedding-capable by construction, and probing `managed`/`cloud`
- // pre-login would false-fail. Resolve the provider string exactly as it
- // will be stored so the probe targets the real endpoint.
+ // embedding-capable by construction, and probing managed/cloud before
+ // login would false-fail. TinyInference owns endpoint verification and
+ // its structured rejection policy.
if let (_, Some(endpoint)) = split_custom_provider(&effective_provider) {
- // Probe dimension-agnostically for non-`text-embedding-3-*` models so the
- // user's guessed `dimensions` can't fail an otherwise-valid endpoint; the
- // real length is detected from the returned vector below (issue #4056).
let api_key = resolve_api_key(&config, "custom");
- let probe = tokio::time::timeout(
- std::time::Duration::from_secs(10),
- probe_custom_embeddings(&endpoint, &api_key, &new_model, new_dims),
- )
- .await;
- {
- // Time-box the probe so a black-hole host can't hang the RPC.
- tracing::debug!(
- provider = effective_provider.as_str(),
- "{LOG_PREFIX} update_settings verifying embeddings endpoint with a test embed"
- );
- // Normalize the timeout/result into one shape, then apply the
- // pure verification policy (`classify_embed_probe`, unit-tested).
- let outcome = match probe {
- Ok(Ok(vectors)) => EmbedProbe::Returned(vectors),
- Ok(Err(e)) => EmbedProbe::Failed(e.to_string()),
- Err(_elapsed) => EmbedProbe::TimedOut,
- };
- // Peek the actual vector length before the policy consumes the
- // outcome — on a pass this is the endpoint's real dimension.
- let probe_actual_dims = match &outcome {
- EmbedProbe::Returned(vectors) => vectors.first().map(|v| v.len()).unwrap_or(0),
- _ => 0,
- };
- if let Some(reject) = classify_embed_probe(outcome) {
- // Log the classified error code (never the raw detail — it can
- // carry endpoint response bodies) so support can distinguish
- // auth vs wrong-model vs unreachable failures (issue #5017).
- let reject_code = reject.error;
- tracing::warn!(
- provider = effective_provider.as_str(),
- reject_code,
- "{LOG_PREFIX} update_settings rejected — embeddings endpoint failed verification"
- );
- // Right-feedback (issue #3761): the probe failed. If the
- // endpoint lists its served models and the requested id
- // isn't among them, the cause is almost certainly a name
- // mismatch (e.g. the user entered `bge-m3` but LM Studio
- // serves `text-embedding-bge-m3`). Replace the generic
- // failure with an actionable message naming the available
- // models and the suggested match. Best-effort and only on
- // the failure path, so a passing config is never blocked by
- // an endpoint that doesn't expose `/models`. Derive the
- // endpoint from the payload OR the already-stored
- // `custom:<url>` provider, so a model-only update to an
- // existing custom endpoint still gets the guidance.
- let (_, stored_endpoint) = split_custom_provider(&effective_provider);
- let listed_endpoint = custom_endpoint.as_deref().or(stored_endpoint.as_deref());
- if let Some(ep) = listed_endpoint {
- let api_key = resolve_api_key(&config, "custom");
- tracing::debug!(
+ tracing::debug!(
+ provider = effective_provider.as_str(),
+ "{LOG_PREFIX} update_settings verifying embeddings endpoint with a test embed"
+ );
+ match verify_custom_endpoint(&endpoint, &api_key, &new_model, new_dims).await {
+ Ok(verified) => {
+ if verified.dimensions != new_dims {
+ tracing::info!(
provider = effective_provider.as_str(),
- requested = new_model.as_str(),
- "{LOG_PREFIX} update_settings: probing endpoint /models for served-id guidance"
+ model = new_model.as_str(),
+ requested = new_dims,
+ detected = verified.dimensions,
+ "{LOG_PREFIX} update_settings auto-detected custom embedding dimension from probe"
);
- match fetch_served_model_ids(ep, &api_key).await {
- Ok(served) => match check_requested_model_served(&new_model, &served) {
- Some(better) => {
- tracing::warn!(
- provider = effective_provider.as_str(),
- requested = new_model.as_str(),
- served = served.len(),
- "{LOG_PREFIX} update_settings: model not in served list — returning name-mismatch guidance"
- );
- return Ok(model_rejection_outcome(better));
- }
- None => {
- tracing::debug!(
- provider = effective_provider.as_str(),
- served = served.len(),
- "{LOG_PREFIX} update_settings: requested model is served (or list empty) — keeping generic verification error"
- );
- }
- },
- Err(e) => {
- tracing::debug!(
- provider = effective_provider.as_str(),
- error = %e,
- "{LOG_PREFIX} update_settings: /models lookup failed — keeping generic verification error"
- );
- }
- }
}
- return Ok(probe_rejection_outcome(reject));
+ new_dims = verified.dimensions;
+ new_sig = verified.signature;
+ sig_changed = new_sig != old_sig;
+ tracing::debug!(
+ provider = effective_provider.as_str(),
+ new_dims,
+ "{LOG_PREFIX} update_settings test embed passed — accepting config"
+ );
}
- // Passed. Adopt the endpoint's real vector length for every model
- // we probed dimension-agnostically — the user can't be expected to
- // know it, and storing the actual size is what keeps the live embed
- // path's length guard from rejecting future embeds (issue #4056).
- // Models with configurable output widths were probed with the
- // requested size and rejected if the server ignored it.
- let detected_dims = final_probe_dims(&new_model, new_dims, probe_actual_dims);
- if detected_dims != new_dims {
- tracing::info!(
+ Err(rejection) => {
+ tracing::warn!(
provider = effective_provider.as_str(),
- model = new_model.as_str(),
- requested = new_dims,
- detected = detected_dims,
- "{LOG_PREFIX} update_settings auto-detected custom embedding dimension from probe"
+ reject_code = rejection.error(),
+ "{LOG_PREFIX} update_settings rejected — embeddings endpoint failed verification"
);
- new_dims = detected_dims;
- new_sig = format_embedding_signature(&effective_provider, &new_model, new_dims);
- sig_changed = new_sig != old_sig;
+ return Ok(match rejection {
+ VerifyRejection::Probe(reject) => probe_rejection_outcome(reject),
+ VerifyRejection::ModelNotServed(reject) => model_rejection_outcome(*reject),
+ });
}
- tracing::debug!(
- provider = effective_provider.as_str(),
- new_dims,
- "{LOG_PREFIX} update_settings test embed passed — accepting config"
- );
}
}