Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(tinyhumans): delegate managed embedding to CloudEmbeddingModel

Steven Enamakeldb5ad7cc6e5a
diff --git a/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs b/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs
index 4dd8abdbd7..c9436752a1 100644
--- a/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs
+++ b/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs
@@ -1,141 +1,75 @@
-//! Managed cloud embeddings: `POST /openai/v1/embeddings` on the hosted backend
-//! with the session JWT or stored API key, wrapped in the core's egress guard.
-//! Registered as the embedding factory for the managed provider id.
+//! Host policy for TinyInference's managed cloud embedding model.
+//!
+//! This module constructs the owning `CloudEmbeddingModel` with OpenHuman's
+//! live credential resolver and the per-request egress disclosure guard. It
+//! does not wrap or forward the embedding implementation.
use std::sync::Arc;
use openhuman_embed::__host::egress::{emit_external_transfer, enforce_egress, EgressDescriptor};
use openhuman_embed::__host::inference_registry::EmbeddingRequest;
use openhuman_embed::__host::security::approval::ApprovalChatContext;
-use tinyinference_embeddings::{
- BearerResolver, CloudEmbeddingModel, EmbeddingEgressGuard, EmbeddingModel,
-};
+use tinyinference_embeddings::{BearerResolver, CloudEmbeddingModel, EmbeddingEgressGuard};
use super::managed_session::resolve_session_bearer;
use crate::backend::url::effective_api_url;
use crate::client::endpoint::is_managed_endpoint_for_api_key;
-/// Host-owned credential resolution and egress disclosure around the
-/// crate-owned cloud transport.
-///
-/// The egress guard (policy check, then the `ExternalTransferPending`
-/// disclosure) runs before the batch reaches the transport, so it can carry the
-/// chat of the turn that made the call. One built without a chat (background
-/// sync, the embedding RPC) still discloses, unrouted.
-pub struct ManagedEmbeddingModel {
- inner: CloudEmbeddingModel,
- base_url: String,
- model: String,
- dimensions: usize,
- bearer: BearerResolver,
- chat: Option<ApprovalChatContext>,
+/// Builds the owning cloud embedding implementation with the credential scope
+/// and turn context supplied by the inference registry.
+pub(super) fn from_request(request: &EmbeddingRequest<'_>) -> CloudEmbeddingModel {
+ build_cloud_model(
+ request.config.and_then(|config| config.api_url.clone()),
+ request.state_dir.clone(),
+ request.secrets_encrypt,
+ request.model,
+ request.dimensions,
+ request.chat.clone(),
+ )
}
-impl ManagedEmbeddingModel {
- /// Builds the embedder for the credential scope `request` carries.
- pub fn from_request(request: &EmbeddingRequest<'_>) -> Self {
- Self::new(
- request.config.and_then(|config| config.api_url.clone()),
- request.state_dir.clone(),
- request.secrets_encrypt,
- request.model,
- request.dimensions,
- request.chat.clone(),
+fn build_cloud_model(
+ api_url: Option<String>,
+ state_dir: std::path::PathBuf,
+ secrets_encrypt: bool,
+ model: &str,
+ dimensions: usize,
+ chat: Option<ApprovalChatContext>,
+) -> CloudEmbeddingModel {
+ let base_url = format!(
+ "{}/openai/v1",
+ effective_api_url(&api_url).trim_end_matches('/')
+ );
+ let key_endpoint = base_url.clone();
+ let bearer: BearerResolver = Arc::new(move || {
+ // A stored TinyHumans API key is the bearer outright, with the
+ // plaintext guard: never put the key on a non-HTTPS, non-loopback wire.
+ if let Some(key) = openhuman_embed::__host::security::credentials::api_key::get_api_key_in(
+ &state_dir,
+ secrets_encrypt,
)
- }
-
- /// Builds the embedder against `api_url` (or the default backend) with
- /// credentials read from `state_dir`.
- pub fn new(
- api_url: Option<String>,
- state_dir: std::path::PathBuf,
- secrets_encrypt: bool,
- model: &str,
- dimensions: usize,
- chat: Option<ApprovalChatContext>,
- ) -> Self {
- let base_url = format!(
- "{}/openai/v1",
- effective_api_url(&api_url).trim_end_matches('/')
- );
- let key_endpoint = base_url.clone();
- let bearer: BearerResolver = Arc::new(move || {
- // A stored TinyHumans API key is the bearer outright, with the
- // plaintext guard: never put the key on a non-HTTPS, non-loopback wire.
- if let Some(key) =
- openhuman_embed::__host::security::credentials::api_key::get_api_key_in(
- &state_dir,
- secrets_encrypt,
- )
- .map_err(|error| tinyinference_embeddings::Error::Embedding(error.to_string()))?
- {
- if !is_managed_endpoint_for_api_key(&key_endpoint) {
- return Err(tinyinference_embeddings::Error::Validation(format!(
- "refusing to send the TinyHumans API key over a non-HTTPS, non-loopback \
- endpoint: {key_endpoint}"
- )));
- }
- log::debug!("[tinyhumans][embeddings] authenticating with api-key");
- return Ok(key);
+ .map_err(|error| tinyinference_embeddings::Error::Embedding(error.to_string()))?
+ {
+ if !is_managed_endpoint_for_api_key(&key_endpoint) {
+ return Err(tinyinference_embeddings::Error::Validation(format!(
+ "refusing to send the TinyHumans API key over a non-HTTPS, non-loopback \
+ endpoint: {key_endpoint}"
+ )));
}
- resolve_session_bearer(
- &state_dir,
- secrets_encrypt,
- None,
- "managed_embedding.resolve_bearer",
- )
- .map_err(|error| tinyinference_embeddings::Error::Embedding(error.to_string()))
- });
- let model = model.to_string();
- let base_url = base_url;
- Self {
- inner: cloud_model(
- &base_url,
- &model,
- dimensions,
- Arc::clone(&bearer),
- chat.as_ref(),
- ),
- base_url,
- model,
- dimensions,
- bearer,
- chat,
- }
- }
-
- /// This embedder with its egress disclosures routed to `chat` (the turn's
- /// thread and client); `None` leaves them unrouted. Shares the transport,
- /// so it is cheap to do per turn.
- #[cfg(test)]
- #[must_use]
- pub fn routed_to(&self, chat: Option<ApprovalChatContext>) -> Self {
- Self {
- inner: cloud_model(
- &self.base_url,
- &self.model,
- self.dimensions,
- Arc::clone(&self.bearer),
- chat.as_ref(),
- ),
- base_url: self.base_url.clone(),
- model: self.model.clone(),
- dimensions: self.dimensions,
- bearer: Arc::clone(&self.bearer),
- chat,
+ log::debug!("[tinyhumans][embeddings] authenticating with api-key");
+ return Ok(key);
}
- }
-}
+ resolve_session_bearer(
+ &state_dir,
+ secrets_encrypt,
+ None,
+ "managed_embedding.resolve_bearer",
+ )
+ .map_err(|error| tinyinference_embeddings::Error::Embedding(error.to_string()))
+ });
-fn cloud_model(
- base_url: &str,
- model: &str,
- dimensions: usize,
- bearer: BearerResolver,
- chat: Option<&ApprovalChatContext>,
-) -> CloudEmbeddingModel {
let guarded_model = model.to_owned();
- let guarded_chat = chat.cloned();
+ let guarded_chat = chat;
let egress_guard: EmbeddingEgressGuard = Arc::new(move |_, _| {
let egress = EgressDescriptor::embedding("cloud", &guarded_model);
enforce_egress(&egress)
@@ -143,33 +77,9 @@ fn cloud_model(
emit_external_transfer(guarded_chat.as_ref(), egress);
Ok(())
});
- CloudEmbeddingModel::new(base_url.to_owned(), model.to_owned(), dimensions, bearer)
- .with_egress_guard(egress_guard)
-}
-#[async_trait::async_trait]
-impl EmbeddingModel for ManagedEmbeddingModel {
- fn name(&self) -> &str {
- self.inner.name()
- }
-
- fn model_id(&self) -> &str {
- self.inner.model_id()
- }
-
- fn dimensions(&self) -> usize {
- self.inner.dimensions()
- }
-
- fn signature(&self) -> String {
- self.inner.signature()
- }
-
- async fn embed(&self, texts: &[String]) -> tinyinference_embeddings::Result<Vec<Vec<f32>>> {
- // The owning transport invokes the host guard after input validation
- // and immediately before resolving credentials or sending a request.
- self.inner.embed(texts).await
- }
+ CloudEmbeddingModel::new(base_url, model.to_owned(), dimensions, bearer)
+ .with_egress_guard(egress_guard)
}
#[cfg(test)]
diff --git a/crates/openhuman-tinyhumans/src/inference/managed_embedding_tests.rs b/crates/openhuman-tinyhumans/src/inference/managed_embedding_tests.rs
index a4a13fa688..7af524a617 100644
--- a/crates/openhuman-tinyhumans/src/inference/managed_embedding_tests.rs
+++ b/crates/openhuman-tinyhumans/src/inference/managed_embedding_tests.rs
@@ -2,7 +2,7 @@ use super::*;
use openhuman_embed::__host::config::Config;
use openhuman_embed::__host::security::credentials::AuthService;
use std::collections::HashMap;
-use tinyinference_embeddings::{DEFAULT_CLOUD_DIMENSIONS, DEFAULT_CLOUD_MODEL};
+use tinyinference_embeddings::{CloudEmbeddingModel, DEFAULT_CLOUD_DIMENSIONS, DEFAULT_CLOUD_MODEL};
fn test_config(tmp: &tempfile::TempDir) -> Config {
Config {
@@ -25,8 +25,8 @@ fn build_through_registry(
config: &Config,
model: &str,
dimensions: usize,
-) -> ManagedEmbeddingModel {
- ManagedEmbeddingModel::from_request(&EmbeddingRequest {
+) -> CloudEmbeddingModel {
+ from_request(&EmbeddingRequest {
config: Some(config),
model,
dimensions,
@@ -72,7 +72,7 @@ async fn api_key_alone_authenticates_cloud_embeddings() {
"tiny_test_emb",
)
.unwrap();
- let provider = ManagedEmbeddingModel::new(
+ let provider = build_cloud_model(
Some(format!("http://{addr}")),
state.path().to_path_buf(),
false,
@@ -102,7 +102,7 @@ async fn api_key_is_refused_over_plaintext_non_loopback() {
"tiny_test_emb",
)
.unwrap();
- let provider = ManagedEmbeddingModel::new(
+ let provider = build_cloud_model(
Some("http://embeddings.example.test".into()),
state.path().to_path_buf(),
false,
@@ -138,8 +138,8 @@ async fn find_embedding_disclosure(
}
}
-fn unreachable_embedder(model: &str) -> ManagedEmbeddingModel {
- ManagedEmbeddingModel::new(
+fn unreachable_embedder(model: &str) -> CloudEmbeddingModel {
+ build_cloud_model(
Some("http://127.0.0.1:0".into()),
std::env::temp_dir().join("openhuman_embeddings_routing_state"),
false,
@@ -168,7 +168,7 @@ async fn a_cloud_embedding_call_in_a_chat_turn_discloses_to_that_chat() {
client_id: "client-embed".into(),
request_id: Some("req-embed".into()),
};
- let embedder = unreachable_embedder(marker).routed_to(Some(chat));
+ let embedder = unreachable_embedder_with_chat(marker, Some(chat));
// The transport has nowhere to go; the disclosure precedes it.
let _ = embedder
@@ -215,8 +215,7 @@ async fn an_unrouted_embedder_discloses_without_a_chat() {
);
}
-/// `routed_to` re-routes a shared embedder per turn without touching the
-/// original: each turn's disclosure reaches its own chat.
+/// Each embedding model carries the approval context for its own turn.
#[tokio::test]
async fn routing_is_per_embedder_not_shared() {
openhuman_embed::__host::core::bus::init()
@@ -227,15 +226,13 @@ async fn routing_is_per_embedder_not_shared() {
.unwrap()
.receiver();
let marker = "embed-routed-per-turn";
- let base = unreachable_embedder(marker);
let chat = |thread: &str| ApprovalChatContext {
thread_id: thread.into(),
client_id: "c".into(),
request_id: None,
};
- let _ = base
- .routed_to(Some(chat("thread-a")))
+ let _ = unreachable_embedder_with_chat(marker, Some(chat("thread-a")))
.embed(&["a".into()])
.await;
assert_eq!(
@@ -245,8 +242,7 @@ async fn routing_is_per_embedder_not_shared() {
.as_deref(),
Some("thread-a")
);
- let _ = base
- .routed_to(Some(chat("thread-b")))
+ let _ = unreachable_embedder_with_chat(marker, Some(chat("thread-b")))
.embed(&["b".into()])
.await;
assert_eq!(
@@ -256,7 +252,7 @@ async fn routing_is_per_embedder_not_shared() {
.as_deref(),
Some("thread-b")
);
- let _ = base.embed(&["c".into()]).await;
+ let _ = unreachable_embedder(marker).embed(&["c".into()]).await;
assert_eq!(find_embedding_disclosure(&mut rx, marker).await.0, None);
}
diff --git a/crates/openhuman-tinyhumans/src/inference/managed_model.rs b/crates/openhuman-tinyhumans/src/inference/managed_model.rs
index 0f443c2159..4278982cd0 100644
--- a/crates/openhuman-tinyhumans/src/inference/managed_model.rs
+++ b/crates/openhuman-tinyhumans/src/inference/managed_model.rs
@@ -5,8 +5,6 @@
//! supplies OpenHuman's live credential resolver and client policy.
use std::path::PathBuf;
-use std::time::Duration;
-
use tinyinference_llm::providers::tinyhumans::{TinyHumansFailurePoint, TinyHumansModel};
use super::managed_session::{publish_managed_auth_failure, resolve_session_bearer};
@@ -90,7 +88,6 @@ pub(super) fn build_chat_model(
.with_cache_namespace("openhuman")
.with_native_tool_calling(native_tool_calling)
.with_thread_id(thread_id)
- .with_http_client(managed_inference_http_client())
.with_error_observer(log_managed_error);
if observe_auth_failures {
@@ -130,32 +127,3 @@ fn log_managed_error(error: &tinyinference_llm::Error, point: TinyHumansFailureP
#[path = "managed_model_tests.rs"]
mod tests;
-/// Connect timeout for managed inference; matches the previous host adapter.
-const MANAGED_INFERENCE_CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
-/// Keep pooled connections below the backend ALB's idle timeout.
-const MANAGED_INFERENCE_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(50);
-
-/// Shared HTTP client retains TLS connections across managed turns.
-fn managed_inference_http_client() -> reqwest::Client {
- static CLIENT: std::sync::OnceLock<reqwest::Client> = std::sync::OnceLock::new();
- CLIENT
- .get_or_init(|| {
- tracing::debug!(
- idle_timeout_secs = MANAGED_INFERENCE_POOL_IDLE_TIMEOUT.as_secs(),
- "[inference][managed] building shared pooled HTTP client"
- );
- reqwest::Client::builder()
- .connect_timeout(MANAGED_INFERENCE_CONNECT_TIMEOUT)
- .pool_idle_timeout(MANAGED_INFERENCE_POOL_IDLE_TIMEOUT)
- .tcp_keepalive(Duration::from_secs(30))
- .build()
- .unwrap_or_else(|error| {
- tracing::warn!(
- error = %error,
- "[inference][managed] pooled client build failed; using reqwest defaults"
- );
- reqwest::Client::new()
- })
- })
- .clone()
-}
diff --git a/crates/openhuman-tinyhumans/src/inference/mod.rs b/crates/openhuman-tinyhumans/src/inference/mod.rs
index 5176a38e76..4a8928b73f 100644
--- a/crates/openhuman-tinyhumans/src/inference/mod.rs
+++ b/crates/openhuman-tinyhumans/src/inference/mod.rs
@@ -7,7 +7,7 @@
//! | Registry | Registered value |
//! | --- | --- |
//! | chat models | TinyInference's [`tinyinference_llm::providers::tinyhumans::TinyHumansModel`] with host credentials, plus its readiness probe |
-//! | embeddings | [`managed_embedding::ManagedEmbeddingModel`] |
+//! | embeddings | TinyInference's `CloudEmbeddingModel` with OpenHuman's credential and egress policy |
//! | catalogs | the managed `/openai/v1/models?catalog=openrouter` listing and its limits request |
//!
//! `register` is the one place it does so; `crate::register::register_all`
@@ -17,7 +17,6 @@ mod backend_error_code;
mod managed_catalog;
mod managed_embedding;
mod managed_model;
-mod managed_reasoning;
mod managed_session;
mod tiers;
#[cfg(test)]
@@ -33,7 +32,6 @@ use openhuman_embed::__host::inference_registry::{
};
use tinyinference_llm::model::ChatModel;
-pub use managed_embedding::ManagedEmbeddingModel;
pub use managed_model::ManagedOptions;
use crate::install::InstallOptions;
@@ -128,12 +126,12 @@ pub(crate) fn register(_options: &InstallOptions) {
resolve_model: Arc::new(tiers::resolve_model),
context_window: Arc::new(tiers::context_window),
default_pricing: Arc::new(tiers::default_pricing),
- without_reasoning: Arc::new(managed_reasoning::without_reasoning),
+ without_reasoning: Arc::new(tinyinference_llm::model::without_reasoning),
}),
},
);
register_embedding(PROVIDER_OPENHUMAN, |request| {
- Ok(Box::new(ManagedEmbeddingModel::from_request(request)))
+ Ok(Box::new(managed_embedding::from_request(request)))
});
register_catalog(PROVIDER_OPENHUMAN, managed_catalog::entry());
register_config_normalizer(PROVIDER_OPENHUMAN, Arc::new(tiers::normalize_config));