Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): thread provider context through catalog and embedding registries

Steven Enamakela49f8b73e969
diff --git a/crates/openhuman-tinyhumans/src/inference/managed_catalog.rs b/crates/openhuman-tinyhumans/src/inference/managed_catalog.rs
index 0a8fdeff44..2b742bb68d 100644
--- a/crates/openhuman-tinyhumans/src/inference/managed_catalog.rs
+++ b/crates/openhuman-tinyhumans/src/inference/managed_catalog.rs
@@ -10,7 +10,7 @@
use openhuman_embed::__host::config::Config;
use openhuman_embed::__host::core::Outcome;
use openhuman_embed::__host::inference_registry::{
- BoxFuture, CatalogEntry, CatalogLimits, CatalogListing, CatalogRequest,
+ BoxFuture, CatalogEntry, CatalogHostRequest, CatalogLimits, CatalogRequest, ProviderContext,
};
use openhuman_embed::__host::security::credentials::session_support::{
classify_session_token, load_app_session_profile, publish_local_session_expiry,
@@ -30,20 +30,25 @@ use crate::client::endpoint::{
const MODELS_PATH: &str = "/openai/v1/models";
/// The registry entry for the managed catalog.
-pub(super) fn entry() -> CatalogEntry {
- let list: CatalogListing = Arc::new(
- |config: Config,
- request: CatalogRequest|
- -> BoxFuture<Result<Outcome<serde_json::Value>, String>> {
- Box::pin(async move { list_models(&config, request).await })
- },
- );
- let limits: CatalogLimits =
- Arc::new(|model: &str, config: &Config| limits_request(model, config));
- CatalogEntry {
- list,
- limits: Some(limits),
- }
+pub(super) fn entry() -> CatalogEntry<ProviderContext> {
+ let list = Arc::new(|request: CatalogRequest<ProviderContext>| -> BoxFuture<Result<serde_json::Value, String>> {
+ Box::pin(async move {
+ let context = request.context;
+ let host = context.catalog.as_ref().expect("catalog host context");
+ let outcome = list_models(
+ &context.config,
+ CatalogHostRequest {
+ provider_key: host.request.provider_key.clone(),
+ client: host.request.client.clone(),
+ },
+ ).await?;
+ host.logs.lock().unwrap_or_else(std::sync::PoisonError::into_inner).extend(outcome.logs);
+ Ok(outcome.value)
+ })
+ });
+ let limits: CatalogLimits<ProviderContext> =
+ Arc::new(|model: &str, context: &ProviderContext| limits_request(model, &context.config));
+ CatalogEntry { list, limits: Some(limits) }
}
fn empty_catalog(reason: &str) -> Result<Outcome<serde_json::Value>, String> {
@@ -105,7 +110,7 @@ pub(super) fn listing_auth(check: SessionTokenCheck, provider_key: &str) -> List
/// The `list_configured_models` listing for the managed provider.
pub(super) async fn list_models(
config: &Config,
- request: CatalogRequest,
+ request: CatalogHostRequest,
) -> Result<Outcome<serde_json::Value>, String> {
// Classify the session directly: a store error propagates (a recoverable
// fault the picker should surface), only an absent or expired session
diff --git a/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs b/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs
index f958d25eea..85128dbee7 100644
--- a/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs
+++ b/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs
@@ -7,7 +7,7 @@
use std::sync::Arc;
use openhuman_embed::__host::egress::{emit_external_transfer, enforce_egress, EgressDescriptor};
-use openhuman_embed::__host::inference_registry::EmbeddingRequest;
+use openhuman_embed::__host::inference_registry::{EmbeddingRequest, ProviderContext};
use openhuman_embed::__host::security::approval::ApprovalChatContext;
use tinyinference_embeddings::{BearerResolver, CloudEmbeddingModel, EmbeddingEgressGuard};
@@ -18,7 +18,7 @@ use crate::client::endpoint::is_managed_endpoint_for_api_key;
/// Builds the owning cloud embedding implementation with the credential scope
/// and turn context supplied by the inference registry.
pub(super) fn from_request(
- request: &EmbeddingRequest<'_, openhuman_embed::__host::inference_registry::ProviderContext>,
+ request: &EmbeddingRequest<'_, ProviderContext>,
) -> CloudEmbeddingModel {
let context = request.context.embedding.as_ref().expect("embedding host context");
build_cloud_model(
diff --git a/crates/openhuman-tinyhumans/src/inference/mod.rs b/crates/openhuman-tinyhumans/src/inference/mod.rs
index f6216c6784..b16f2d01b0 100644
--- a/crates/openhuman-tinyhumans/src/inference/mod.rs
+++ b/crates/openhuman-tinyhumans/src/inference/mod.rs
@@ -27,8 +27,9 @@ use std::sync::Arc;
use openhuman_embed::__host::config::Config;
use openhuman_embed::__host::inference_registry::{
register_backend_error_classifier, register_catalog, register_chat_model,
- register_config_normalizer, register_embedding, BoxFuture, ChatModelEntry, ChatModelFactory,
- ChatModelRequest, ProviderContext, ProviderModelMetadata, ReadinessProbe, PROVIDER_OPENHUMAN,
+ register_config_normalizer, register_embedding, ChatModelEntry, ChatModelFactory,
+ ChatModelRequest, EmbeddingRequest, ProviderContext, ProviderModelMetadata, ReadinessProbe,
+ PROVIDER_OPENHUMAN,
};
use tinyinference_llm::model::ChatModel;
@@ -135,9 +136,12 @@ pub(crate) fn register(_options: &InstallOptions) {
}),
},
);
- register_embedding(PROVIDER_OPENHUMAN, |request| {
- Ok(Box::new(managed_embedding::from_request(request)))
- });
+ register_embedding(
+ PROVIDER_OPENHUMAN,
+ Arc::new(|request: &EmbeddingRequest<'_, ProviderContext>| {
+ Ok(Box::new(managed_embedding::from_request(request)))
+ }),
+ );
register_catalog(PROVIDER_OPENHUMAN, managed_catalog::entry());
register_config_normalizer(PROVIDER_OPENHUMAN, Arc::new(tiers::normalize_config));
register_backend_error_classifier(Arc::new(backend_error_code::classify));