Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): thread provider context through model catalog lookups

Steven Enamakela97dcd271f61
diff --git a/crates/openhuman-core/src/agent/session_host/test_support.rs b/crates/openhuman-core/src/agent/session_host/test_support.rs
index 474c893efe..2e2b1c0079 100644
--- a/crates/openhuman-core/src/agent/session_host/test_support.rs
+++ b/crates/openhuman-core/src/agent/session_host/test_support.rs
@@ -1,7 +1,9 @@
use super::*;
use crate::inference::provider::factory::PROVIDER_OPENHUMAN;
-use crate::inference::registry::{ChatModelEntry, ChatModelRequest, ProviderContext, ProviderModelMetadata};
+use crate::inference::registry::{
+ ChatModelEntry, ChatModelRequest, ProviderContext, ProviderModelMetadata,
+};
use std::sync::Arc;
use tinyinference_llm::model::{ChatModel, ModelRequest, ModelResponse};
@@ -67,7 +69,8 @@ fn model_fixture_entry() -> ChatModelEntry<ProviderContext> {
.map(|hint_role| {
if hint_role == "chat" {
config
- .config.default_model
+ .config
+ .default_model
.as_deref()
.filter(|configured| !configured.starts_with("hint:"))
.map(str::to_string)
diff --git a/crates/openhuman-core/src/inference/context_window.rs b/crates/openhuman-core/src/inference/context_window.rs
index fcbe6821a0..8a2398a1ad 100644
--- a/crates/openhuman-core/src/inference/context_window.rs
+++ b/crates/openhuman-core/src/inference/context_window.rs
@@ -232,7 +232,8 @@ pub(crate) fn model_limits_request_for(
.filter(|model| !model.is_empty())
.unwrap_or(model);
let context = crate::inference::registry::ProviderContext::new(
- config, &crate::security::egress::EgressLedger::none(),
+ config,
+ &crate::security::egress::EgressLedger::none(),
);
return crate::inference::registry::catalog_for(
extensions,
@@ -260,7 +261,8 @@ pub(crate) fn model_limits_request_for(
route.model.trim()
};
let context = crate::inference::registry::ProviderContext::new(
- config, &crate::security::egress::EgressLedger::none(),
+ config,
+ &crate::security::egress::EgressLedger::none(),
);
return crate::inference::registry::catalog_for(
extensions,
diff --git a/crates/openhuman-core/src/inference/embedding_host/factory.rs b/crates/openhuman-core/src/inference/embedding_host/factory.rs
index 93a29b3832..a4de4dc1a7 100644
--- a/crates/openhuman-core/src/inference/embedding_host/factory.rs
+++ b/crates/openhuman-core/src/inference/embedding_host/factory.rs
@@ -11,10 +11,10 @@ use tinyinference_embeddings::{EmbeddingModel, UnavailableEmbeddingModel};
use tinyinference_providers::route::custom_provider_string;
use crate::inference::provider::factory::PROVIDER_OPENHUMAN;
-use tinyinference_llm::providers::overrides::EmbeddingRequest;
use crate::inference::registry::ProviderContext;
use crate::security::approval::ApprovalChatContext;
use crate::security::credentials::embedding_scope::credential_scope;
+use tinyinference_llm::providers::overrides::EmbeddingRequest;
pub(super) fn settings(
provider: &str,
@@ -41,9 +41,16 @@ pub(super) fn managed_factory(
log::debug!("[embeddings::factory] managed model served by a registered factory");
let (state_dir, secrets_encrypt) = credential_scope(config.as_ref());
let context = ProviderContext::for_embedding(
- config.as_ref(), chat.clone(), state_dir, secrets_encrypt,
+ config.as_ref(),
+ chat.clone(),
+ state_dir,
+ secrets_encrypt,
);
- return registered(&EmbeddingRequest { model, dimensions, context: &context });
+ return registered(&EmbeddingRequest {
+ model,
+ dimensions,
+ context: &context,
+ });
}
log::debug!("[embeddings::factory] no managed embedding provider registered");
let message = format!(
diff --git a/crates/openhuman-core/src/inference/model_context_tests.rs b/crates/openhuman-core/src/inference/model_context_tests.rs
index 923e25606d..cbfd913cc1 100644
--- a/crates/openhuman-core/src/inference/model_context_tests.rs
+++ b/crates/openhuman-core/src/inference/model_context_tests.rs
@@ -101,7 +101,9 @@ fn managed_vision_capability_is_absent_without_provider_policy() {
#[test]
fn model_supports_vision_uses_the_scoped_provider_capability_hook() {
use crate::core::slots::Extensions;
- use crate::inference::registry::{register_chat_model_in, ChatModelEntry, ProviderModelMetadata};
+ use crate::inference::registry::{
+ register_chat_model_in, ChatModelEntry, ProviderModelMetadata,
+ };
use std::sync::Arc;
let extensions = Extensions::new();
diff --git a/crates/openhuman-core/src/inference/provider/factory/chat_model.rs b/crates/openhuman-core/src/inference/provider/factory/chat_model.rs
index 062818cdcd..c92ed40e1b 100644
--- a/crates/openhuman-core/src/inference/provider/factory/chat_model.rs
+++ b/crates/openhuman-core/src/inference/provider/factory/chat_model.rs
@@ -506,7 +506,8 @@ pub async fn probe_inference_readiness(
return Ok(());
};
let provider_context = crate::inference::registry::ProviderContext::new(
- config, &crate::security::egress::EgressLedger::none(),
+ config,
+ &crate::security::egress::EgressLedger::none(),
);
let model_id = entry
.models
@@ -525,7 +526,8 @@ pub async fn probe_inference_readiness(
role: role.to_string(),
model: model_id.clone(),
context: provider_context,
- }).await;
+ })
+ .await;
log::debug!(
"[flows][inference-probe] role={role} model={model_id} probe result: {}",
if result.is_ok() { "ready" } else { "not ready" }
diff --git a/crates/openhuman-core/src/inference/provider/factory/managed_backend.rs b/crates/openhuman-core/src/inference/provider/factory/managed_backend.rs
index 2f3c4bf782..53b8f29c0e 100644
--- a/crates/openhuman-core/src/inference/provider/factory/managed_backend.rs
+++ b/crates/openhuman-core/src/inference/provider/factory/managed_backend.rs
@@ -67,7 +67,11 @@ pub(super) fn managed_chat_model_for(
return Err(managed_unavailable_error());
};
let model = match entry.models.as_ref() {
- Some(models) => (models.resolve_model)(role, &crate::inference::registry::ProviderContext::new(config, egress), model_override),
+ Some(models) => (models.resolve_model)(
+ role,
+ &crate::inference::registry::ProviderContext::new(config, egress),
+ model_override,
+ ),
None => model_override
.or(config.default_model.as_deref())
.map(str::trim)
diff --git a/crates/openhuman-core/src/inference/provider/factory_tests.rs b/crates/openhuman-core/src/inference/provider/factory_tests.rs
index c90e5a0dd0..3f0e6a0f7d 100644
--- a/crates/openhuman-core/src/inference/provider/factory_tests.rs
+++ b/crates/openhuman-core/src/inference/provider/factory_tests.rs
@@ -74,7 +74,8 @@ fn install_test_managed_provider() -> Arc<std::sync::Mutex<Vec<bool>>> {
Some(model) if !model.starts_with("hint:") => Some(model.into()),
Some(_) => None,
None => config
- .config.default_model
+ .config
+ .default_model
.clone()
.or_else(|| Some("fixture/managed-default".into())),
}),
diff --git a/crates/openhuman-core/src/inference/provider/models/catalog_listing.rs b/crates/openhuman-core/src/inference/provider/models/catalog_listing.rs
index 29475a689c..34e59080e4 100644
--- a/crates/openhuman-core/src/inference/provider/models/catalog_listing.rs
+++ b/crates/openhuman-core/src/inference/provider/models/catalog_listing.rs
@@ -71,21 +71,33 @@ fn catalog_request(
}
None => crate::config::build_runtime_proxy_client_with_timeouts(service_key, 30, 10),
};
- Ok(crate::inference::registry::CatalogHostRequest { provider_key, client })
+ Ok(crate::inference::registry::CatalogHostRequest {
+ provider_key,
+ client,
+ })
}
async fn run_registered_catalog(
provider_id: &str,
config: &crate::config::Config,
request: crate::inference::registry::CatalogHostRequest,
- catalog: tinyinference_llm::providers::overrides::CatalogEntry<crate::inference::registry::ProviderContext>,
+ catalog: tinyinference_llm::providers::overrides::CatalogEntry<
+ crate::inference::registry::ProviderContext,
+ >,
) -> Result<crate::core::Outcome<serde_json::Value>, String> {
let context = crate::inference::registry::ProviderContext::for_catalog(config, request);
let call = context.catalog.as_ref().expect("catalog context");
let logs = Arc::clone(&call.logs);
- let listing = (catalog.list)(tinyinference_llm::providers::overrides::CatalogRequest { context }).await?;
- let logs = std::mem::take(&mut *logs.lock().unwrap_or_else(std::sync::PoisonError::into_inner));
- log::debug!("[providers][list_models] provider_id={provider_id} served by native provider catalog");
+ let listing =
+ (catalog.list)(tinyinference_llm::providers::overrides::CatalogRequest { context }).await?;
+ let logs = std::mem::take(
+ &mut *logs
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner),
+ );
+ log::debug!(
+ "[providers][list_models] provider_id={provider_id} served by native provider catalog"
+ );
Ok(crate::core::Outcome::new(listing, logs))
}
@@ -159,7 +171,13 @@ pub async fn list_configured_models_from_config(
) {
Some(catalog) => {
let request = catalog_request(config, Some(&entry))?;
- run_registered_catalog(crate::inference::provider::factory::PROVIDER_OPENHUMAN, config, request, catalog).await
+ run_registered_catalog(
+ crate::inference::provider::factory::PROVIDER_OPENHUMAN,
+ config,
+ request,
+ catalog,
+ )
+ .await
}
None => Ok(no_managed_catalog()),
};
diff --git a/crates/openhuman-core/src/inference/provider_model_tests.rs b/crates/openhuman-core/src/inference/provider_model_tests.rs
index 3e91a817ef..2fb406224d 100644
--- a/crates/openhuman-core/src/inference/provider_model_tests.rs
+++ b/crates/openhuman-core/src/inference/provider_model_tests.rs
@@ -33,7 +33,9 @@ fn abstract_hint_markers_do_not_become_raw_model_ids() {
#[test]
fn registered_model_defaults_and_aliases_are_context_scoped() {
use crate::core::slots::Extensions;
- use crate::inference::registry::{register_chat_model_in, ChatModelEntry, ProviderModelMetadata};
+ use crate::inference::registry::{
+ register_chat_model_in, ChatModelEntry, ProviderModelMetadata,
+ };
use std::sync::Arc;
let extensions = Extensions::new();
@@ -242,9 +244,15 @@ fn native_provider_metadata_resolves_by_context_and_preserves_request_fields() {
is_raw_model: Arc::new(move |model| model == format!("{label}/raw")),
supports_vision: Arc::new(move |model| model == format!("{label}/vision")),
resolve_model: Arc::new(move |role, context: &ProviderContext, pin| {
- Some(format!("{label}:{}:{}", context.config.default_model.as_deref().unwrap_or("unset"), pin.unwrap_or(role)))
+ Some(format!(
+ "{label}:{}:{}",
+ context.config.default_model.as_deref().unwrap_or("unset"),
+ pin.unwrap_or(role)
+ ))
+ }),
+ context_window: Arc::new(move |model| {
+ (model == format!("{label}/raw")).then_some(window)
}),
- context_window: Arc::new(move |model| (model == format!("{label}/raw")).then_some(window)),
default_pricing: Arc::new(move || Some((format!("{label}/priced"), 1.25, 2.5))),
without_reasoning: Arc::new(move |mut request| {
request.metadata["reasoning_removed_by"] = json!(label);
@@ -264,23 +272,78 @@ fn native_provider_metadata_resolves_by_context_and_preserves_request_fields() {
register_chat_model_in(&b_extensions, provider, entry("context-b", 303));
let verify = |extensions: Option<&Extensions>, label: &str, window: u64| {
- assert_eq!(crate::inference::registry::provider_default_model_for(extensions, provider).as_deref(), Some(format!("{label}/default").as_str()));
- assert_eq!(crate::inference::registry::provider_model_role_for(extensions, provider, &format!("{label}/alias")).as_deref(), Some(format!("{label}/role").as_str()));
- assert!(crate::inference::registry::provider_model_is_alias_for(extensions, provider, &format!("{label}/alias")));
- assert!(crate::inference::registry::provider_model_is_raw_for(extensions, provider, &format!("{label}/raw")));
- assert!(crate::inference::registry::provider_model_supports_vision_for(extensions, provider, &format!("{label}/vision")));
- let config = Config { default_model: Some(format!("{label}/config")), ..Config::default() };
- assert_eq!(crate::inference::registry::resolve_provider_model_for(extensions, provider, "chat", &config, Some("pin")), Some(format!("{label}:{label}/config:pin")));
- assert_eq!(crate::inference::registry::provider_model_context_window_for(extensions, provider, &format!("{label}/raw")), Some(window));
- assert_eq!(crate::inference::registry::provider_default_pricing_for(extensions, provider), Some((format!("{label}/priced"), 1.25, 2.5)));
+ assert_eq!(
+ crate::inference::registry::provider_default_model_for(extensions, provider).as_deref(),
+ Some(format!("{label}/default").as_str())
+ );
+ assert_eq!(
+ crate::inference::registry::provider_model_role_for(
+ extensions,
+ provider,
+ &format!("{label}/alias")
+ )
+ .as_deref(),
+ Some(format!("{label}/role").as_str())
+ );
+ assert!(crate::inference::registry::provider_model_is_alias_for(
+ extensions,
+ provider,
+ &format!("{label}/alias")
+ ));
+ assert!(crate::inference::registry::provider_model_is_raw_for(
+ extensions,
+ provider,
+ &format!("{label}/raw")
+ ));
+ assert!(
+ crate::inference::registry::provider_model_supports_vision_for(
+ extensions,
+ provider,
+ &format!("{label}/vision")
+ )
+ );
+ let config = Config {
+ default_model: Some(format!("{label}/config")),
+ ..Config::default()
+ };
+ assert_eq!(
+ crate::inference::registry::resolve_provider_model_for(
+ extensions,
+ provider,
+ "chat",
+ &config,
+ Some("pin")
+ ),
+ Some(format!("{label}:{label}/config:pin"))
+ );
+ assert_eq!(
+ crate::inference::registry::provider_model_context_window_for(
+ extensions,
+ provider,
+ &format!("{label}/raw")
+ ),
+ Some(window)
+ );
+ assert_eq!(
+ crate::inference::registry::provider_default_pricing_for(extensions, provider),
+ Some((format!("{label}/priced"), 1.25, 2.5))
+ );
let mut request = tinyinference_llm::model::ModelRequest::new(vec![]);
request.model = Some(format!("{label}/raw"));
request.max_tokens = Some(37);
request.metadata = json!({"caller": "kept"});
- let request = crate::inference::registry::provider_request_without_reasoning_for(extensions, provider, request);
- assert_eq!(request.model.as_deref(), Some(format!("{label}/raw").as_str()));
+ let request = crate::inference::registry::provider_request_without_reasoning_for(
+ extensions, provider, request,
+ );
+ assert_eq!(
+ request.model.as_deref(),
+ Some(format!("{label}/raw").as_str())
+ );
assert_eq!(request.max_tokens, Some(37));
- assert_eq!(request.metadata, json!({"caller": "kept", "reasoning_removed_by": label}));
+ assert_eq!(
+ request.metadata,
+ json!({"caller": "kept", "reasoning_removed_by": label})
+ );
};
verify(None, "process", 101);
@@ -290,23 +353,45 @@ fn native_provider_metadata_resolves_by_context_and_preserves_request_fields() {
verify(Some(&b_extensions), "context-b", 303);
let ambient_a = crate::core::runtime::CoreContext::for_test_with_extensions(
- crate::core::runtime::DomainSet::full(), None, a_extensions.clone(),
+ crate::core::runtime::DomainSet::full(),
+ None,
+ a_extensions.clone(),
);
crate::core::runtime::CoreContext::sync_scope(ambient_a, || {
- assert_eq!(crate::inference::registry::provider_default_model(provider).as_deref(), Some("context-a/default"));
- assert_eq!(crate::inference::registry::provider_default_pricing(provider).as_ref().map(|price| price.0.as_str()), Some("context-a/priced"));
- let request = crate::inference::registry::provider_request_without_reasoning(provider, tinyinference_llm::model::ModelRequest::new(vec![]));
+ assert_eq!(
+ crate::inference::registry::provider_default_model(provider).as_deref(),
+ Some("context-a/default")
+ );
+ assert_eq!(
+ crate::inference::registry::provider_default_pricing(provider)
+ .as_ref()
+ .map(|price| price.0.as_str()),
+ Some("context-a/priced")
+ );
+ let request = crate::inference::registry::provider_request_without_reasoning(
+ provider,
+ tinyinference_llm::model::ModelRequest::new(vec![]),
+ );
assert_eq!(request.metadata["reasoning_removed_by"], "context-a");
- assert_eq!(crate::inference::registry::provider_model_context_window_for(None, provider, "process/raw"), Some(101), "explicit process-only lookup must ignore ambient registration");
+ assert_eq!(
+ crate::inference::registry::provider_model_context_window_for(
+ None,
+ provider,
+ "process/raw"
+ ),
+ Some(101),
+ "explicit process-only lookup must ignore ambient registration"
+ );
});
unregister_chat_model(provider);
}
-
#[test]
fn managed_options_builder_uses_the_ambient_native_provider_entry() {
use crate::core::slots::Extensions;
- use crate::inference::registry::{register_chat_model_in, unregister_chat_model, ChatModelEntry, ProviderContext};
+ use crate::inference::registry::{
+ register_chat_model_in, unregister_chat_model, ChatModelEntry, ProviderContext,
+ };
use std::sync::Arc;
let _guard = crate::inference::inference_test_guard();
@@ -320,8 +405,13 @@ fn managed_options_builder_uses_the_ambient_native_provider_entry() {
ChatModelEntry::<ProviderContext> {
build: Arc::new(|request| {
Ok((
- Arc::new(tinyagents_harness::testkit::ScriptedModel::replies(vec!["ambient"])),
- request.model_override.unwrap_or("ambient/model").to_string(),
+ Arc::new(tinyagents_harness::testkit::ScriptedModel::replies(vec![
+ "ambient",
+ ])),
+ request
+ .model_override
+ .unwrap_or("ambient/model")
+ .to_string(),
))
}),
probe: None,
@@ -329,7 +419,9 @@ fn managed_options_builder_uses_the_ambient_native_provider_entry() {
},
);
let context = crate::core::runtime::CoreContext::for_test_with_extensions(
- crate::core::runtime::DomainSet::full(), None, extensions,
+ crate::core::runtime::DomainSet::full(),
+ None,
+ extensions,
);
let built = crate::core::runtime::CoreContext::sync_scope(context, || {
crate::inference::provider::factory::make_openhuman_backend_model_for_options(
@@ -338,5 +430,8 @@ fn managed_options_builder_uses_the_ambient_native_provider_entry() {
None,
)
});
- assert_eq!(built.expect("ambient registration must be visible").1, "ambient/pinned");
+ assert_eq!(
+ built.expect("ambient registration must be visible").1,
+ "ambient/pinned"
+ );
}
diff --git a/crates/openhuman-core/src/inference/readiness_tests.rs b/crates/openhuman-core/src/inference/readiness_tests.rs
index a4155dba5a..2b9973e551 100644
--- a/crates/openhuman-core/src/inference/readiness_tests.rs
+++ b/crates/openhuman-core/src/inference/readiness_tests.rs
@@ -5,7 +5,9 @@ use tempfile::TempDir;
async fn cached_probe_serves_definitive_negative_results() {
use crate::core::runtime::{CoreContext, DispatchCtx, DomainSet};
use crate::core::slots::Extensions;
- use crate::inference::registry::{register_chat_model_in, ChatModelEntry, ProviderModelMetadata};
+ use crate::inference::registry::{
+ register_chat_model_in, ChatModelEntry, ProviderModelMetadata,
+ };
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
diff --git a/crates/openhuman-core/src/inference/registry.rs b/crates/openhuman-core/src/inference/registry.rs
index 6b0e83d09a..af3779a4d2 100644
--- a/crates/openhuman-core/src/inference/registry.rs
+++ b/crates/openhuman-core/src/inference/registry.rs
@@ -16,7 +16,6 @@
use std::sync::Arc;
-
use crate::config::Config;
use crate::core::slots::{Extensions, HookSlot};
use crate::security::approval::ApprovalChatContext;
@@ -61,7 +60,12 @@ pub struct CatalogHostRequest {
impl ProviderContext {
pub(crate) fn new(config: &Config, egress: &EgressLedger) -> Self {
- Self { config: config.clone(), egress: egress.clone(), embedding: None, catalog: None }
+ Self {
+ config: config.clone(),
+ egress: egress.clone(),
+ embedding: None,
+ catalog: None,
+ }
}
pub(crate) fn for_embedding(
@@ -74,7 +78,12 @@ impl ProviderContext {
Self {
config: base,
egress: EgressLedger::none(),
- embedding: Some(EmbeddingCallContext { config: config.cloned(), chat, state_dir, secrets_encrypt }),
+ embedding: Some(EmbeddingCallContext {
+ config: config.cloned(),
+ chat,
+ state_dir,
+ secrets_encrypt,
+ }),
catalog: None,
}
}
@@ -84,18 +93,23 @@ impl ProviderContext {
config: config.clone(),
egress: EgressLedger::none(),
embedding: None,
- catalog: Some(CatalogCallContext { request, logs: Arc::new(std::sync::Mutex::new(Vec::new())) }),
+ catalog: Some(CatalogCallContext {
+ request,
+ logs: Arc::new(std::sync::Mutex::new(Vec::new())),
+ }),
}
}
}
pub use tinyinference_llm::providers::overrides::{
BoxFuture, CatalogEntry, CatalogLimits, CatalogRequest, ChatModelEntry, ChatModelFactory,
- ChatModelRequest, EmbeddingFactory, EmbeddingRequest, ProviderModelMetadata,
- ProviderOverrides, ReadinessProbe,
+ ChatModelRequest, EmbeddingFactory, EmbeddingRequest, ProviderModelMetadata, ProviderOverrides,
+ ReadinessProbe,
};
-pub(crate) fn provider_overrides(extensions: Option<Extensions>) -> ProviderOverrides<ProviderContext> {
+pub(crate) fn provider_overrides(
+ extensions: Option<Extensions>,
+) -> ProviderOverrides<ProviderContext> {
let chat_extensions = extensions.clone();
let embedding_extensions = extensions.clone();
ProviderOverrides::new(
@@ -180,7 +194,8 @@ pub fn classify_backend_error(
.map(|classifier| classifier(provider, error))
}
-static CHAT_MODELS: HookSlot<ChatModelEntry<ProviderContext>> = HookSlot::new("inference::chat_models");
+static CHAT_MODELS: HookSlot<ChatModelEntry<ProviderContext>> =
+ HookSlot::new("inference::chat_models");
/// Registers the chat-model provider `provider_id` for the whole process.
pub fn register_chat_model(provider_id: impl Into<String>, entry: ChatModelEntry<ProviderContext>) {
@@ -217,7 +232,9 @@ pub(crate) fn chat_model_for(
}
/// The registered provider's model hooks, if it has any.
-pub(crate) fn provider_model_hooks(provider_id: &str) -> Option<ProviderModelMetadata<ProviderContext>> {
+pub(crate) fn provider_model_hooks(
+ provider_id: &str,
+) -> Option<ProviderModelMetadata<ProviderContext>> {
chat_model(provider_id)?.models
}
@@ -325,7 +342,11 @@ pub fn resolve_provider_model(
model_override: Option<&str>,
) -> Option<String> {
let hooks = provider_model_hooks(provider_id)?;
- (hooks.resolve_model)(role, &ProviderContext::new(config, &EgressLedger::none()), model_override)
+ (hooks.resolve_model)(
+ role,
+ &ProviderContext::new(config, &EgressLedger::none()),
+ model_override,
+ )
}
pub(crate) fn resolve_provider_model_for(
@@ -336,7 +357,11 @@ pub(crate) fn resolve_provider_model_for(
model_override: Option<&str>,
) -> Option<String> {
let hooks = chat_model_for(extensions, provider_id)?.models?;
- (hooks.resolve_model)(role, &ProviderContext::new(config, &EgressLedger::none()), model_override)
+ (hooks.resolve_model)(
+ role,
+ &ProviderContext::new(config, &EgressLedger::none()),
+ model_override,
+ )
}
/// The provider's model-specific context window, if known.
@@ -411,15 +436,23 @@ pub(crate) fn normalize_config(config: &mut Config) -> bool {
// Embeddings
// ---------------------------------------------------------------------------
-static EMBEDDINGS: HookSlot<EmbeddingFactory<ProviderContext>> = HookSlot::new("inference::embeddings");
+static EMBEDDINGS: HookSlot<EmbeddingFactory<ProviderContext>> =
+ HookSlot::new("inference::embeddings");
/// Registers the embedding provider `provider_id` for the whole process.
-pub fn register_embedding(provider_id: impl Into<String>, factory: EmbeddingFactory<ProviderContext>) {
+pub fn register_embedding(
+ provider_id: impl Into<String>,
+ factory: EmbeddingFactory<ProviderContext>,
+) {
EMBEDDINGS.register(provider_id, factory);
}
/// Registers the embedding provider `provider_id` for the context built from `extensions`.
-pub fn register_embedding_in(extensions: &Extensions, provider_id: impl Into<String>, factory: EmbeddingFactory<ProviderContext>) {
+pub fn register_embedding_in(
+ extensions: &Extensions,
+ provider_id: impl Into<String>,
+ factory: EmbeddingFactory<ProviderContext>,
+) {
EMBEDDINGS.register_in(extensions, provider_id, factory);
}
diff --git a/crates/openhuman-core/src/inference/registry_tests.rs b/crates/openhuman-core/src/inference/registry_tests.rs
index 9dfba10f69..495ae99858 100644
--- a/crates/openhuman-core/src/inference/registry_tests.rs
+++ b/crates/openhuman-core/src/inference/registry_tests.rs
@@ -1,6 +1,4 @@
use super::*;
-use tinyinference_llm::model::ChatModel;
-use tinyinference_llm::model::discover::DiscoveryRequest;
use crate::inference::context_window::model_limits_request;
use crate::inference::provider::factory::{
create_chat_model_from_string, make_openhuman_backend_model_for_thread, PROVIDER_OPENHUMAN,
@@ -8,6 +6,8 @@ use crate::inference::provider::factory::{
use async_trait::async_trait;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Mutex;
+use tinyinference_llm::model::discover::DiscoveryRequest;
+use tinyinference_llm::model::ChatModel;
use tinyinference_llm::model::{ModelRequest, ModelResponse};
type SeenModelRequests = Arc<Mutex<Vec<(String, Option<String>, Option<String>)>>>;
@@ -52,7 +52,8 @@ fn fake_entry(seen: SeenModelRequests) -> ChatModelEntry<ProviderContext> {
Some(model) if !model.starts_with("hint:") => Some(model.to_string()),
Some(_) => None,
None => config
- .config.default_model
+ .config
+ .default_model
.clone()
.or_else(|| Some("host/fake-default".to_string())),
}),
@@ -127,11 +128,14 @@ fn a_registered_embedding_factory_builds_the_managed_embedding_model() {
let _guard = crate::inference::inference_test_guard();
let calls = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&calls);
- register_embedding(PROVIDER_OPENHUMAN, Arc::new(move |request| {
- assert_eq!(request.dimensions, 8);
- counter.fetch_add(1, Ordering::SeqCst);
- Ok(Box::new(tinyinference_embeddings::NoopEmbeddingModel))
- }));
+ register_embedding(
+ PROVIDER_OPENHUMAN,
+ Arc::new(move |request| {
+ assert_eq!(request.dimensions, 8);
+ counter.fetch_add(1, Ordering::SeqCst);
+ Ok(Box::new(tinyinference_embeddings::NoopEmbeddingModel))
+ }),
+ );
let built = crate::inference::embedding_host::create_embedding_provider_with_credentials(
"cloud",
"any-model",
@@ -150,9 +154,7 @@ async fn a_registered_catalog_answers_the_model_listing_for_its_provider() {
"fake-catalog",
CatalogEntry {
list: Arc::new(|_| {
- Box::pin(async {
- Ok(serde_json::json!({ "models": [{ "id": "registered" }] }))
- })
+ Box::pin(async { Ok(serde_json::json!({ "models": [{ "id": "registered" }] })) })
}),
limits: None,
},
@@ -240,7 +242,14 @@ async fn a_registered_catalog_receives_the_entrys_provider_key() {
PROVIDER_OPENHUMAN,
CatalogEntry {
list: Arc::new(move |request: CatalogRequest<ProviderContext>| {
- *sink.lock().unwrap() = request.context.catalog.as_ref().unwrap().request.provider_key.clone();
+ *sink.lock().unwrap() = request
+ .context
+ .catalog
+ .as_ref()
+ .unwrap()
+ .request
+ .provider_key
+ .clone();
Box::pin(async { Ok(serde_json::json!({ "models": [] })) })
}),
limits: None,
diff --git a/crates/openhuman-core/src/inference/turn_models_tests.rs b/crates/openhuman-core/src/inference/turn_models_tests.rs
index 39386d6946..2776866e64 100644
--- a/crates/openhuman-core/src/inference/turn_models_tests.rs
+++ b/crates/openhuman-core/src/inference/turn_models_tests.rs
@@ -31,7 +31,7 @@ use tinyagents_harness::host::PrebuiltModelResolver as TurnModelResolver;
fn scoped_model_entry(
name: &'static str,
supports_vision: bool,
-) -> crate::inference::registry::ChatModelEntry::<crate::inference::registry::ProviderContext> {
+) -> crate::inference::registry::ChatModelEntry<crate::inference::registry::ProviderContext> {
use crate::inference::registry::ProviderModelMetadata;
crate::inference::registry::ChatModelEntry::<crate::inference::registry::ProviderContext> {
diff --git a/crates/openhuman-core/src/web_chat/suggestions_tests.rs b/crates/openhuman-core/src/web_chat/suggestions_tests.rs
index 740097fbb1..f656e551bf 100644
--- a/crates/openhuman-core/src/web_chat/suggestions_tests.rs
+++ b/crates/openhuman-core/src/web_chat/suggestions_tests.rs
@@ -7,7 +7,9 @@ use tinyinference_llm::model::{ChatModel, ModelRequest, ModelResponse};
use super::*;
use crate::inference::provider::factory::test_provider_override;
use crate::inference::provider::factory::PROVIDER_OPENHUMAN;
-use crate::inference::registry::{ChatModelEntry, ChatModelRequest, ProviderContext, ProviderModelMetadata};
+use crate::inference::registry::{
+ ChatModelEntry, ChatModelRequest, ProviderContext, ProviderModelMetadata,
+};
use crate::web_chat::subscribe_web_channel_events;
// ── parse_suggestions / strip_markdown_fence: pure unit tests ──────────────
diff --git a/crates/openhuman-embed/examples/support/mod.rs b/crates/openhuman-embed/examples/support/mod.rs
index 9726888bc9..92c630493f 100644
--- a/crates/openhuman-embed/examples/support/mod.rs
+++ b/crates/openhuman-embed/examples/support/mod.rs
@@ -229,7 +229,8 @@ pub struct PointedInference;
impl PointedInference {
pub fn install(base_url: &str) {
use openhuman_embed::__host::inference_registry::{
- register_chat_model, ChatModelEntry, ChatModelRequest, ProviderContext, PROVIDER_OPENHUMAN,
+ register_chat_model, ChatModelEntry, ChatModelRequest, ProviderContext,
+ PROVIDER_OPENHUMAN,
};
use tinyinference_llm::providers::openai::OpenAiModel;
diff --git a/crates/openhuman-embed/tests/common/mod.rs b/crates/openhuman-embed/tests/common/mod.rs
index 26a067fd09..27701c2a38 100644
--- a/crates/openhuman-embed/tests/common/mod.rs
+++ b/crates/openhuman-embed/tests/common/mod.rs
@@ -260,8 +260,8 @@ pub struct PointedInference;
impl PointedInference {
pub fn install(base_url: &str) {
use openhuman_embed::__host::inference_registry::{
- register_chat_model, ChatModelEntry, ChatModelRequest, ProviderContext, ProviderModelMetadata,
- PROVIDER_OPENHUMAN,
+ register_chat_model, ChatModelEntry, ChatModelRequest, ProviderContext,
+ ProviderModelMetadata, PROVIDER_OPENHUMAN,
};
use tinyinference_llm::providers::openai::OpenAiModel;
diff --git a/crates/openhuman-tinyhumans/src/inference/managed_catalog.rs b/crates/openhuman-tinyhumans/src/inference/managed_catalog.rs
index 2b742bb68d..5b545d93cb 100644
--- a/crates/openhuman-tinyhumans/src/inference/managed_catalog.rs
+++ b/crates/openhuman-tinyhumans/src/inference/managed_catalog.rs
@@ -31,24 +31,33 @@ const MODELS_PATH: &str = "/openai/v1/models";
/// The registry entry for the managed catalog.
pub(super) fn entry() -> CatalogEntry<ProviderContext> {
- let list = Arc::new(|request: CatalogRequest<ProviderContext>| -> BoxFuture<Result<serde_json::Value, String>> {
- Box::pin(async move {
- let context = request.context;
- let host = context.catalog.as_ref().expect("catalog host context");
- let outcome = list_models(
- &context.config,
- CatalogHostRequest {
- provider_key: host.request.provider_key.clone(),
- client: host.request.client.clone(),
- },
- ).await?;
- host.logs.lock().unwrap_or_else(std::sync::PoisonError::into_inner).extend(outcome.logs);
- Ok(outcome.value)
- })
- });
+ let list = Arc::new(
+ |request: CatalogRequest<ProviderContext>| -> BoxFuture<Result<serde_json::Value, String>> {
+ Box::pin(async move {
+ let context = request.context;
+ let host = context.catalog.as_ref().expect("catalog host context");
+ let outcome = list_models(
+ &context.config,
+ CatalogHostRequest {
+ provider_key: host.request.provider_key.clone(),
+ client: host.request.client.clone(),
+ },
+ )
+ .await?;
+ host.logs
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .extend(outcome.logs);
+ Ok(outcome.value)
+ })
+ },
+ );
let limits: CatalogLimits<ProviderContext> =
Arc::new(|model: &str, context: &ProviderContext| limits_request(model, &context.config));
- CatalogEntry { list, limits: Some(limits) }
+ CatalogEntry {
+ list,
+ limits: Some(limits),
+ }
}
fn empty_catalog(reason: &str) -> Result<Outcome<serde_json::Value>, String> {
diff --git a/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs b/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs
index 85128dbee7..9fb1d922ed 100644
--- a/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs
+++ b/crates/openhuman-tinyhumans/src/inference/managed_embedding.rs
@@ -17,12 +17,17 @@ use crate::client::endpoint::is_managed_endpoint_for_api_key;
/// Builds the owning cloud embedding implementation with the credential scope
/// and turn context supplied by the inference registry.
-pub(super) fn from_request(
- request: &EmbeddingRequest<'_, ProviderContext>,
-) -> CloudEmbeddingModel {
- let context = request.context.embedding.as_ref().expect("embedding host context");
+pub(super) fn from_request(request: &EmbeddingRequest<'_, ProviderContext>) -> CloudEmbeddingModel {
+ let context = request
+ .context
+ .embedding
+ .as_ref()
+ .expect("embedding host context");
build_cloud_model(
- context.config.as_ref().and_then(|config| config.api_url.clone()),
+ context
+ .config
+ .as_ref()
+ .and_then(|config| config.api_url.clone()),
context.state_dir.clone(),
context.secrets_encrypt,
request.model,