Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): pass provider context to catalog limits and listings

Steven Enamakel1a89c6360a8f
diff --git a/crates/openhuman-core/src/inference/context_window.rs b/crates/openhuman-core/src/inference/context_window.rs
index 083c133290..fcbe6821a0 100644
--- a/crates/openhuman-core/src/inference/context_window.rs
+++ b/crates/openhuman-core/src/inference/context_window.rs
@@ -231,12 +231,15 @@ pub(crate) fn model_limits_request_for(
.map(|route| route.model.as_str())
.filter(|model| !model.is_empty())
.unwrap_or(model);
+ let context = crate::inference::registry::ProviderContext::new(
+ config, &crate::security::egress::EgressLedger::none(),
+ );
return crate::inference::registry::catalog_for(
extensions,
crate::inference::provider::factory::PROVIDER_OPENHUMAN,
)
.and_then(|entry| entry.limits)
- .and_then(|limits| limits(model, config));
+ .and_then(|limits| limits(model, &context));
}
let route = route?;
@@ -256,12 +259,15 @@ pub(crate) fn model_limits_request_for(
} else {
route.model.trim()
};
+ let context = crate::inference::registry::ProviderContext::new(
+ config, &crate::security::egress::EgressLedger::none(),
+ );
return crate::inference::registry::catalog_for(
extensions,
crate::inference::provider::factory::PROVIDER_OPENHUMAN,
)
.and_then(|entry| entry.limits)
- .and_then(|limits| limits(model, config));
+ .and_then(|limits| limits(model, &context));
}
let resolution = crate::inference::provider::factory::resolve_cloud_slug_for(
role,
diff --git a/crates/openhuman-core/src/inference/provider/models/catalog_listing.rs b/crates/openhuman-core/src/inference/provider/models/catalog_listing.rs
index a2ef9b5852..4a68401613 100644
--- a/crates/openhuman-core/src/inference/provider/models/catalog_listing.rs
+++ b/crates/openhuman-core/src/inference/provider/models/catalog_listing.rs
@@ -48,7 +48,7 @@ pub(super) fn resolve_local_runtime_key(
fn catalog_request(
config: &crate::config::Config,
entry: Option<&crate::config::schema::cloud_providers::CloudProviderCreds>,
-) -> Result<crate::inference::registry::CatalogRequest, String> {
+) -> Result<crate::inference::registry::CatalogHostRequest, String> {
let service_key = "providers.list_models";
let slug = entry.map(|e| e.slug.trim()).unwrap_or_default();
let provider_key = match entry {
@@ -70,10 +70,22 @@ fn catalog_request(
}
None => crate::config::build_runtime_proxy_client_with_timeouts(service_key, 30, 10),
};
- Ok(crate::inference::registry::CatalogRequest {
- provider_key,
- client,
- })
+ Ok(crate::inference::registry::CatalogHostRequest { provider_key, client })
+}
+
+async fn run_registered_catalog(
+ provider_id: &str,
+ config: &crate::config::Config,
+ request: crate::inference::registry::CatalogHostRequest,
+ catalog: tinyinference_llm::providers::overrides::CatalogEntry<crate::inference::registry::ProviderContext>,
+) -> Result<crate::core::Outcome<serde_json::Value>, String> {
+ let context = crate::inference::registry::ProviderContext::for_catalog(config, request);
+ let call = context.catalog.as_ref().expect("catalog context");
+ let logs = Arc::clone(&call.logs);
+ let listing = (catalog.list)(tinyinference_llm::providers::overrides::CatalogRequest { context }).await?;
+ let logs = std::mem::take(&mut *logs.lock().unwrap_or_else(std::sync::PoisonError::into_inner));
+ log::debug!("[providers][list_models] provider_id={provider_id} served by native provider catalog");
+ Ok(crate::core::Outcome::new(listing, logs))
}
/// The empty listing the managed provider answers with when no hosted catalog
@@ -120,7 +132,7 @@ pub async fn list_configured_models_from_config(
.iter()
.find(|e| e.id == provider_id || e.slug == provider_id);
let request = catalog_request(config, entry)?;
- return (catalog.list)(config.clone(), request).await;
+ return run_registered_catalog(&provider_id, config, request, catalog).await;
}
if provider_id == crate::inference::provider::factory::PROVIDER_OPENHUMAN {
return Ok(no_managed_catalog());
@@ -146,7 +158,7 @@ pub async fn list_configured_models_from_config(
) {
Some(catalog) => {
let request = catalog_request(config, Some(&entry))?;
- (catalog.list)(config.clone(), request).await
+ run_registered_catalog(PROVIDER_OPENHUMAN, config, request, catalog).await
}
None => Ok(no_managed_catalog()),
};