Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): move model limits request logic to context_window

Steven Enamakele21251ede008
diff --git a/crates/openhuman-core/src/attachments/provider.rs b/crates/openhuman-core/src/attachments/provider.rs
index e83cebe4a3..17149c82dc 100644
--- a/crates/openhuman-core/src/attachments/provider.rs
+++ b/crates/openhuman-core/src/attachments/provider.rs
@@ -50,7 +50,7 @@ pub(crate) fn wrap(
format!("{provider}:{model}")
};
if let Some(request) =
- crate::inference::provider::factory::model_limits_request("chat", &route, model, &config)
+ crate::inference::context_window::model_limits_request("chat", &route, model, &config)
{
if let Some(inputs) = tinyinference_llm::model::discover::model_limits_cache()
.get_variant(&request.endpoint, &request.model, &request.cache_variant())
diff --git a/crates/openhuman-core/src/attachments/provider_tests.rs b/crates/openhuman-core/src/attachments/provider_tests.rs
index 9ee636fa2a..388f82d812 100644
--- a/crates/openhuman-core/src/attachments/provider_tests.rs
+++ b/crates/openhuman-core/src/attachments/provider_tests.rs
@@ -321,7 +321,7 @@ fn endpoint_discovery_denial_overrides_offline_vision_facts() {
auth_style: AuthStyle::None,
..Default::default()
}];
- let request = crate::inference::provider::factory::model_limits_request(
+ let request = crate::inference::context_window::model_limits_request(
"chat",
"attachment-probe:gpt-4o",
"gpt-4o",
diff --git a/crates/openhuman-core/src/inference/README.md b/crates/openhuman-core/src/inference/README.md
index 163ad40f00..92264e93c2 100644
--- a/crates/openhuman-core/src/inference/README.md
+++ b/crates/openhuman-core/src/inference/README.md
@@ -143,7 +143,7 @@ caching, remembered route limits, and the shared fallback order:
local profile fallback
```
-The discovery request comes from [`provider/factory/discovery.rs`](./provider/factory/discovery.rs) (`model_limits_request`), built
+The host context-window resolver builds the discovery request at the point it needs it (`context_window.rs`), using
from the same endpoint and credential resolution the chat factory uses. Route-aware synchronous
projection uses the exact discovery identity when available; model-only displays use a conservative
route projection and cannot borrow a learned limit from an unrelated provider. [`model_context.rs`](./model_context.rs) also answers vision capability through
diff --git a/crates/openhuman-core/src/inference/context_window.rs b/crates/openhuman-core/src/inference/context_window.rs
index bdedd3f975..154cc7783d 100644
--- a/crates/openhuman-core/src/inference/context_window.rs
+++ b/crates/openhuman-core/src/inference/context_window.rs
@@ -178,9 +178,7 @@ fn route_input(
});
route.discovery = match local_kind {
None => match extensions {
- Some(extensions) => {
- model_limits_request_for(role, provider, model, config, extensions)
- }
+ Some(extensions) => model_limits_request_for(role, provider, model, config, extensions),
None => model_limits_request(role, provider, model, config),
},
Some(LocalProviderKind::Ollama) => None,
@@ -245,9 +243,14 @@ pub(crate) fn model_limits_request_for(
let RouteKind::Cloud { slug } = route.kind() else {
return None;
};
- let entry = config.cloud_providers.iter().find(|entry| entry.slug == slug)?;
- if crate::config::schema::cloud_providers::is_hosted_auth_style_for(extensions, entry.auth_style)
- {
+ let entry = config
+ .cloud_providers
+ .iter()
+ .find(|entry| entry.slug == slug)?;
+ if crate::config::schema::cloud_providers::is_hosted_auth_style_for(
+ extensions,
+ entry.auth_style,
+ ) {
let model = if route.model.trim().is_empty() {
model
} else {
@@ -280,9 +283,9 @@ pub(crate) fn model_limits_request_for(
codex: Some(&resolution.codex),
..ProviderBuildOptions::default()
};
- ProviderRegistry::new(crate::inference::provider::factory::CloudCredentialResolver(
- &resolution.key,
- ))
+ ProviderRegistry::new(
+ crate::inference::provider::factory::CloudCredentialResolver(&resolution.key),
+ )
.discovery_request(&spec, &options)
}
diff --git a/crates/openhuman-core/src/inference/context_window_discovery_tests.rs b/crates/openhuman-core/src/inference/context_window_discovery_tests.rs
index 1d0d59b18b..9b39dfe1f7 100644
--- a/crates/openhuman-core/src/inference/context_window_discovery_tests.rs
+++ b/crates/openhuman-core/src/inference/context_window_discovery_tests.rs
@@ -1,8 +1,6 @@
use super::*;
use crate::config::schema::cloud_providers::{AuthStyle, CloudProviderCreds};
-use crate::inference::provider::factory::{
- BYOK_INCOMPLETE_SENTINEL, CLAUDE_AGENT_SDK_PROVIDER,
-};
+use crate::inference::provider::factory::{BYOK_INCOMPLETE_SENTINEL, CLAUDE_AGENT_SDK_PROVIDER};
use crate::security::credentials::AuthService;
use tempfile::TempDir;
diff --git a/crates/openhuman-core/src/inference/provider/factory.rs b/crates/openhuman-core/src/inference/provider/factory.rs
index d2ba2c388b..a74330242c 100644
--- a/crates/openhuman-core/src/inference/provider/factory.rs
+++ b/crates/openhuman-core/src/inference/provider/factory.rs
@@ -27,12 +27,12 @@ pub use chat_model::{
create_chat_model_in, create_chat_model_with_model_id_in, probe_inference_readiness,
};
pub(crate) use chat_model::{resolves_to_managed_backend, OpenHumanModelFactory};
+pub(crate) use cloud_slug::resolve_cloud_slug_for;
+pub(crate) use cloud_spec::{cloud_provider_spec, CloudCredentialResolver};
pub use credentials::{auth_key_for_slug, lookup_key_for_slug};
pub(crate) use credentials::{
openai_bearer_is_oauth, redact_endpoint_origin, resolve_codex_routing,
};
-pub(crate) use cloud_slug::resolve_cloud_slug_for;
-pub(crate) use cloud_spec::{cloud_provider_spec, CloudCredentialResolver};
pub(crate) use local_runtime::create_local_chat_model_from_string;
pub(crate) use managed_backend::make_openhuman_backend_model;
pub(crate) use routing::role_uses_implicit_cloud_fallback;
diff --git a/crates/openhuman-core/src/inference/provider/factory/discovery.rs b/crates/openhuman-core/src/inference/provider/factory/discovery.rs
deleted file mode 100644
index 5f6e52bb0d..0000000000
--- a/crates/openhuman-core/src/inference/provider/factory/discovery.rs
+++ /dev/null
@@ -1,139 +0,0 @@
-//! Where to ask a provider for a model's limits: the [`DiscoveryRequest`] for a
-//! role's provider string, built from the same endpoint and credential
-//! resolution the chat factory uses, so the discovered window describes the
-//! endpoint the turn actually talks to.
-
-use super::*;
-use tinyinference_llm::model::discover::DiscoveryRequest;
-use tinyinference_llm::providers::registry::{ProviderBuildOptions, ProviderRegistry};
-use tinyinference_providers::route::{ProviderRoute, RouteKind};
-
-/// The discovery request for `provider` (a role's resolved provider string)
-/// serving `model`, or `None` when the route has no listing to ask:
-///
-/// - `openhuman` / an `OpenhumanJwt` entry: the registered managed catalog's
-/// limits request, keyed by the managed inference base so overflow
-/// corrections from the managed chat model land on the same entry.
-/// - `<slug>:<model>`: the cloud entry's OpenAI-compatible `{endpoint}/models`
-/// with its stored key (Bearer, or `x-api-key` for Anthropic).
-/// - Local runtimes, subprocess providers, ChatGPT-OAuth (Codex) routing and
-/// the BYOK-incomplete sentinel: `None` (they have their own profiles, or no
-/// listing that reports windows).
-pub(crate) fn model_limits_request(
- role: &str,
- provider: &str,
- model: &str,
- config: &Config,
-) -> Option<DiscoveryRequest> {
- let extensions = crate::core::runtime::CoreContext::current_extensions();
- model_limits_request_for(role, provider, model, config, extensions.as_ref())
-}
-
-pub(crate) fn model_limits_request_for(
- role: &str,
- provider: &str,
- model: &str,
- config: &Config,
- extensions: Option<&crate::core::slots::Extensions>,
-) -> Option<DiscoveryRequest> {
- let p = provider.trim();
- if p.is_empty() || p == "cloud" {
- return managed_limits_request(model, config, extensions);
- }
- let route = ProviderRoute::parse(p);
- if p == PROVIDER_OPENHUMAN
- || route
- .as_ref()
- .is_some_and(|route| matches!(route.kind(), RouteKind::Managed))
- {
- return managed_limits_request(
- route
- .as_ref()
- .map(|route| route.model.as_str())
- .filter(|model| !model.is_empty())
- .unwrap_or(model),
- config,
- extensions,
- );
- }
- let Some(route) = route else {
- log::debug!("[model_limits] provider route has no discoverable listing role={role}");
- return None;
- };
- let RouteKind::Cloud { slug } = route.kind() else {
- log::debug!("[model_limits] provider route has no discoverable listing role={role}");
- return None;
- };
- let entry = config.cloud_providers.iter().find(|e| e.slug == slug)?;
- if crate::config::schema::cloud_providers::is_hosted_auth_style_for(
- extensions,
- entry.auth_style,
- ) {
- let model = if route.model.trim().is_empty() {
- model
- } else {
- route.model.trim()
- };
- return managed_limits_request(model, config, extensions);
- }
-
- let resolution = match resolve_cloud_slug_for(role, &slug, &route.model, config, extensions) {
- Ok(resolution) => resolution,
- Err(error) => {
- log::debug!(
- "[model_limits] cloud slug unresolvable for discovery role={role} slug={slug}: {error}"
- );
- return None;
- }
- };
- if resolution.codex.using_oauth {
- // The ChatGPT backend's catalogue is a different shape and carries no
- // per-model window; the Codex model hints cover it.
- log::debug!("[model_limits] codex oauth routing has no window listing slug={slug}");
- return None;
- }
- if resolution.entry.auth_style == AuthStyle::OpenhumanJwt {
- return None;
- }
- let spec =
- super::cloud_spec::cloud_provider_spec(resolution.entry, &resolution.effective_model);
- let options = ProviderBuildOptions {
- auth_style: Some(resolution.entry.auth_style.into()),
- codex: Some(&resolution.codex),
- ..ProviderBuildOptions::default()
- };
- match ProviderRegistry::new(super::cloud_spec::CloudCredentialResolver(&resolution.key))
- .discovery_request(&spec, &options)
- {
- Ok(request) => request,
- Err(error) => {
- log::debug!(
- "[model_limits] registry discovery unavailable slug={}: {error}",
- spec.provider
- );
- None
- }
- }
-}
-
-/// The managed backend's catalogue listing request, from the registered
-/// catalog's limits function (`inference::registry`). With none registered the
-/// managed route has no listing to ask.
-fn managed_limits_request(
- model: &str,
- config: &Config,
- extensions: Option<&crate::core::slots::Extensions>,
-) -> Option<DiscoveryRequest> {
- let Some(limits) = crate::inference::registry::catalog_for(extensions, PROVIDER_OPENHUMAN)
- .and_then(|entry| entry.limits)
- else {
- log::debug!("[model_limits] no managed catalog registered");
- return None;
- };
- log::debug!("[model_limits] managed limits served by a registered catalog");
- limits(model, config)
-}
-
-#[cfg(test)]
-#[path = "discovery_tests.rs"]
-mod tests;
diff --git a/crates/openhuman-core/src/inference/provider/factory_explicit_ctx_tests.rs b/crates/openhuman-core/src/inference/provider/factory_explicit_ctx_tests.rs
index ceecd0a25b..648513afe9 100644
--- a/crates/openhuman-core/src/inference/provider/factory_explicit_ctx_tests.rs
+++ b/crates/openhuman-core/src/inference/provider/factory_explicit_ctx_tests.rs
@@ -217,7 +217,7 @@ async fn hosted_cloud_slug_uses_explicit_auth_policy_and_model_entry() {
let observed = CoreContext::scope(context_b, async {
let resolved = dispatch.resolved().expect("bound dispatch context");
- let request = crate::inference::provider::factory::model_limits_request_for(
+ let request = crate::inference::context_window::model_limits_request_for(
"chat",
"openhuman",
"fixture/model-v1",
diff --git a/crates/openhuman-core/src/inference/registry_tests.rs b/crates/openhuman-core/src/inference/registry_tests.rs
index 88a5c97a5e..71f65395dc 100644
--- a/crates/openhuman-core/src/inference/registry_tests.rs
+++ b/crates/openhuman-core/src/inference/registry_tests.rs
@@ -1,7 +1,7 @@
use super::*;
+use crate::inference::context_window::model_limits_request;
use crate::inference::provider::factory::{
- create_chat_model_from_string, make_openhuman_backend_model_for_thread, model_limits_request,
- PROVIDER_OPENHUMAN,
+ create_chat_model_from_string, make_openhuman_backend_model_for_thread, PROVIDER_OPENHUMAN,
};
use async_trait::async_trait;
use std::sync::atomic::{AtomicUsize, Ordering};