Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): restructure context window resolution logic

Steven Enamakel52aa5d278153
diff --git a/crates/openhuman-core/src/inference/context_window.rs b/crates/openhuman-core/src/inference/context_window.rs
index 58c1225934..fde44e0d90 100644
--- a/crates/openhuman-core/src/inference/context_window.rs
+++ b/crates/openhuman-core/src/inference/context_window.rs
@@ -1,40 +1,25 @@
-//! Resolving a turn's context window from the provider.
+//! Resolve context windows using TinyInference's route resolver.
//!
-//! The window drives pre-dispatch trimming and the compaction trigger, so a
-//! wrong one is expensive both ways: too small and a 1M-token model compacts
-//! at 128k; too large and the provider rejects the request. The order is:
-//!
-//! 1. **Config override**: a non-zero `context_window` on the matching
-//! `config.model_registry` entry.
-//! 2. **Provider-reported**: the provider's own model listing, discovered
-//! through `tinyinference_llm::model::discover` (bounded, cached per
-//! endpoint and model), lowered by any window the provider stated in a
-//! context-overflow error on that endpoint (learned by the chat adapters).
-//! Ollama's OpenAI-compatible listing carries no window, so for the local
-//! Ollama provider and any endpoint that looks like an Ollama server
-//! (including a custom OpenAI-compatible provider at `:11434/v1`) the same
-//! discovery asks the native `POST /api/show`; its result, or its failure,
-//! is cached and the whole lookup is time-bounded.
-//! 3. **Local runtime profile** for local providers (Ollama, LM Studio, ...).
-//! 4. **Static guess**: the tier aliases, the cost catalog, and the generic
-//! id-pattern table ([`super::model_context::static_context_window_for_model`]),
-//! logged as a guess at `warn` once per model.
-//!
-//! The provider-sourced value is remembered per model, so the synchronous
-//! [`super::model_context::context_window_for_model`] (usage meters, the
-//! context breakdown) reports the same window the turn used.
+//! OpenHuman supplies its configuration override, the selected provider's
+//! discovery request, catalog and managed-model facts, and the HTTP client
+//! configured with the user's CA and proxy policy. TinyInference owns route
+//! normalization, discovery, caching, remembered limits, and static fallback
+//! precedence.
-use std::collections::HashSet;
-use std::sync::{Mutex, OnceLock};
+use std::sync::OnceLock;
-use tinyinference_llm::model::context_window::{ContextWindowInput, ContextWindowResolver};
-use tinyinference_llm::model::context_window::{ResolvedWindow, WindowSource};
+use tinyinference_llm::model::context_window::{
+ ContextWindowResolver, ContextWindowRouteInput, ManagedDefaultWindow, ResolvedWindow,
+};
use tinyinference_llm::model::discover::{
- model_limits_cache, DiscoveryRequest, ModelLimitsCache, ModelListingFetcher,
+ DiscoveryRequest, ModelLimitsCache, ModelListingFetcher, model_limits_cache,
};
+use tinyinference_local::profile::LocalProviderKind;
use crate::config::Config;
+const MODEL_LIMITS_SERVICE: &str = "inference.model_limits";
+
/// The user's explicit window for `model`, from `config.model_registry`.
pub(crate) fn config_override(model: &str, config: &Config) -> Option<u64> {
let model = model.trim();
@@ -45,111 +30,19 @@ pub(crate) fn config_override(model: &str, config: &Config) -> Option<u64> {
.map(|entry| u64::from(entry.context_window))
}
-/// The process-wide resolver: precedence and the per-route/endpoint memory of
-/// provider-sourced windows live in `tinyinference-llm`.
+/// The process-wide resolver: precedence and route/model memory live in
+/// `tinyinference-llm`.
fn resolver() -> &'static ContextWindowResolver {
static RESOLVER: OnceLock<ContextWindowResolver> = OnceLock::new();
RESOLVER.get_or_init(ContextWindowResolver::new)
}
-/// The last provider-sourced window resolved for `model` in this process.
+/// The last provider-sourced window remembered for a route/model projection.
pub(crate) fn remembered_window(provider: &str, model: &str) -> Option<u64> {
resolver().remembered_window(provider, model)
}
-/// Logs a static guess at `warn` the first time per model, `debug` after.
-fn warn_static_guess(model: &str, window: Option<u64>, provider: &str) {
- static WARNED: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();
- let first = WARNED
- .get_or_init(|| Mutex::new(HashSet::new()))
- .lock()
- .unwrap_or_else(std::sync::PoisonError::into_inner)
- .insert(model.to_string());
- let route = provider.split(':').next().unwrap_or_default();
- if first {
- tracing::warn!(
- model,
- route,
- context_window = ?window,
- "[model_context] context window is a static guess — the provider did not report one"
- );
- } else {
- tracing::debug!(
- model,
- route,
- context_window = ?window,
- "[model_context] context window is a static guess"
- );
- }
-}
-
-/// Discovery request for the built-in local Ollama provider: its OpenAI
-/// listing carries no window, so the native `/api/show` probe is forced on.
-/// A custom OpenAI-compatible provider pointed at an Ollama server gets the
-/// same probe from the endpoint auto-detection in `tinyinference-llm`.
-fn ollama_limits_request(model: &str, config: &Config) -> Option<DiscoveryRequest> {
- let root = tinyinference_local::ollama::ollama_base_url_from_override(
- config.local_ai.base_url.as_deref(),
- );
- let model = model
- .split_once('@')
- .map_or(model, |(model, _)| model)
- .trim();
- if model.is_empty() {
- return None;
- }
- Some(
- DiscoveryRequest::new(format!("{}/v1", root.trim_end_matches('/')), model)
- .with_ollama_native(true),
- )
-}
-
-/// The fetcher discovery uses. Unit tests never reach the network.
-fn default_fetcher(_config: &Config, _provider: &str) -> Box<dyn ModelListingFetcher> {
- #[cfg(test)]
- {
- Box::new(OfflineFetcher)
- }
- #[cfg(not(test))]
- {
- let provider_slug = _provider
- .split_once(':')
- .map_or(_provider, |(slug, _)| slug)
- .trim();
- let service_key = "inference.model_limits";
- let client = _config
- .cloud_provider_ca_certs
- .get(provider_slug)
- .filter(|pem| !pem.is_empty())
- .and_then(|pem| {
- crate::util::tls::client_with_ca_bundle_with_timeouts(pem, service_key, 5, 3).ok()
- })
- .unwrap_or_else(|| {
- crate::config::build_runtime_proxy_client_with_timeouts(service_key, 5, 3)
- });
- Box::new(tinyinference_llm::model::discover::ReqwestListingFetcher::new(client))
- }
-}
-
-/// A fetcher that never reaches the network (every request fails).
-#[cfg(test)]
-struct OfflineFetcher;
-
-#[cfg(test)]
-#[async_trait::async_trait]
-impl ModelListingFetcher for OfflineFetcher {
- async fn get_json(
- &self,
- url: &str,
- _headers: &[(String, String)],
- ) -> tinyinference_llm::Result<serde_json::Value> {
- Err(tinyinference_llm::Error::Catalog(format!(
- "offline in unit tests: {url}"
- )))
- }
-}
-
-/// Resolve the context window for `model` on the `role`'s `provider` route.
+/// Resolves the window for `model` on the role's selected provider route.
pub(crate) async fn resolve_context_window(
role: &str,
provider: &str,
@@ -161,8 +54,8 @@ pub(crate) async fn resolve_context_window(
.window
}
-/// Resolve a context window using provider metadata from an explicitly
-/// selected runtime context.
+/// Resolves a window using provider metadata from an explicitly selected
+/// runtime context.
pub(crate) async fn resolve_context_window_in(
role: &str,
provider: &str,
@@ -182,20 +75,23 @@ async fn resolve_context_window_inner(
config: &Config,
extensions: Option<Option<&crate::core::slots::Extensions>>,
) -> ResolvedWindow {
- let fetcher = default_fetcher(config, provider);
- resolve_context_window_with_inner(
- fetcher.as_ref(),
- model_limits_cache(),
- role,
- provider,
- model,
- config,
- extensions,
- )
- .await
+ let local_kind = tinyinference_local::profile::kind_from_provider_string(provider);
+ let route = route_input(role, provider, model, config, extensions, local_kind);
+ resolver()
+ .resolve_for_route(
+ provider,
+ model,
+ local_kind,
+ route,
+ model_limits_cache(),
+ || model_limits_client(config, provider),
+ )
+ .await
}
-/// [`resolve_context_window`] with an explicit fetcher and cache.
+/// Test seam that injects an offline listing fetcher while preserving the
+/// production route-input construction.
+#[cfg(test)]
pub(crate) async fn resolve_context_window_with(
fetcher: &dyn ModelListingFetcher,
cache: &ModelLimitsCache,
@@ -204,93 +100,155 @@ pub(crate) async fn resolve_context_window_with(
model: &str,
config: &Config,
) -> ResolvedWindow {
- resolve_context_window_with_inner(fetcher, cache, role, provider, model, config, None).await
+ let local_kind = tinyinference_local::profile::kind_from_provider_string(provider);
+ let route = route_input(role, provider, model, config, None, local_kind);
+ resolver()
+ .resolve_for_route_with_fetcher(provider, model, local_kind, route, cache, fetcher)
+ .await
}
-async fn resolve_context_window_with_inner(
- fetcher: &dyn ModelListingFetcher,
- cache: &ModelLimitsCache,
+/// Host-owned route facts shared by async resolution and synchronous
+/// projections. The native resolver owns the fallback order and URL handling.
+fn route_input(
role: &str,
provider: &str,
model: &str,
config: &Config,
extensions: Option<Option<&crate::core::slots::Extensions>>,
-) -> ResolvedWindow {
- let local_kind = tinyinference_local::profile::kind_from_provider_string(provider);
- let request = match local_kind {
- None => match extensions {
- Some(extensions) => crate::inference::provider::factory::model_limits_request_for(
- role, provider, model, config, extensions,
- ),
+ local_kind: Option<LocalProviderKind>,
+) -> ContextWindowRouteInput {
+ let mut route = static_route_facts(model, extensions);
+ route.config_override = config_override(model, config);
+ route.local_window = local_kind.and_then(|kind| {
+ tinyinference_local::profile::context_window_with_local_fallback(model, None, Some(kind))
+ });
+ route.discovery = match local_kind {
+ None => Some(match extensions {
+ Some(extensions) => {
+ crate::inference::provider::factory::model_limits_request_for(
+ role, provider, model, config, extensions,
+ )
+ }
None => crate::inference::provider::factory::model_limits_request(
role, provider, model, config,
),
- },
- Some(tinyinference_local::profile::LocalProviderKind::Ollama) => {
- ollama_limits_request(model, config)
- }
+ })
+ .flatten(),
+ Some(LocalProviderKind::Ollama) => None,
Some(_) => None,
};
- let static_window = match extensions {
- Some(extensions) => {
- super::model_context::static_context_window_for_model_in(extensions, model)
- }
- None => super::model_context::static_context_window_for_model(model),
- };
- let local_window = local_kind.and_then(|kind| {
- tinyinference_local::profile::context_window_with_local_fallback(
- model,
- static_window,
- Some(kind),
- )
- });
- let resolved = resolver()
- .resolve(
+ if local_kind == Some(LocalProviderKind::Ollama) {
+ route.endpoint = Some(tinyinference_local::ollama::ollama_base_url_from_override(
+ config.local_ai.base_url.as_deref(),
+ ));
+ }
+ route
+}
+
+/// Static hints owned outside TinyInference's generic model catalog.
+fn static_route_facts(
+ model: &str,
+ extensions: Option<Option<&crate::core::slots::Extensions>>,
+) -> ContextWindowRouteInput {
+ let provider = crate::inference::provider::factory::PROVIDER_OPENHUMAN;
+ let managed_default = match extensions {
+ Some(extensions) => crate::inference::registry::provider_default_model_for(
+ extensions,
provider,
- model,
- ContextWindowInput {
- config_override: config_override(model, config),
- discovery: request.as_ref(),
- static_window,
- has_local_profile: local_kind.is_some(),
- local_window,
- },
- fetcher,
- cache,
)
- .await;
- if local_kind.is_none()
- && matches!(
- resolved.source,
- WindowSource::StaticGuess | WindowSource::Unknown
- )
- {
- warn_static_guess(model, resolved.window, provider);
+ .and_then(|managed_model| {
+ crate::inference::registry::provider_model_context_window_for(
+ extensions,
+ provider,
+ &managed_model,
+ )
+ .map(|window| ManagedDefaultWindow {
+ model: managed_model,
+ window,
+ })
+ }),
+ None => crate::inference::registry::provider_default_model(provider).and_then(
+ |managed_model| {
+ crate::inference::registry::provider_model_context_window(provider, &managed_model)
+ .map(|window| ManagedDefaultWindow {
+ model: managed_model,
+ window,
+ })
+ },
+ ),
+ };
+ ContextWindowRouteInput {
+ catalog_window: tinyagents_registry::pricing::lookup(model.trim())
+ .map(|price| u64::from(price.context_window)),
+ managed_default,
+ ..Default::default()
}
- if let (Some(window), Some(request)) = (resolved.window, request.as_ref()) {
- if matches!(
- resolved.source,
- WindowSource::ProviderReported | WindowSource::LearnedFromOverflow
- ) {
- tracing::info!(
- role,
- model = %request.model,
- endpoint = %request.endpoint,
- context_window = window,
- max_output_tokens = ?resolved.max_output_tokens,
- source = resolved.source.label(),
- "[model_context] context window from provider"
- );
- }
+}
+
+/// Builds the host-configured model-listing client lazily, only when the
+/// native resolver determines discovery can run.
+fn model_limits_client(config: &Config, provider: &str) -> reqwest::Client {
+ model_limits_client_with(
+ config,
+ provider,
+ |pem, service, read_timeout, connect_timeout| {
+ crate::util::tls::client_with_ca_bundle_with_timeouts(
+ pem,
+ service,
+ read_timeout,
+ connect_timeout,
+ )
+ .ok()
+ },
+ crate::config::build_runtime_proxy_client_with_timeouts,
+ )
+}
+
+/// Projects a synchronous context window for a provider/model route without
+/// performing discovery. A full discovery request, when available, scopes any
+/// remembered provider result to the exact endpoint and auth/cache identity.
+pub(crate) fn project_context_window(
+ role: &str,
+ provider: &str,
+ model: &str,
+ config: &Config,
+) -> ResolvedWindow {
+ let local_kind = tinyinference_local::profile::kind_from_provider_string(provider);
+ let route = route_input(role, provider, model, config, None, local_kind);
+ resolver().project_for_route(provider, model, local_kind, route)
+}
+
+/// Projects a static model-only window. The empty route key prevents a catalog
+/// or usage display from borrowing a learned limit from an unrelated provider.
+pub(crate) fn project_context_window_for_model(
+ model: &str,
+ extensions: Option<&crate::core::slots::Extensions>,
+) -> ResolvedWindow {
+ let route = static_route_facts(model, extensions.map(Some));
+ resolver().project_for_route("", model, None, route)
+}
+
+/// Creates a client that carries OpenHuman's configured CA and proxy policy.
+/// Unit tests can supply factories to verify selection without network access.
+fn model_limits_client_with(
+ config: &Config,
+ provider: &str,
+ ca_client: impl FnOnce(&str, &str, u64, u64) -> Option<reqwest::Client>,
+ proxy_client: impl FnOnce(&str, u64, u64) -> reqwest::Client,
+) -> reqwest::Client {
+ let provider_slug = provider
+ .split_once(':')
+ .map_or(provider, |(slug, _)| slug)
+ .trim();
+ if let Some(client) = config
+ .cloud_provider_ca_certs
+ .get(provider_slug)
+ .filter(|pem| !pem.is_empty())
+ .and_then(|pem| ca_client(pem, MODEL_LIMITS_SERVICE, 5, 3))
+ {
+ return client;
}
- tracing::debug!(
- role,
- model,
- context_window = ?resolved.window,
- source = resolved.source.label(),
- "[model_context] resolved context window"
- );
- resolved
+ proxy_client(MODEL_LIMITS_SERVICE, 5, 3)
}
#[cfg(test)]