The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(inference): restructure context window resolution logic
Steven Enamakel52aa5d278153
diff --git a/crates/openhuman-core/src/inference/context_window.rs b/crates/openhuman-core/src/inference/context_window.rsindex 58c1225934..fde44e0d90 100644--- a/crates/openhuman-core/src/inference/context_window.rs+++ b/crates/openhuman-core/src/inference/context_window.rs@@ -1,40 +1,25 @@-//! Resolving a turn's context window from the provider.+//! Resolve context windows using TinyInference's route resolver.//!-//! The window drives pre-dispatch trimming and the compaction trigger, so a-//! wrong one is expensive both ways: too small and a 1M-token model compacts-//! at 128k; too large and the provider rejects the request. The order is:-//!-//! 1. **Config override**: a non-zero `context_window` on the matching-//! `config.model_registry` entry.-//! 2. **Provider-reported**: the provider's own model listing, discovered-//! through `tinyinference_llm::model::discover` (bounded, cached per-//! endpoint and model), lowered by any window the provider stated in a-//! context-overflow error on that endpoint (learned by the chat adapters).-//! Ollama's OpenAI-compatible listing carries no window, so for the local-//! Ollama provider and any endpoint that looks like an Ollama server-//! (including a custom OpenAI-compatible provider at `:11434/v1`) the same-//! discovery asks the native `POST /api/show`; its result, or its failure,-//! is cached and the whole lookup is time-bounded.-//! 3. **Local runtime profile** for local providers (Ollama, LM Studio, ...).-//! 4. **Static guess**: the tier aliases, the cost catalog, and the generic-//! id-pattern table ([`super::model_context::static_context_window_for_model`]),-//! logged as a guess at `warn` once per model.-//!-//! The provider-sourced value is remembered per model, so the synchronous-//! [`super::model_context::context_window_for_model`] (usage meters, the-//! context breakdown) reports the same window the turn used.+//! OpenHuman supplies its configuration override, the selected provider's+//! discovery request, catalog and managed-model facts, and the HTTP client+//! configured with the user's CA and proxy policy. TinyInference owns route+//! normalization, discovery, caching, remembered limits, and static fallback+//! precedence.-use std::collections::HashSet;-use std::sync::{Mutex, OnceLock};+use std::sync::OnceLock;-use tinyinference_llm::model::context_window::{ContextWindowInput, ContextWindowResolver};-use tinyinference_llm::model::context_window::{ResolvedWindow, WindowSource};+use tinyinference_llm::model::context_window::{+ ContextWindowResolver, ContextWindowRouteInput, ManagedDefaultWindow, ResolvedWindow,+};use tinyinference_llm::model::discover::{- model_limits_cache, DiscoveryRequest, ModelLimitsCache, ModelListingFetcher,+ DiscoveryRequest, ModelLimitsCache, ModelListingFetcher, model_limits_cache,};+use tinyinference_local::profile::LocalProviderKind;use crate::config::Config;+const MODEL_LIMITS_SERVICE: &str = "inference.model_limits";+/// The user's explicit window for `model`, from `config.model_registry`.pub(crate) fn config_override(model: &str, config: &Config) -> Option<u64> {let model = model.trim();@@ -45,111 +30,19 @@ pub(crate) fn config_override(model: &str, config: &Config) -> Option<u64> {.map(|entry| u64::from(entry.context_window))}-/// The process-wide resolver: precedence and the per-route/endpoint memory of-/// provider-sourced windows live in `tinyinference-llm`.+/// The process-wide resolver: precedence and route/model memory live in+/// `tinyinference-llm`.fn resolver() -> &'static ContextWindowResolver {static RESOLVER: OnceLock<ContextWindowResolver> = OnceLock::new();RESOLVER.get_or_init(ContextWindowResolver::new)}-/// The last provider-sourced window resolved for `model` in this process.+/// The last provider-sourced window remembered for a route/model projection.pub(crate) fn remembered_window(provider: &str, model: &str) -> Option<u64> {resolver().remembered_window(provider, model)}-/// Logs a static guess at `warn` the first time per model, `debug` after.-fn warn_static_guess(model: &str, window: Option<u64>, provider: &str) {- static WARNED: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();- let first = WARNED- .get_or_init(|| Mutex::new(HashSet::new()))- .lock()- .unwrap_or_else(std::sync::PoisonError::into_inner)- .insert(model.to_string());- let route = provider.split(':').next().unwrap_or_default();- if first {- tracing::warn!(- model,- route,- context_window = ?window,- "[model_context] context window is a static guess — the provider did not report one"- );- } else {- tracing::debug!(- model,- route,- context_window = ?window,- "[model_context] context window is a static guess"- );- }-}--/// Discovery request for the built-in local Ollama provider: its OpenAI-/// listing carries no window, so the native `/api/show` probe is forced on.-/// A custom OpenAI-compatible provider pointed at an Ollama server gets the-/// same probe from the endpoint auto-detection in `tinyinference-llm`.-fn ollama_limits_request(model: &str, config: &Config) -> Option<DiscoveryRequest> {- let root = tinyinference_local::ollama::ollama_base_url_from_override(- config.local_ai.base_url.as_deref(),- );- let model = model- .split_once('@')- .map_or(model, |(model, _)| model)- .trim();- if model.is_empty() {- return None;- }- Some(- DiscoveryRequest::new(format!("{}/v1", root.trim_end_matches('/')), model)- .with_ollama_native(true),- )-}--/// The fetcher discovery uses. Unit tests never reach the network.-fn default_fetcher(_config: &Config, _provider: &str) -> Box<dyn ModelListingFetcher> {- #[cfg(test)]- {- Box::new(OfflineFetcher)- }- #[cfg(not(test))]- {- let provider_slug = _provider- .split_once(':')- .map_or(_provider, |(slug, _)| slug)- .trim();- let service_key = "inference.model_limits";- let client = _config- .cloud_provider_ca_certs- .get(provider_slug)- .filter(|pem| !pem.is_empty())- .and_then(|pem| {- crate::util::tls::client_with_ca_bundle_with_timeouts(pem, service_key, 5, 3).ok()- })- .unwrap_or_else(|| {- crate::config::build_runtime_proxy_client_with_timeouts(service_key, 5, 3)- });- Box::new(tinyinference_llm::model::discover::ReqwestListingFetcher::new(client))- }-}--/// A fetcher that never reaches the network (every request fails).-#[cfg(test)]-struct OfflineFetcher;--#[cfg(test)]-#[async_trait::async_trait]-impl ModelListingFetcher for OfflineFetcher {- async fn get_json(- &self,- url: &str,- _headers: &[(String, String)],- ) -> tinyinference_llm::Result<serde_json::Value> {- Err(tinyinference_llm::Error::Catalog(format!(- "offline in unit tests: {url}"- )))- }-}--/// Resolve the context window for `model` on the `role`'s `provider` route.+/// Resolves the window for `model` on the role's selected provider route.pub(crate) async fn resolve_context_window(role: &str,provider: &str,@@ -161,8 +54,8 @@ pub(crate) async fn resolve_context_window(.window}-/// Resolve a context window using provider metadata from an explicitly-/// selected runtime context.+/// Resolves a window using provider metadata from an explicitly selected+/// runtime context.pub(crate) async fn resolve_context_window_in(role: &str,provider: &str,@@ -182,20 +75,23 @@ async fn resolve_context_window_inner(config: &Config,extensions: Option<Option<&crate::core::slots::Extensions>>,) -> ResolvedWindow {- let fetcher = default_fetcher(config, provider);- resolve_context_window_with_inner(- fetcher.as_ref(),- model_limits_cache(),- role,- provider,- model,- config,- extensions,- )- .await+ let local_kind = tinyinference_local::profile::kind_from_provider_string(provider);+ let route = route_input(role, provider, model, config, extensions, local_kind);+ resolver()+ .resolve_for_route(+ provider,+ model,+ local_kind,+ route,+ model_limits_cache(),+ || model_limits_client(config, provider),+ )+ .await}-/// [`resolve_context_window`] with an explicit fetcher and cache.+/// Test seam that injects an offline listing fetcher while preserving the+/// production route-input construction.+#[cfg(test)]pub(crate) async fn resolve_context_window_with(fetcher: &dyn ModelListingFetcher,cache: &ModelLimitsCache,@@ -204,93 +100,155 @@ pub(crate) async fn resolve_context_window_with(model: &str,config: &Config,) -> ResolvedWindow {- resolve_context_window_with_inner(fetcher, cache, role, provider, model, config, None).await+ let local_kind = tinyinference_local::profile::kind_from_provider_string(provider);+ let route = route_input(role, provider, model, config, None, local_kind);+ resolver()+ .resolve_for_route_with_fetcher(provider, model, local_kind, route, cache, fetcher)+ .await}-async fn resolve_context_window_with_inner(- fetcher: &dyn ModelListingFetcher,- cache: &ModelLimitsCache,+/// Host-owned route facts shared by async resolution and synchronous+/// projections. The native resolver owns the fallback order and URL handling.+fn route_input(role: &str,provider: &str,model: &str,config: &Config,extensions: Option<Option<&crate::core::slots::Extensions>>,-) -> ResolvedWindow {- let local_kind = tinyinference_local::profile::kind_from_provider_string(provider);- let request = match local_kind {- None => match extensions {- Some(extensions) => crate::inference::provider::factory::model_limits_request_for(- role, provider, model, config, extensions,- ),+ local_kind: Option<LocalProviderKind>,+) -> ContextWindowRouteInput {+ let mut route = static_route_facts(model, extensions);+ route.config_override = config_override(model, config);+ route.local_window = local_kind.and_then(|kind| {+ tinyinference_local::profile::context_window_with_local_fallback(model, None, Some(kind))+ });+ route.discovery = match local_kind {+ None => Some(match extensions {+ Some(extensions) => {+ crate::inference::provider::factory::model_limits_request_for(+ role, provider, model, config, extensions,+ )+ }None => crate::inference::provider::factory::model_limits_request(role, provider, model, config,),- },- Some(tinyinference_local::profile::LocalProviderKind::Ollama) => {- ollama_limits_request(model, config)- }+ })+ .flatten(),+ Some(LocalProviderKind::Ollama) => None,Some(_) => None,};- let static_window = match extensions {- Some(extensions) => {- super::model_context::static_context_window_for_model_in(extensions, model)- }- None => super::model_context::static_context_window_for_model(model),- };- let local_window = local_kind.and_then(|kind| {- tinyinference_local::profile::context_window_with_local_fallback(- model,- static_window,- Some(kind),- )- });- let resolved = resolver()- .resolve(+ if local_kind == Some(LocalProviderKind::Ollama) {+ route.endpoint = Some(tinyinference_local::ollama::ollama_base_url_from_override(+ config.local_ai.base_url.as_deref(),+ ));+ }+ route+}++/// Static hints owned outside TinyInference's generic model catalog.+fn static_route_facts(+ model: &str,+ extensions: Option<Option<&crate::core::slots::Extensions>>,+) -> ContextWindowRouteInput {+ let provider = crate::inference::provider::factory::PROVIDER_OPENHUMAN;+ let managed_default = match extensions {+ Some(extensions) => crate::inference::registry::provider_default_model_for(+ extensions,provider,- model,- ContextWindowInput {- config_override: config_override(model, config),- discovery: request.as_ref(),- static_window,- has_local_profile: local_kind.is_some(),- local_window,- },- fetcher,- cache,)- .await;- if local_kind.is_none()- && matches!(- resolved.source,- WindowSource::StaticGuess | WindowSource::Unknown- )- {- warn_static_guess(model, resolved.window, provider);+ .and_then(|managed_model| {+ crate::inference::registry::provider_model_context_window_for(+ extensions,+ provider,+ &managed_model,+ )+ .map(|window| ManagedDefaultWindow {+ model: managed_model,+ window,+ })+ }),+ None => crate::inference::registry::provider_default_model(provider).and_then(+ |managed_model| {+ crate::inference::registry::provider_model_context_window(provider, &managed_model)+ .map(|window| ManagedDefaultWindow {+ model: managed_model,+ window,+ })+ },+ ),+ };+ ContextWindowRouteInput {+ catalog_window: tinyagents_registry::pricing::lookup(model.trim())+ .map(|price| u64::from(price.context_window)),+ managed_default,+ ..Default::default()}- if let (Some(window), Some(request)) = (resolved.window, request.as_ref()) {- if matches!(- resolved.source,- WindowSource::ProviderReported | WindowSource::LearnedFromOverflow- ) {- tracing::info!(- role,- model = %request.model,- endpoint = %request.endpoint,- context_window = window,- max_output_tokens = ?resolved.max_output_tokens,- source = resolved.source.label(),- "[model_context] context window from provider"- );- }+}++/// Builds the host-configured model-listing client lazily, only when the+/// native resolver determines discovery can run.+fn model_limits_client(config: &Config, provider: &str) -> reqwest::Client {+ model_limits_client_with(+ config,+ provider,+ |pem, service, read_timeout, connect_timeout| {+ crate::util::tls::client_with_ca_bundle_with_timeouts(+ pem,+ service,+ read_timeout,+ connect_timeout,+ )+ .ok()+ },+ crate::config::build_runtime_proxy_client_with_timeouts,+ )+}++/// Projects a synchronous context window for a provider/model route without+/// performing discovery. A full discovery request, when available, scopes any+/// remembered provider result to the exact endpoint and auth/cache identity.+pub(crate) fn project_context_window(+ role: &str,+ provider: &str,+ model: &str,+ config: &Config,+) -> ResolvedWindow {+ let local_kind = tinyinference_local::profile::kind_from_provider_string(provider);+ let route = route_input(role, provider, model, config, None, local_kind);+ resolver().project_for_route(provider, model, local_kind, route)+}++/// Projects a static model-only window. The empty route key prevents a catalog+/// or usage display from borrowing a learned limit from an unrelated provider.+pub(crate) fn project_context_window_for_model(+ model: &str,+ extensions: Option<&crate::core::slots::Extensions>,+) -> ResolvedWindow {+ let route = static_route_facts(model, extensions.map(Some));+ resolver().project_for_route("", model, None, route)+}++/// Creates a client that carries OpenHuman's configured CA and proxy policy.+/// Unit tests can supply factories to verify selection without network access.+fn model_limits_client_with(+ config: &Config,+ provider: &str,+ ca_client: impl FnOnce(&str, &str, u64, u64) -> Option<reqwest::Client>,+ proxy_client: impl FnOnce(&str, u64, u64) -> reqwest::Client,+) -> reqwest::Client {+ let provider_slug = provider+ .split_once(':')+ .map_or(provider, |(slug, _)| slug)+ .trim();+ if let Some(client) = config+ .cloud_provider_ca_certs+ .get(provider_slug)+ .filter(|pem| !pem.is_empty())+ .and_then(|pem| ca_client(pem, MODEL_LIMITS_SERVICE, 5, 3))+ {+ return client;}- tracing::debug!(- role,- model,- context_window = ?resolved.window,- source = resolved.source.label(),- "[model_context] resolved context window"- );- resolved+ proxy_client(MODEL_LIMITS_SERVICE, 5, 3)}#[cfg(test)]