Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): update context window projection logic

Steven Enamakeld555cee806a8
diff --git a/crates/openhuman-core/src/inference/context_window.rs b/crates/openhuman-core/src/inference/context_window.rs
index fde44e0d90..05bbbb0b4c 100644
--- a/crates/openhuman-core/src/inference/context_window.rs
+++ b/crates/openhuman-core/src/inference/context_window.rs
@@ -123,7 +123,7 @@ fn route_input(
tinyinference_local::profile::context_window_with_local_fallback(model, None, Some(kind))
});
route.discovery = match local_kind {
- None => Some(match extensions {
+ None => match extensions {
Some(extensions) => {
crate::inference::provider::factory::model_limits_request_for(
role, provider, model, config, extensions,
@@ -132,8 +132,7 @@ fn route_input(
None => crate::inference::provider::factory::model_limits_request(
role, provider, model, config,
),
- })
- .flatten(),
+ },
Some(LocalProviderKind::Ollama) => None,
Some(_) => None,
};
diff --git a/crates/openhuman-core/src/inference/model_context.rs b/crates/openhuman-core/src/inference/model_context.rs
index a2c1350d7c..c32c1b8f23 100644
--- a/crates/openhuman-core/src/inference/model_context.rs
+++ b/crates/openhuman-core/src/inference/model_context.rs
@@ -1,21 +1,12 @@
-//! Known model context-window sizes for pre-inference budgeting.
-//!
-//! Provider `/models` responses may include `context_length` / `context_window`,
-//! but the agent harness must enforce limits **before** the first dispatch —
-//! otherwise long histories produce upstream `400 Bad Request` errors when usage
-//! metadata is not yet available.
+//! Context-window projections and user-configured model capabilities.
-/// Resolve the context window (in tokens) for a model id or registered provider alias.
+/// Resolves a model-only context window using owner-provided static hints.
///
-/// Returns the static model estimate. Call [`context_window_for_route`] when
-/// the provider route is known; provider-discovered values are stored per
-/// route and model. Thread usage prefers its persisted per-turn value before
-/// consulting either lookup.
-///
-/// Returns `None` when the model is unknown — callers should skip pre-dispatch
-/// trimming rather than guess.
+/// This projection uses an empty route identity so it cannot borrow provider
+/// discovery from an unrelated endpoint. Returns `None` when no owner or
+/// managed-provider hint is available.
pub fn context_window_for_model(model: &str) -> Option<u64> {
- static_context_window_for_model(model)
+ crate::inference::context_window::project_context_window_for_model(model, None).window
}
/// Resolve the synchronous window for a model on a specific provider route.
@@ -24,79 +15,20 @@ pub(crate) fn context_window_for_route(
model: &str,
config: &crate::config::Config,
) -> Option<u64> {
- crate::inference::context_window::config_override(model, config)
- .or_else(|| crate::inference::context_window::remembered_window(provider, model))
- .or_else(|| static_context_window_for_model(model))
+ crate::inference::context_window::project_context_window("chat", provider, model, config).window
}
-/// The static guess for a model's window: provider metadata, then the cost catalog,
-/// then the generic id-pattern hints. These never override a provider-reported
-/// window; [`crate::inference::context_window`] consults them last.
+/// Projects static owner hints for a model without a provider route.
pub(crate) fn static_context_window_for_model(model: &str) -> Option<u64> {
- static_context_window_for_model_with_provider_window(
- model,
- crate::inference::registry::provider_model_context_window(
- crate::inference::provider::factory::PROVIDER_OPENHUMAN,
- model.trim(),
- ),
- )
+ crate::inference::context_window::project_context_window_for_model(model, None).window
}
-/// Context-aware counterpart used when a dispatch carries its own provider
-/// registry. An empty context does not borrow model metadata from ambient work.
+/// Context-aware counterpart for a dispatch with its own provider registry.
pub(crate) fn static_context_window_for_model_in(
extensions: Option<&crate::core::slots::Extensions>,
model: &str,
) -> Option<u64> {
- static_context_window_for_model_with_provider_window(
- model,
- crate::inference::registry::provider_model_context_window_for(
- extensions,
- crate::inference::provider::factory::PROVIDER_OPENHUMAN,
- model.trim(),
- ),
- )
-}
-
-fn static_context_window_for_model_with_provider_window(
- model: &str,
- provider_window: Option<u64>,
-) -> Option<u64> {
- let normalized = model.trim();
- if normalized.is_empty() {
- return None;
- }
-
- if let Some(window) = provider_window {
- return Some(window);
- }
-
- // Keep conservative estimates for generic small-model ids.
- if normalized.starts_with("gemma") || normalized.contains(":1b") || normalized.contains("270m")
- {
- return Some(8_192);
- }
-
- if let Some(price) = tinyagents_registry::pricing::lookup(normalized) {
- tracing::debug!(
- model = normalized,
- catalog_model = price.model_id,
- context_window = price.context_window,
- "[model_context] matched cost catalog row"
- );
- return Some(u64::from(price.context_window));
- }
-
- if let Some(window) = tinyinference_llm::model::context_window_for_model_id(normalized) {
- tracing::debug!(
- model = normalized,
- context_window = window,
- "[model_context] matched tinyagents model context hint"
- );
- return Some(window);
- }
-
- None
+ crate::inference::context_window::project_context_window_for_model(model, extensions).window
}
/// Whether the model resolved for a chat hint/agent/profile accepts image input
diff --git a/crates/openhuman-core/src/inference/provider_model_tests.rs b/crates/openhuman-core/src/inference/provider_model_tests.rs
index 77758698d7..4bc52af906 100644
--- a/crates/openhuman-core/src/inference/provider_model_tests.rs
+++ b/crates/openhuman-core/src/inference/provider_model_tests.rs
@@ -58,6 +58,14 @@ fn registered_model_defaults_and_aliases_are_context_scoped() {
}),
},
);
+ assert_eq!(
+ crate::inference::registry::provider_default_model_for(
+ Some(&extensions),
+ "scoped-provider"
+ )
+ .as_deref(),
+ Some("scoped/model")
+ );
let context = crate::core::runtime::CoreContext::for_test_with_extensions(
crate::core::runtime::DomainSet::full(),
None,
diff --git a/crates/openhuman-core/src/inference/registry.rs b/crates/openhuman-core/src/inference/registry.rs
index 77f9d38b15..939105ddb1 100644
--- a/crates/openhuman-core/src/inference/registry.rs
+++ b/crates/openhuman-core/src/inference/registry.rs
@@ -229,6 +229,16 @@ pub fn provider_default_model(provider_id: &str) -> Option<String> {
(hooks.default_model)()
}
+/// The registered provider's default model selected by an explicit context,
+/// falling back to process registration when that context has no entry.
+pub(crate) fn provider_default_model_for(
+ extensions: Option<&Extensions>,
+ provider_id: &str,
+) -> Option<String> {
+ let hooks = chat_model_for(extensions, provider_id)?.models?;
+ (hooks.default_model)()
+}
+
/// Default pricing supplied by the registered provider, if any.
pub fn provider_default_pricing(provider_id: &str) -> Option<(String, f64, f64)> {
let hooks = provider_model_hooks(provider_id)?;