Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): enhance context window handling for provider models

Steven Enamakel90d11b30bfb1
diff --git a/crates/openhuman-core/src/inference/README.md b/crates/openhuman-core/src/inference/README.md
index 7bd18b1c43..2a7b4e9636 100644
--- a/crates/openhuman-core/src/inference/README.md
+++ b/crates/openhuman-core/src/inference/README.md
@@ -139,8 +139,9 @@ caching, remembered route limits, and the shared fallback order:
2. provider-reported discovered limits and learned overflow limits for
the exact endpoint/model/cache identity
3. remembered provider limits for the route/model identity
- 4. static/local managed default, workload tier, pricing catalog,
- model-id hint, then local profile fallback
+ 4. static/local registered provider model metadata, managed default,
+ workload tier, pricing catalog, model-id hint, then
+ local profile fallback
```
The discovery request comes from [`provider/factory/discovery.rs`](./provider/factory/discovery.rs) (`model_limits_request`), built
diff --git a/crates/openhuman-core/src/inference/context_window.rs b/crates/openhuman-core/src/inference/context_window.rs
index 2b7aa8794e..8b95732d0d 100644
--- a/crates/openhuman-core/src/inference/context_window.rs
+++ b/crates/openhuman-core/src/inference/context_window.rs
@@ -176,6 +176,14 @@ fn static_route_facts(
extensions: Option<Option<&crate::core::slots::Extensions>>,
) -> ContextWindowRouteInput {
let provider = crate::inference::provider::factory::PROVIDER_OPENHUMAN;
+ let provider_model_window = match extensions {
+ Some(extensions) => crate::inference::registry::provider_model_context_window_for(
+ extensions,
+ provider,
+ model.trim(),
+ ),
+ None => crate::inference::registry::provider_model_context_window(provider, model.trim()),
+ };
let managed_default = match extensions {
Some(extensions) => crate::inference::registry::provider_default_model_for(
extensions, provider,
@@ -204,6 +212,7 @@ fn static_route_facts(
ContextWindowRouteInput {
catalog_window: tinyagents_registry::pricing::lookup(model.trim())
.map(|price| u64::from(price.context_window)),
+ provider_model_window,
managed_default,
..Default::default()
}
diff --git a/crates/openhuman-core/src/inference/provider_model_tests.rs b/crates/openhuman-core/src/inference/provider_model_tests.rs
index 24d98452a7..6cc90d4f65 100644
--- a/crates/openhuman-core/src/inference/provider_model_tests.rs
+++ b/crates/openhuman-core/src/inference/provider_model_tests.rs
@@ -52,7 +52,11 @@ fn registered_model_defaults_and_aliases_are_context_scoped() {
is_raw_model: Arc::new(|model| model == "scoped/model"),
supports_vision: Arc::new(|_| false),
resolve_model: Arc::new(|_, _, _| None),
- context_window: Arc::new(|model| (model == "scoped/model").then_some(77_777)),
+ context_window: Arc::new(|model| match model {
+ "scoped/model" => Some(77_777),
+ "hint:reasoning" => Some(88_888),
+ _ => None,
+ }),
default_pricing: Arc::new(|| None),
without_reasoning: Arc::new(|request| request),
}),
@@ -67,12 +71,22 @@ fn registered_model_defaults_and_aliases_are_context_scoped() {
Some("scoped/model")
);
assert_eq!(
- crate::inference::model_context::static_context_window_for_model_in(
- Some(&extensions),
- "scoped/model"
- ),
+ crate::inference::context_window::project_context_window_for_model(
+ "scoped/model",
+ Some(Some(&extensions))
+ )
+ .window,
Some(77_777)
);
+ assert_eq!(
+ crate::inference::context_window::project_context_window_for_model(
+ "hint:reasoning",
+ Some(Some(&extensions))
+ )
+ .window,
+ Some(88_888),
+ "a registered provider hook can override a workload tier alias"
+ );
let context = crate::core::runtime::CoreContext::for_test_with_extensions(
crate::core::runtime::DomainSet::full(),
None,
@@ -86,6 +100,24 @@ fn registered_model_defaults_and_aliases_are_context_scoped() {
None
);
crate::core::runtime::CoreContext::sync_scope(context, || {
+ assert_eq!(
+ crate::inference::context_window::project_context_window_for_model(
+ "scoped/model",
+ None
+ )
+ .window,
+ Some(77_777),
+ "the ambient projection uses the active provider registry"
+ );
+ assert_ne!(
+ crate::inference::context_window::project_context_window_for_model(
+ "scoped/model",
+ Some(None)
+ )
+ .window,
+ Some(77_777),
+ "an explicit process-only projection does not borrow ambient context"
+ );
assert_eq!(
crate::inference::registry::provider_default_model(
crate::inference::provider::factory::PROVIDER_OPENHUMAN