Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): update context window projection logic

Steven Enamakel3990dd65bd30
diff --git a/crates/openhuman-core/src/inference/context_window.rs b/crates/openhuman-core/src/inference/context_window.rs
index c1dede6cda..c99e4adacf 100644
--- a/crates/openhuman-core/src/inference/context_window.rs
+++ b/crates/openhuman-core/src/inference/context_window.rs
@@ -254,14 +254,14 @@ pub(crate) fn project_context_window(
resolver().project_for_route(provider, model, local_kind, route)
}
-/// Projects a static model-only window. The empty route key prevents a catalog
-/// or usage display from borrowing a learned limit from an unrelated provider.
+/// Projects a static model-only window without borrowing remembered state from
+/// the valid empty-provider route used by managed inference.
pub(crate) fn project_context_window_for_model(
model: &str,
extensions: Option<Option<&crate::core::slots::Extensions>>,
) -> ResolvedWindow {
let route = static_route_facts(model, extensions);
- resolver().project_for_route("", model, None, route)
+ ContextWindowResolver::new().project_for_route("", model, None, route)
}
/// Creates a client that carries OpenHuman's configured CA and proxy policy.
diff --git a/crates/openhuman-core/src/inference/context_window_tests.rs b/crates/openhuman-core/src/inference/context_window_tests.rs
index 09c0403727..23c6cb39b3 100644
--- a/crates/openhuman-core/src/inference/context_window_tests.rs
+++ b/crates/openhuman-core/src/inference/context_window_tests.rs
@@ -3,6 +3,8 @@ use std::sync::{Arc, Mutex as StdMutex};
use serde_json::{json, Value};
use tempfile::TempDir;
+use tinyinference_llm::model::context_window::{ContextWindowResolver, ContextWindowRouteInput};
+use tinyinference_llm::model::discover::DiscoveryRequest;
use tinyinference_llm::model::discover::ModelLimitsCache;
use crate::config::schema::cloud_providers::{AuthStyle, CloudProviderCreds};
@@ -150,6 +152,50 @@ async fn deepseek_v41_flash_uses_provider_window_not_static_128k() {
);
}
+#[tokio::test]
+async fn model_only_projection_does_not_reuse_empty_provider_discovery() {
+ let model = V41_FLASH;
+ let endpoint = "https://managed.example.test/v1";
+ let fetcher = FakeFetcher::default().with(
+ "https://managed.example.test/v1/models",
+ json!({ "data": [{ "id": model, "context_length": 777_777 }] }),
+ );
+ let cache = ModelLimitsCache::default();
+ let resolver = ContextWindowResolver::new();
+ let discovery = DiscoveryRequest::new(endpoint, model);
+
+ let discovered = resolver
+ .resolve_for_route_with_fetcher(
+ "",
+ model,
+ None,
+ ContextWindowRouteInput {
+ discovery: Some(discovery.clone()),
+ ..Default::default()
+ },
+ &cache,
+ &fetcher,
+ )
+ .await;
+ assert_eq!(discovered.window, Some(777_777));
+
+ let route_projection = resolver.project_for_route(
+ "",
+ model,
+ None,
+ ContextWindowRouteInput {
+ discovery: Some(discovery),
+ ..Default::default()
+ },
+ );
+ assert_eq!(route_projection.window, Some(777_777));
+
+ let model_projection =
+ crate::inference::context_window::project_context_window_for_model(model, None);
+ assert_eq!(model_projection.window, Some(128_000));
+ assert_ne!(model_projection.window, route_projection.window);
+}
+
#[tokio::test]
async fn second_resolution_is_served_from_cache() {
let tmp = TempDir::new().unwrap();