Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): parameterize chat model entries by provider context

Steven Enamakel22dc1e391030
diff --git a/crates/openhuman-core/src/agent/session_host/test_support.rs b/crates/openhuman-core/src/agent/session_host/test_support.rs
index 98d05aaf1e..c64ced071f 100644
--- a/crates/openhuman-core/src/agent/session_host/test_support.rs
+++ b/crates/openhuman-core/src/agent/session_host/test_support.rs
@@ -25,9 +25,9 @@ pub(crate) fn with_fixture_model_name(builder: SessionHostBuilder) -> SessionHos
/// Installs an explicit generic model fixture for tests that construct a
/// session through the managed-provider factory. HookSlot test registrations
/// are thread-local, so each test opts in on the thread that builds its host.
-fn model_fixture_entry() -> ChatModelEntry {
+fn model_fixture_entry() -> ChatModelEntry<ProviderContext> {
ChatModelEntry {
- build: Arc::new(|request: &ChatModelRequest<'_>| {
+ build: Arc::new(|request: &ChatModelRequest<'_, ProviderContext>| {
let model = request
.model_override
.or(request.default_model)
diff --git a/crates/openhuman-core/src/inference/provider/factory_explicit_ctx_tests.rs b/crates/openhuman-core/src/inference/provider/factory_explicit_ctx_tests.rs
index 601c3c0017..d9bd844851 100644
--- a/crates/openhuman-core/src/inference/provider/factory_explicit_ctx_tests.rs
+++ b/crates/openhuman-core/src/inference/provider/factory_explicit_ctx_tests.rs
@@ -10,7 +10,7 @@ async fn managed_factory_and_probe_use_the_explicit_provider_entry() {
use tinyinference_llm::message::Message;
use tinyinference_llm::model::ModelRequest;
- fn entry(name: &'static str, probed: Arc<AtomicBool>) -> ChatModelEntry {
+ fn entry(name: &'static str, probed: Arc<AtomicBool>) -> ChatModelEntry<ProviderContext> {
ChatModelEntry {
build: Arc::new(move |_| {
Ok((
@@ -140,7 +140,7 @@ async fn hosted_cloud_slug_uses_explicit_auth_policy_and_model_entry() {
use tinyinference_llm::message::Message;
use tinyinference_llm::model::ModelRequest;
- fn entry(name: &'static str) -> ChatModelEntry {
+ fn entry(name: &'static str) -> ChatModelEntry<ProviderContext> {
ChatModelEntry {
build: Arc::new(move |_| {
Ok((
diff --git a/crates/openhuman-core/src/inference/provider_model_tests.rs b/crates/openhuman-core/src/inference/provider_model_tests.rs
index 2f3e7d5226..7ba0212e15 100644
--- a/crates/openhuman-core/src/inference/provider_model_tests.rs
+++ b/crates/openhuman-core/src/inference/provider_model_tests.rs
@@ -229,7 +229,7 @@ fn native_provider_metadata_resolves_by_context_and_preserves_request_fields() {
use serde_json::json;
use std::sync::Arc;
- fn entry(label: &'static str, window: u64) -> ChatModelEntry {
+ fn entry(label: &'static str, window: u64) -> ChatModelEntry<ProviderContext> {
ChatModelEntry {
build: Arc::new(|_| panic!("metadata lookup must not build a model")),
probe: None,
diff --git a/crates/openhuman-core/src/inference/registry_tests.rs b/crates/openhuman-core/src/inference/registry_tests.rs
index 8aa23d5d7c..be71b1e41a 100644
--- a/crates/openhuman-core/src/inference/registry_tests.rs
+++ b/crates/openhuman-core/src/inference/registry_tests.rs
@@ -1,4 +1,5 @@
use super::*;
+use tinyinference_llm::model::ChatModel;
use crate::inference::context_window::model_limits_request;
use crate::inference::provider::factory::{
create_chat_model_from_string, make_openhuman_backend_model_for_thread, PROVIDER_OPENHUMAN,
@@ -23,7 +24,7 @@ impl ChatModel<()> for FakeModel {
}
}
-fn fake_entry(seen: SeenModelRequests) -> ChatModelEntry {
+fn fake_entry(seen: SeenModelRequests) -> ChatModelEntry<ProviderContext> {
ChatModelEntry {
build: Arc::new(move |request| {
seen.lock().unwrap().push((
diff --git a/crates/openhuman-core/src/web_chat/suggestions_tests.rs b/crates/openhuman-core/src/web_chat/suggestions_tests.rs
index 1584c0ada1..db6a6b5e11 100644
--- a/crates/openhuman-core/src/web_chat/suggestions_tests.rs
+++ b/crates/openhuman-core/src/web_chat/suggestions_tests.rs
@@ -91,7 +91,7 @@ fn install_suggestions_model_fixture() {
crate::inference::registry::register_chat_model(
PROVIDER_OPENHUMAN,
ChatModelEntry {
- build: Arc::new(|_request: &ChatModelRequest<'_>| {
+ build: Arc::new(|_request: &ChatModelRequest<'_, ProviderContext>| {
Ok((
Arc::new(ScriptedTextModel {
text: "[]".to_string(),
diff --git a/crates/openhuman-embed/examples/support/mod.rs b/crates/openhuman-embed/examples/support/mod.rs
index 50c7491674..38d917a6c8 100644
--- a/crates/openhuman-embed/examples/support/mod.rs
+++ b/crates/openhuman-embed/examples/support/mod.rs
@@ -234,7 +234,7 @@ impl PointedInference {
use tinyinference_llm::providers::openai::OpenAiModel;
let base_url = base_url.trim_end_matches('/').to_string();
- let build = move |request: &ChatModelRequest<'_>| {
+ let build = move |request: &ChatModelRequest<'_, ProviderContext>| {
let model = request
.model_override
.or(request.default_model)
diff --git a/crates/openhuman-embed/tests/common/mod.rs b/crates/openhuman-embed/tests/common/mod.rs
index 3382002603..67a56cec51 100644
--- a/crates/openhuman-embed/tests/common/mod.rs
+++ b/crates/openhuman-embed/tests/common/mod.rs
@@ -266,7 +266,7 @@ impl PointedInference {
use tinyinference_llm::providers::openai::OpenAiModel;
let base_url = base_url.trim_end_matches('/').to_string();
- let build = move |request: &ChatModelRequest<'_>| {
+ let build = move |request: &ChatModelRequest<'_, ProviderContext>| {
let model = request
.model_override
.or(request.default_model)
diff --git a/crates/openhuman-tinyhumans/src/inference/mod.rs b/crates/openhuman-tinyhumans/src/inference/mod.rs
index c997d3e814..f6216c6784 100644
--- a/crates/openhuman-tinyhumans/src/inference/mod.rs
+++ b/crates/openhuman-tinyhumans/src/inference/mod.rs
@@ -49,7 +49,7 @@ fn managed_options(config: &Config) -> ManagedOptions {
/// Builds the managed chat model for one registry request. The core has already
/// resolved the concrete model id (`model_override`) and disclosed the egress.
pub(crate) fn build_chat_model(
- request: &ChatModelRequest<'_>,
+ request: &ChatModelRequest<'_, ProviderContext>,
) -> anyhow::Result<(Arc<dyn ChatModel<()>>, String)> {
let pinned = request
.model_override