Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): adjust turn model profile handling

Steven Enamakel77f26e40f57f
diff --git a/crates/openhuman-core/src/inference/turn_models.rs b/crates/openhuman-core/src/inference/turn_models.rs
index be33ece50f..40d2c53a8a 100644
--- a/crates/openhuman-core/src/inference/turn_models.rs
+++ b/crates/openhuman-core/src/inference/turn_models.rs
@@ -413,24 +413,39 @@ impl TurnModelSource {
thread_id: Option<&str>,
) -> anyhow::Result<TurnModels> {
if let Some(direct) = &self.direct_model {
- let mut profile = direct.profile().cloned().unwrap_or_default();
- if let Some(window) = context_window.filter(|window| *window > 0) {
- profile.max_input_tokens = Some(window);
- }
+ let profile = direct.profile().cloned();
let provider_id = profile
- .provider
+ .as_ref()
+ .and_then(|profile| profile.provider.clone())
.clone()
.unwrap_or_else(|| "injected".to_string());
- let native_tools = profile.tool_calling;
- let supports_vision = profile.modalities.image_in;
+ let native_tools = profile.as_ref().is_some_and(|profile| profile.tool_calling);
+ let supports_vision = profile
+ .as_ref()
+ .is_some_and(|profile| profile.modalities.image_in);
let is_local =
crate::agent::tinyagents::turn_policy::provider_is_self_hosted(&provider_id, None);
- let context_window = context_window.or(profile.max_input_tokens);
- let primary: Arc<dyn tinyinference_llm::model::ChatModel<()>> = Arc::new(
- ProfileOverrideModel::new(direct.clone(), profile)
- .with_request_model(model)
- .with_request_temperature(temperature),
- );
+ let context_window = context_window.or_else(|| {
+ profile
+ .as_ref()
+ .and_then(|profile| profile.max_input_tokens)
+ });
+ let primary: Arc<dyn tinyinference_llm::model::ChatModel<()>> =
+ if let Some(window) = context_window.filter(|window| *window > 0) {
+ let mut profile = profile.unwrap_or_default();
+ profile.max_input_tokens = Some(window);
+ Arc::new(
+ ProfileOverrideModel::new(direct.clone(), profile)
+ .with_request_model(model)
+ .with_request_temperature(temperature),
+ )
+ } else {
+ Arc::new(
+ ProfileOverrideModel::new_preserving_profile(direct.clone())
+ .with_request_model(model)
+ .with_request_temperature(temperature),
+ )
+ };
let (primary, summarizer) = match &self.attachment_config {
Some(config) => (
crate::attachments::provider::wrap_injected(
@@ -512,9 +527,8 @@ impl TurnModelSource {
thread_id: Option<&str>,
) -> anyhow::Result<Arc<dyn tinyinference_llm::model::ChatModel<()>>> {
if let Some(direct) = &self.direct_model {
- let profile = direct.profile().cloned().unwrap_or_default();
let summarizer: Arc<dyn tinyinference_llm::model::ChatModel<()>> = Arc::new(
- ProfileOverrideModel::new(direct.clone(), profile)
+ ProfileOverrideModel::new_preserving_profile(direct.clone())
.with_request_model(model)
.with_request_temperature(temperature),
);
diff --git a/crates/openhuman-core/src/inference/turn_models_tests.rs b/crates/openhuman-core/src/inference/turn_models_tests.rs
index 0d1c3c499e..6b715f2418 100644
--- a/crates/openhuman-core/src/inference/turn_models_tests.rs
+++ b/crates/openhuman-core/src/inference/turn_models_tests.rs
@@ -6,6 +6,53 @@ use tinyagents_harness::host::{ModelResolveRequest, ModelResolver};
use tinyinference_llm::message::ContentBlock;
use tinyinference_llm::model::{ChatModel, ModelRequest, ModelResponse};
+#[tokio::test]
+async fn native_turn_factory_applies_default_temperature_and_preserves_explicit_override() {
+ struct TemperatureProbe(Arc<std::sync::Mutex<Vec<Option<f64>>>>);
+
+ #[async_trait]
+ impl ChatModel<()> for TemperatureProbe {
+ async fn invoke(
+ &self,
+ _state: &(),
+ request: ModelRequest,
+ ) -> tinyinference_llm::Result<ModelResponse> {
+ self.0.lock().unwrap().push(request.temperature);
+ Ok(ModelResponse::assistant("ok"))
+ }
+ }
+
+ let _guard = crate::inference::inference_test_guard_async().await;
+ let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
+ let _override = crate::inference::provider::factory::test_provider_override::install_model(
+ Arc::new(TemperatureProbe(Arc::clone(&seen))),
+ );
+ let config = Arc::new(crate::config::Config::default());
+ let ctx = crate::core::runtime::DispatchCtx::unbound();
+ let models = TurnModelSource::new_crate_native(&ctx, "chat", config)
+ .build("hint:chat", 0.3, None, None)
+ .expect("native model set");
+
+ models
+ .primary
+ .invoke(
+ &(),
+ ModelRequest::new(vec![tinyinference_llm::message::Message::user("default")]),
+ )
+ .await
+ .expect("default request");
+ models
+ .primary
+ .invoke(
+ &(),
+ ModelRequest::new(vec![tinyinference_llm::message::Message::user("explicit")])
+ .with_temperature(0.9),
+ )
+ .await
+ .expect("explicit request");
+ assert_eq!(*seen.lock().unwrap(), vec![Some(0.3), Some(0.9)]);
+}
+
use super::TurnModels;
use crate::agent::tinyagents::TurnModelSource;
use tinyagents_harness::host::PrebuiltModelResolver as TurnModelResolver;