Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(inference): apply ephemeral route provider options to cloud models

Steven Enamakelcbbc03da50ca
diff --git a/crates/openhuman-core/src/config/schema/ephemeral_route_tests.rs b/crates/openhuman-core/src/config/schema/ephemeral_route_tests.rs
index 569c674c80..4f6fb0ed44 100644
--- a/crates/openhuman-core/src/config/schema/ephemeral_route_tests.rs
+++ b/crates/openhuman-core/src/config/schema/ephemeral_route_tests.rs
@@ -128,6 +128,33 @@ fn apply_registers_the_entry_and_pins_the_turn_roles() {
assert_eq!(config.coding_provider.as_deref(), Some(expected.as_str()));
}
+#[test]
+fn apply_carries_provider_options_and_model_reasoning_effort() {
+ let mut config = config_with_model("provider/model");
+ let mut route = route();
+ route.provider_options = Some(serde_json::json!({"provider": {"sort": "latency"}}));
+ route.reasoning_effort = Some("high".into());
+
+ apply(&mut config, route);
+
+ assert_eq!(
+ config.ephemeral_route.as_ref().and_then(|route| route.provider_options.clone()),
+ Some(serde_json::json!({"provider": {"sort": "latency"}}))
+ );
+ assert_eq!(
+ config.runtime.reasoning_effort_by_model.get("provider/model"),
+ Some(&"high".to_string())
+ );
+}
+
+#[test]
+fn an_ephemeral_route_without_options_keeps_the_provider_default() {
+ let mut config = config_with_model("provider/model");
+ apply(&mut config, route());
+
+ assert_eq!(config.ephemeral_route.as_ref().unwrap().provider_options, None);
+}
+
#[test]
fn apply_replaces_an_explicit_role_pin() {
// The route is the caller's whole statement about where this turn runs, and
diff --git a/crates/openhuman-core/src/inference/provider/factory/cloud_slug.rs b/crates/openhuman-core/src/inference/provider/factory/cloud_slug.rs
index c34ebe5914..93cb953260 100644
--- a/crates/openhuman-core/src/inference/provider/factory/cloud_slug.rs
+++ b/crates/openhuman-core/src/inference/provider/factory/cloud_slug.rs
@@ -457,6 +457,14 @@ fn try_create_cloud_slug_chat_model_from_string_with_native_tools_and_access(
})
.map(|route| route.headers.clone())
.unwrap_or_default();
+ let default_provider_options = config
+ .ephemeral_route
+ .as_ref()
+ .filter(|route| {
+ slug == crate::config::schema::ephemeral_route::EPHEMERAL_ROUTE_SLUG
+ && route.endpoint == entry.endpoint
+ })
+ .and_then(|route| route.provider_options.clone());
let mut extra_query_params: Vec<(String, String)> = Vec::new();
let mut user_agent: Option<String> = None;
let mut codex_route: Option<&tinyinference_llm::providers::openai::codex::OpenAiCodexRouting> =
@@ -579,6 +587,7 @@ fn try_create_cloud_slug_chat_model_from_string_with_native_tools_and_access(
temperature_override,
temperature_unsupported_models: config.temperature_unsupported_models.as_slice(),
native_tool_calling,
+ default_provider_options,
extra_headers: extra_headers.as_slice(),
extra_query_params: extra_query_params.as_slice(),
user_agent: user_agent.as_deref(),
diff --git a/crates/openhuman-core/src/inference/provider/factory/cloud_spec.rs b/crates/openhuman-core/src/inference/provider/factory/cloud_spec.rs
index bf50243ec0..f263f564ee 100644
--- a/crates/openhuman-core/src/inference/provider/factory/cloud_spec.rs
+++ b/crates/openhuman-core/src/inference/provider/factory/cloud_spec.rs
@@ -52,6 +52,7 @@ pub(super) struct CloudBuild<'a> {
pub(super) temperature_override: Option<f64>,
pub(super) temperature_unsupported_models: &'a [String],
pub(super) native_tool_calling: bool,
+ pub(super) default_provider_options: Option<serde_json::Value>,
pub(super) extra_headers: &'a [(String, String)],
pub(super) extra_query_params: &'a [(String, String)],
pub(super) user_agent: Option<&'a str>,
@@ -66,6 +67,7 @@ impl<'a> CloudBuild<'a> {
temperature_override: self.temperature_override,
temperature_unsupported_models: self.temperature_unsupported_models,
native_tool_calling: Some(self.native_tool_calling),
+ default_provider_options: self.default_provider_options.clone(),
extra_headers: self.extra_headers,
extra_query_params: self.extra_query_params,
user_agent: self.user_agent,
diff --git a/crates/openhuman-core/src/inference/provider/factory/cloud_spec_tests.rs b/crates/openhuman-core/src/inference/provider/factory/cloud_spec_tests.rs
index 9cf06d2e33..9dd62477dc 100644
--- a/crates/openhuman-core/src/inference/provider/factory/cloud_spec_tests.rs
+++ b/crates/openhuman-core/src/inference/provider/factory/cloud_spec_tests.rs
@@ -19,6 +19,7 @@ fn build<'a>(entry: &CloudProviderCreds, model: &str, key: &'a str) -> CloudBuil
temperature_override: None,
temperature_unsupported_models: &[],
native_tool_calling: true,
+ default_provider_options: None,
extra_headers: &[],
extra_query_params: &[],
user_agent: None,
diff --git a/crates/openhuman-core/src/inference/provider/factory/discovery.rs b/crates/openhuman-core/src/inference/provider/factory/discovery.rs
index 56a66821cc..ae8d18023f 100644
--- a/crates/openhuman-core/src/inference/provider/factory/discovery.rs
+++ b/crates/openhuman-core/src/inference/provider/factory/discovery.rs
@@ -74,6 +74,14 @@ pub(crate) fn model_limits_request(
temperature_override: None,
temperature_unsupported_models: &temperature_unsupported,
native_tool_calling: true,
+ default_provider_options: config
+ .ephemeral_route
+ .as_ref()
+ .filter(|route| {
+ slug == crate::config::schema::ephemeral_route::EPHEMERAL_ROUTE_SLUG
+ && route.endpoint == resolution.entry.endpoint
+ })
+ .and_then(|route| route.provider_options.clone()),
extra_headers: &extra_headers,
extra_query_params: &[],
user_agent: None,
diff --git a/crates/openhuman-embed/src/agent/build.rs b/crates/openhuman-embed/src/agent/build.rs
index d3fb69ebf9..5f73a02ad7 100644
--- a/crates/openhuman-embed/src/agent/build.rs
+++ b/crates/openhuman-embed/src/agent/build.rs
@@ -281,7 +281,12 @@ pub(crate) fn instantiate(runtime: &Runtime, spec: AgentSpec) -> Result<AgentInn
Some(route.base_url.clone()),
Some(route.api_key.clone()),
)
- .map(|scoped| scoped.with_headers(route.headers.clone()))
+ .map(|scoped| {
+ scoped
+ .with_headers(route.headers.clone())
+ .with_provider_options(route.provider_options.clone())
+ .with_reasoning_effort(route.reasoning_effort.map(|effort| effort.as_str().to_string()))
+ })
});
let mut overrides = openhuman_core::agent::host_overrides::HostOverrides::default();
overrides.parent = runtime.core_runtime().context().host_overrides();
diff --git a/crates/openhuman-embed/src/turn.rs b/crates/openhuman-embed/src/turn.rs
index 087e52118c..a315dba0ca 100644
--- a/crates/openhuman-embed/src/turn.rs
+++ b/crates/openhuman-embed/src/turn.rs
@@ -461,6 +461,8 @@ impl Turn {
self.request.inference_url = Some(route.base_url);
self.request.api_key = Some(route.api_key);
self.request.inference_headers = route.headers;
+ self.request.inference_provider_options = route.provider_options;
+ self.request.inference_reasoning_effort = route.reasoning_effort.map(|effort| effort.as_str().to_string());
self
}
@@ -632,6 +634,9 @@ async fn dispatch(
request.api_key.clone(),
)
.map(|route| route.with_headers(request.inference_headers.clone()));
+ let route = route
+ .map(|route| route.with_provider_options(request.inference_provider_options.clone()))
+ .map(|route| route.with_reasoning_effort(request.inference_reasoning_effort.clone()));
agent_chat_reply_for_controlled(
&mut config,
target,
@@ -671,6 +676,9 @@ async fn dispatch(
request.api_key,
)
.map(|route| route.with_headers(request.inference_headers));
+ let route = route
+ .map(|route| route.with_provider_options(request.inference_provider_options))
+ .map(|route| route.with_reasoning_effort(request.inference_reasoning_effort));
let host = options.tools.or_else(|| inner.composed_host_tools());
let target = AgentChatTarget::Definition {
definition: &inner.definition,
diff --git a/crates/openhuman-embed/src/turn_types.rs b/crates/openhuman-embed/src/turn_types.rs
index 28f1a22fbd..f1fafb9285 100644
--- a/crates/openhuman-embed/src/turn_types.rs
+++ b/crates/openhuman-embed/src/turn_types.rs
@@ -206,6 +206,12 @@ pub struct TurnRequest {
/// Additional headers owned by this turn route.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub inference_headers: Vec<(String, String)>,
+ /// Provider-specific defaults for the per-turn route.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub inference_provider_options: Option<serde_json::Value>,
+ /// Reasoning effort for the per-turn route's model.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub inference_reasoning_effort: Option<String>,
/// Bearer half of the per-call route. Paired with `inference_url`.
#[serde(skip_serializing_if = "Option::is_none")]
pub api_key: Option<String>,
@@ -226,6 +232,8 @@ impl TurnRequest {
cwd: None,
inference_url: None,
inference_headers: Vec::new(),
+ inference_provider_options: None,
+ inference_reasoning_effort: None,
api_key: None,
agent_id: None,
}