Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(inference): delegate managed model to TinyHumansModel

Steven Enamakelf753d43a846a
diff --git a/crates/openhuman-tinyhumans/src/inference/managed_model.rs b/crates/openhuman-tinyhumans/src/inference/managed_model.rs
index 7f86be7f25..f905f8a5c7 100644
--- a/crates/openhuman-tinyhumans/src/inference/managed_model.rs
+++ b/crates/openhuman-tinyhumans/src/inference/managed_model.rs
@@ -1,36 +1,18 @@
-//! The managed TinyHumans inference model as a host [`ChatModel`] (issue #4727,
-//! Motion B), owned by this crate and registered with the core's chat-model
-//! registry ([`super::register`]).
+//! OpenHuman host configuration for TinyHumans' managed chat model.
//!
-//! The managed backend can't be a plain crate `OpenAiModel` preset: it uses a
-//! **dynamic** session JWT (fetched per call), emits the `thread_id` extension so
-//! the backend groups InferenceLog entries + aligns KV-cache keys, and relies on
-//! the `openhuman.usage/billing` response envelope for charged-USD / cached-token
-//! accounting. This `ChatModel` bridges all three onto the crate wire client:
-//!
-//! * **Dynamic JWT**: `invoke`/`stream` resolve the current bearer and build a
-//! fresh crate `OpenAiModel` (Bearer) per call ([`managed_session`](super::managed_session)).
-//! * **`thread_id`**: injected into `ModelRequest.provider_options` so the crate
-//! flattens it into the request body as the top-level `thread_id` field.
-//! * **Billing envelope**: [`managed_usage`](super::managed_usage) re-projects the
-//! `openhuman.{billing,usage}` envelope into the shape the cost bridge reads.
+//! Transport, request adaptation, response projection and managed-auth
+//! provenance belong to TinyInference's `TinyHumansModel`. This module only
+//! supplies OpenHuman's live credential resolver and client policy.
use std::path::PathBuf;
use std::time::Duration;
-use tinyinference_llm::message::Message;
-use tinyinference_llm::model::{ChatModel, Modalities, ModelProfile, ModelRequest, ProviderError};
-use tinyinference_llm::providers::openai::OpenAiModel;
-use tinyinference_llm::providers::ProviderRequestOptions;
-use tinyinference_llm::Error as TiError;
+use tinyinference_llm::providers::tinyhumans::TinyHumansModel;
-use super::managed_session::resolve_session_bearer;
+use super::managed_session::{publish_managed_auth_failure, resolve_session_bearer};
use crate::backend::url::effective_api_url;
use crate::client::endpoint::is_managed_endpoint_for_api_key;
-/// Provider label the wire client stamps on its errors.
-pub const PROVIDER_LABEL: &str = "OpenHuman";
-
/// Where the managed model reads its credentials from.
#[derive(Debug, Clone, Default)]
pub struct ManagedOptions {
@@ -43,306 +25,80 @@ pub struct ManagedOptions {
pub auth_profile_override: Option<String>,
}
-/// connection settings (for JWT + base-URL resolution) and the default model id
-/// sent when a request doesn't override it.
-pub struct OpenHumanBackendModel {
- options: ManagedOptions,
- api_url: Option<String>,
- pub(super) default_model: String,
- native_tool_calling: bool,
- pub(super) profile: ModelProfile,
- /// Normalized OpenHuman conversation thread. This is deliberately owned by
- /// the managed backend model: BYOK and third-party models must never see
- /// this backend-only extension.
- pub(super) thread_id: Option<String>,
-}
-
-impl OpenHumanBackendModel {
- pub fn new(
- api_url: Option<&str>,
- options: &ManagedOptions,
- default_model: impl Into<String>,
- ) -> Self {
- Self {
- options: options.clone(),
- api_url: api_url
- .map(str::trim)
- .filter(|url| !url.is_empty())
- .map(ToOwned::to_owned),
- default_model: resolve_model(&default_model.into()),
- native_tool_calling: true,
- profile: ModelProfile {
- provider: Some("managed".to_string()),
- modalities: Modalities {
- image_in: true,
- ..Modalities::default()
- },
- tool_calling: true,
- parallel_tool_calls: true,
- streaming: true,
- streaming_tool_chunks: true,
- ..ModelProfile::default()
- },
- thread_id: None,
- }
- }
-
- /// Attach the explicit run thread used by OpenHuman's managed inference
- /// endpoint. Blank values mean no backend thread rather than an empty wire
- /// field.
- pub fn with_thread_id(mut self, thread_id: Option<impl AsRef<str>>) -> Self {
- self.thread_id = thread_id.and_then(|thread_id| {
- let thread_id = thread_id.as_ref().trim();
- (!thread_id.is_empty()).then(|| thread_id.to_owned())
- });
- self
- }
-
- /// Force prompt-guided tool calling for toolsets that exceed the managed
- /// backend's native grammar ceiling.
- pub fn with_native_tool_calling(mut self, enabled: bool) -> Self {
- self.native_tool_calling = enabled;
- self.profile.tool_calling = enabled;
- self.profile.parallel_tool_calls = enabled;
- self.profile.streaming_tool_chunks = enabled;
- self
- }
-
- pub(super) fn state_dir(&self) -> PathBuf {
- self.options.openhuman_dir.clone().unwrap_or_else(|| {
+impl ManagedOptions {
+ fn state_dir(&self) -> PathBuf {
+ self.openhuman_dir.clone().unwrap_or_else(|| {
directories::UserDirs::new()
.map(|dirs| dirs.home_dir().join(".openhuman"))
.unwrap_or_else(|| PathBuf::from(".openhuman"))
})
}
- pub(super) fn resolve_bearer(&self) -> anyhow::Result<String> {
- // A stored API key (library runtime) is the bearer outright: the
- // OpenAI-compatible managed endpoint accepts it as `Bearer <key>`,
- // and there is no session — so no `exp` and no signed-out state — to
- // consult.
+ fn resolve_bearer(&self, endpoint: &str) -> anyhow::Result<String> {
if let Some(key) = openhuman_embed::__host::security::credentials::api_key::get_api_key_in(
&self.state_dir(),
- self.options.secrets_encrypt,
+ self.secrets_encrypt,
)? {
- // Refuse to send the key over a plaintext channel it could leak
- // from. Scoped to this managed-key path only — `base_url()`
- // comes from the backend transport, which a library host can point
- // at anything (a BYOK/local endpoint legitimately runs over
- // plain HTTP on loopback), so this cannot tighten
- // `normalize_api_base_url` itself without breaking those.
- let endpoint = self.base_url()?;
- if !is_managed_endpoint_for_api_key(&endpoint) {
+ if !is_managed_endpoint_for_api_key(endpoint) {
anyhow::bail!(
"refusing to send the TinyHumans API key to an unmanaged or insecure \
endpoint: {endpoint} — use the managed backend or loopback for local testing"
);
}
- log::debug!(
- "[providers][openhuman-backend] authenticating managed inference with api-key"
- );
+ log::debug!("[providers][openhuman-backend] authenticating managed inference with api-key");
return Ok(key);
}
resolve_session_bearer(
&self.state_dir(),
- self.options.secrets_encrypt,
- self.options.auth_profile_override.as_deref(),
+ self.secrets_encrypt,
+ self.auth_profile_override.as_deref(),
"openhuman_backend_model.resolve_bearer",
)
}
-
- /// The managed OpenAI-compatible endpoint, from the configured `api_url`
- /// override or this crate's backend URL defaults.
- pub(super) fn base_url(&self) -> anyhow::Result<String> {
- let base = effective_api_url(&self.api_url);
- Ok(format!("{}/openai/v1", base.trim_end_matches('/')))
- }
-
- /// Resolve the current JWT + base URL and build a fresh crate `OpenAiModel`
- /// (Bearer). Rebuilt per call because the session JWT rotates, but every
- /// build shares one pooled HTTP client ([`managed_inference_http_client`]),
- /// so consecutive calls reuse a warm TLS connection to the backend.
- pub(super) fn build_wire_model(&self) -> tinyinference_llm::Result<OpenAiModel> {
- let token = self
- .resolve_bearer()
- .map_err(|e| TiError::Model(e.to_string()))?;
- let base_url = self.base_url().map_err(|e| TiError::Model(e.to_string()))?;
- // The hosted API is chat-completions only (no `/v1/responses`); auth is a
- // plain bearer JWT. The tier/model rides `request.model`, which the backend
- // resolves — the baked default only applies when a request omits it.
- Ok(
- OpenAiModel::compatible_provider(PROVIDER_LABEL, token, base_url, &self.default_model)
- .with_native_tool_calling(self.native_tool_calling)
- .with_request_options(ProviderRequestOptions {
- http: Some(managed_inference_http_client()),
- ..ProviderRequestOptions::default()
- }),
- )
- }
-
- /// Probe whether the managed backend account actually has a working
- /// inference provider configured, cheaply and without inflating usage
- /// (issue B45 — flows provider-connectivity author gate).
- ///
- /// [`build_wire_model`](Self::build_wire_model) only resolves the session
- /// JWT and builds the request client — it says nothing about whether the
- /// account has a provider API key configured server-side. That only
- /// surfaces on a real completion attempt, as an HTTP 400
- /// `{"success":false,"error":"API key not configured for provider","errorCode":"BAD_REQUEST"}`.
- /// Previously the first time a flows author found this out was mid-run,
- /// deep inside a tinyflows `agent` node. This probe moves that discovery
- /// to author time by issuing one minimal completion (`"ping"`,
- /// `max_tokens: 1`) and classifying the result.
- ///
- /// Fails OPEN on everything except a definitive client-configuration
- /// error: a 5-second timeout, a transport failure, a 5xx, or any other
- /// non-matching provider error all return `Ok(())` so a flaky backend or
- /// slow network never blocks authoring. Only a backend-confirmed "no
- /// provider configured for this account" response returns `Err` —
- /// carrying the backend's own error string so the author sees exactly
- /// what run time would have shown them.
- pub async fn probe_readiness(&self) -> Result<(), String> {
- log::debug!(
- "[flows][inference-probe] entering probe_readiness model={}",
- self.default_model
- );
-
- let model = match self.build_wire_model() {
- Ok(model) => model,
- Err(e) => {
- // The flows readiness gate's Layer 1 (sign-in / session
- // checks) is responsible for catching a genuinely
- // absent/expired session before this ever runs — a
- // construction failure reaching here is a race, not a
- // provider-configuration problem, so fail open rather than
- // duplicate or contradict that gate's message.
- log::debug!(
- "[flows][inference-probe] wire model construction failed, failing open: {e}"
- );
- return Ok(());
- }
- };
-
- let request = ModelRequest::new(vec![Message::user("ping")]).with_max_tokens(1);
-
- let outcome =
- match tokio::time::timeout(Duration::from_secs(5), model.invoke(&(), request)).await {
- Ok(result) => result,
- Err(_) => {
- log::debug!(
- "[flows][inference-probe] model={} timed out after 5s, failing open",
- self.default_model
- );
- return Ok(());
- }
- };
-
- match outcome {
- Ok(_) => {
- log::debug!(
- "[flows][inference-probe] model={} probe completion succeeded — provider ready",
- self.default_model
- );
- Ok(())
- }
- Err(TiError::Provider(err)) => {
- if is_provider_not_configured_error(&err) {
- log::warn!(
- "[flows][inference-probe] model={} backend reports no provider configured: {}",
- self.default_model,
- err.message
- );
- Err(err.message.clone())
- } else if err.status.is_some_and(|status| status >= 500) {
- log::debug!(
- "[flows][inference-probe] model={} backend {:?}, failing open: {}",
- self.default_model,
- err.status,
- err.message
- );
- Ok(())
- } else {
- // Any other structured provider failure (401, 429, a
- // malformed request, …) is not the definitive "provider
- // not configured" signal this probe exists to catch —
- // fail open rather than risk a false-positive
- // author-time block.
- log::debug!(
- "[flows][inference-probe] model={} non-definitive provider error {:?}, \
- failing open: {}",
- self.default_model,
- err.status,
- err.message
- );
- Ok(())
- }
- }
- Err(e) => {
- log::debug!(
- "[flows][inference-probe] model={} transport/model error, failing open: {e}",
- self.default_model
- );
- Ok(())
- }
- }
- }
-}
-
-/// Whether `err` is the definitive "no inference provider configured for this
-/// account" signal the managed backend returns as an HTTP 400 with body
-/// `{"success":false,"error":"API key not configured for provider","errorCode":"BAD_REQUEST"}`.
-///
-/// Deliberately narrow: matches ONLY a 400 whose message contains the specific
-/// `"api key not configured for provider"` phrasing, or (as a `BAD_REQUEST`-
-/// coded tolerance for message wording drift) the narrower `"not configured
-/// for provider"` substring — never a bare `"not configured"`, which an
-/// unrelated 400 (a malformed request naming some other unconfigured field,
-/// a validation error, …) could also contain. Every other 4xx/5xx/transport
-/// failure fails open (see [`OpenHumanBackendModel::probe_readiness`]'s doc).
-fn is_provider_not_configured_error(err: &ProviderError) -> bool {
- if err.status != Some(400) {
- return false;
- }
- let message = err.message.to_ascii_lowercase();
- let code_is_bad_request = err
- .code
- .as_deref()
- .is_some_and(|c| c.eq_ignore_ascii_case("BAD_REQUEST"));
- message.contains("api key not configured for provider")
- || (code_is_bad_request && message.contains("not configured for provider"))
}
-fn resolve_model(model: &str) -> String {
- let trimmed = model.trim();
- if trimmed.is_empty() {
- log::debug!(
- "[providers][openhuman-backend] empty model passed to OpenHuman backend; \
- substituting default `{}` (TAURI-RUST-RS)",
- super::tiers::MODEL_MANAGED_DEFAULT
- );
- super::tiers::MODEL_MANAGED_DEFAULT.to_string()
- } else {
- trimmed.to_string()
+/// Constructs the owning TinyInference model with OpenHuman's rotating
+/// credential source and host-specific connection policy.
+pub(super) fn build_chat_model(
+ api_url: Option<&str>,
+ options: &ManagedOptions,
+ model: &str,
+ native_tool_calling: bool,
+ thread_id: Option<&str>,
+ observe_auth_failures: bool,
+) -> anyhow::Result<TinyHumansModel> {
+ let endpoint = effective_api_url(&api_url.map(str::to_owned));
+ let resolver_options = options.clone();
+ let resolver_endpoint = endpoint.clone();
+ let mut managed = TinyHumansModel::new(
+ endpoint,
+ model,
+ model,
+ move || {
+ resolver_options
+ .resolve_bearer(&resolver_endpoint)
+ .map_err(|error| tinyinference_llm::Error::Model(error.to_string()))
+ },
+ )?
+ .with_cache_namespace("openhuman")
+ .with_native_tool_calling(native_tool_calling)
+ .with_thread_id(thread_id)
+ .with_http_client(managed_inference_http_client());
+
+ if observe_auth_failures {
+ managed = managed.with_managed_auth_failure_handler(|error, context| {
+ publish_managed_auth_failure(error, context);
+ });
}
+ Ok(managed)
}
-/// Connect timeout for the shared managed-inference client; matches the
-/// adapter's own default.
+/// Connect timeout for managed inference; matches the previous host adapter.
const MANAGED_INFERENCE_CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
-
-/// How long an idle pooled connection is kept. Below the AWS ALB's default 60s
-/// idle timeout in front of the backend, so the pool never hands out a
-/// connection the load balancer has already closed.
+/// Keep pooled connections below the backend ALB's idle timeout.
const MANAGED_INFERENCE_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(50);
-/// The one HTTP client every managed-inference call in the process shares.
-///
-/// `OpenAiModel` otherwise builds its own `reqwest::Client` per instance, and
-/// [`OpenHumanBackendModel::build_wire_model`] builds an instance per model
-/// call, so each call paid a fresh TCP + TLS handshake to the backend (about
-/// 150-300 ms measured to `api.tinyhumans.ai`) before its first token. A
-/// `reqwest::Client` is a cheap handle over a shared connection pool, so
-/// cloning it keeps connections warm across calls, turns and threads.
+/// Shared HTTP client retains TLS connections across managed turns.
fn managed_inference_http_client() -> reqwest::Client {
static CLIENT: std::sync::OnceLock<reqwest::Client> = std::sync::OnceLock::new();
CLIENT
@@ -367,6 +123,3 @@ fn managed_inference_http_client() -> reqwest::Client {
.clone()
}
-#[cfg(test)]
-#[path = "managed_model_tests.rs"]
-mod tests;
diff --git a/crates/openhuman-tinyhumans/src/inference/managed_session.rs b/crates/openhuman-tinyhumans/src/inference/managed_session.rs
index 4a164f2152..8bf99e051f 100644
--- a/crates/openhuman-tinyhumans/src/inference/managed_session.rs
+++ b/crates/openhuman-tinyhumans/src/inference/managed_session.rs
@@ -3,6 +3,7 @@
//! rejected session.
use tinyinference_llm::model::ProviderError;
+use tinyinference_llm::providers::tinyhumans::{TinyHumansAuthFailure, TinyHumansAuthFailureKind, TinyHumansOperation};
use tinyinference_llm::Error as TiError;
/// Resolve the persisted app-session token for a managed inference caller.
@@ -91,30 +92,32 @@ fn maybe_publish_local_session_expiry(operation: &str) {
);
}
-/// Publish a `SessionExpired` event when the backend rejects a crate-native
-/// model call with `401`/`403` Unauthorized — mirroring the check in
-/// the core `CrateBackedProvider::invoke` which the
-/// crate-native path bypasses.
-pub(super) fn maybe_publish_session_expired(err: &TiError, operation: &str) {
- if let TiError::Provider(pe) = err {
- maybe_publish_provider_session_expired(pe, operation);
- }
-}
-
-pub(super) fn maybe_publish_provider_session_expired(pe: &ProviderError, operation: &str) {
- if pe.provider.as_str() == "OpenHuman" && matches!(pe.status, Some(401 | 403)) {
- let reason = tinyinference_core::sanitize::sanitize_api_error(&pe.message);
- openhuman_embed::__host::core::bus::BUS.publish(
- openhuman_embed::__host::core::events::DomainEvent::SessionExpired {
- source: format!(
- "openhuman_backend_model.{}({})",
- operation,
- pe.status.unwrap_or(0)
- ),
- reason,
- },
- );
- }
+/// Publish expiry only from TinyInference's explicit managed-endpoint
+/// provenance. The provider label is a wire diagnostic and is not an auth
+/// ownership signal.
+pub(super) fn publish_managed_auth_failure(
+ error: &ProviderError,
+ context: TinyHumansAuthFailure,
+) {
+ let status = match context.kind() {
+ TinyHumansAuthFailureKind::Unauthorized => 401,
+ TinyHumansAuthFailureKind::Forbidden => 403,
+ TinyHumansAuthFailureKind::Other => return,
+ };
+ let operation = match context.operation() {
+ TinyHumansOperation::Invoke => "invoke",
+ TinyHumansOperation::Stream => "stream",
+ // Readiness deliberately remains fail-open and does not sign the user
+ // out, matching its previous host behavior.
+ TinyHumansOperation::Readiness => return,
+ };
+ let reason = tinyinference_core::sanitize::sanitize_api_error(&error.message);
+ openhuman_embed::__host::core::bus::BUS.publish(
+ openhuman_embed::__host::core::events::DomainEvent::SessionExpired {
+ source: format!("openhuman_backend_model.{operation}({status})"),
+ reason,
+ },
+ );
}
#[cfg(test)]