The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(inference): delegate managed model to TinyHumansModel
Steven Enamakelf753d43a846a
diff --git a/crates/openhuman-tinyhumans/src/inference/managed_model.rs b/crates/openhuman-tinyhumans/src/inference/managed_model.rsindex 7f86be7f25..f905f8a5c7 100644--- a/crates/openhuman-tinyhumans/src/inference/managed_model.rs+++ b/crates/openhuman-tinyhumans/src/inference/managed_model.rs@@ -1,36 +1,18 @@-//! The managed TinyHumans inference model as a host [`ChatModel`] (issue #4727,-//! Motion B), owned by this crate and registered with the core's chat-model-//! registry ([`super::register`]).+//! OpenHuman host configuration for TinyHumans' managed chat model.//!-//! The managed backend can't be a plain crate `OpenAiModel` preset: it uses a-//! **dynamic** session JWT (fetched per call), emits the `thread_id` extension so-//! the backend groups InferenceLog entries + aligns KV-cache keys, and relies on-//! the `openhuman.usage/billing` response envelope for charged-USD / cached-token-//! accounting. This `ChatModel` bridges all three onto the crate wire client:-//!-//! * **Dynamic JWT**: `invoke`/`stream` resolve the current bearer and build a-//! fresh crate `OpenAiModel` (Bearer) per call ([`managed_session`](super::managed_session)).-//! * **`thread_id`**: injected into `ModelRequest.provider_options` so the crate-//! flattens it into the request body as the top-level `thread_id` field.-//! * **Billing envelope**: [`managed_usage`](super::managed_usage) re-projects the-//! `openhuman.{billing,usage}` envelope into the shape the cost bridge reads.+//! Transport, request adaptation, response projection and managed-auth+//! provenance belong to TinyInference's `TinyHumansModel`. This module only+//! supplies OpenHuman's live credential resolver and client policy.use std::path::PathBuf;use std::time::Duration;-use tinyinference_llm::message::Message;-use tinyinference_llm::model::{ChatModel, Modalities, ModelProfile, ModelRequest, ProviderError};-use tinyinference_llm::providers::openai::OpenAiModel;-use tinyinference_llm::providers::ProviderRequestOptions;-use tinyinference_llm::Error as TiError;+use tinyinference_llm::providers::tinyhumans::TinyHumansModel;-use super::managed_session::resolve_session_bearer;+use super::managed_session::{publish_managed_auth_failure, resolve_session_bearer};use crate::backend::url::effective_api_url;use crate::client::endpoint::is_managed_endpoint_for_api_key;-/// Provider label the wire client stamps on its errors.-pub const PROVIDER_LABEL: &str = "OpenHuman";-/// Where the managed model reads its credentials from.#[derive(Debug, Clone, Default)]pub struct ManagedOptions {@@ -43,306 +25,80 @@ pub struct ManagedOptions {pub auth_profile_override: Option<String>,}-/// connection settings (for JWT + base-URL resolution) and the default model id-/// sent when a request doesn't override it.-pub struct OpenHumanBackendModel {- options: ManagedOptions,- api_url: Option<String>,- pub(super) default_model: String,- native_tool_calling: bool,- pub(super) profile: ModelProfile,- /// Normalized OpenHuman conversation thread. This is deliberately owned by- /// the managed backend model: BYOK and third-party models must never see- /// this backend-only extension.- pub(super) thread_id: Option<String>,-}--impl OpenHumanBackendModel {- pub fn new(- api_url: Option<&str>,- options: &ManagedOptions,- default_model: impl Into<String>,- ) -> Self {- Self {- options: options.clone(),- api_url: api_url- .map(str::trim)- .filter(|url| !url.is_empty())- .map(ToOwned::to_owned),- default_model: resolve_model(&default_model.into()),- native_tool_calling: true,- profile: ModelProfile {- provider: Some("managed".to_string()),- modalities: Modalities {- image_in: true,- ..Modalities::default()- },- tool_calling: true,- parallel_tool_calls: true,- streaming: true,- streaming_tool_chunks: true,- ..ModelProfile::default()- },- thread_id: None,- }- }-- /// Attach the explicit run thread used by OpenHuman's managed inference- /// endpoint. Blank values mean no backend thread rather than an empty wire- /// field.- pub fn with_thread_id(mut self, thread_id: Option<impl AsRef<str>>) -> Self {- self.thread_id = thread_id.and_then(|thread_id| {- let thread_id = thread_id.as_ref().trim();- (!thread_id.is_empty()).then(|| thread_id.to_owned())- });- self- }-- /// Force prompt-guided tool calling for toolsets that exceed the managed- /// backend's native grammar ceiling.- pub fn with_native_tool_calling(mut self, enabled: bool) -> Self {- self.native_tool_calling = enabled;- self.profile.tool_calling = enabled;- self.profile.parallel_tool_calls = enabled;- self.profile.streaming_tool_chunks = enabled;- self- }-- pub(super) fn state_dir(&self) -> PathBuf {- self.options.openhuman_dir.clone().unwrap_or_else(|| {+impl ManagedOptions {+ fn state_dir(&self) -> PathBuf {+ self.openhuman_dir.clone().unwrap_or_else(|| {directories::UserDirs::new().map(|dirs| dirs.home_dir().join(".openhuman")).unwrap_or_else(|| PathBuf::from(".openhuman"))})}- pub(super) fn resolve_bearer(&self) -> anyhow::Result<String> {- // A stored API key (library runtime) is the bearer outright: the- // OpenAI-compatible managed endpoint accepts it as `Bearer <key>`,- // and there is no session — so no `exp` and no signed-out state — to- // consult.+ fn resolve_bearer(&self, endpoint: &str) -> anyhow::Result<String> {if let Some(key) = openhuman_embed::__host::security::credentials::api_key::get_api_key_in(&self.state_dir(),- self.options.secrets_encrypt,+ self.secrets_encrypt,)? {- // Refuse to send the key over a plaintext channel it could leak- // from. Scoped to this managed-key path only — `base_url()`- // comes from the backend transport, which a library host can point- // at anything (a BYOK/local endpoint legitimately runs over- // plain HTTP on loopback), so this cannot tighten- // `normalize_api_base_url` itself without breaking those.- let endpoint = self.base_url()?;- if !is_managed_endpoint_for_api_key(&endpoint) {+ if !is_managed_endpoint_for_api_key(endpoint) {anyhow::bail!("refusing to send the TinyHumans API key to an unmanaged or insecure \endpoint: {endpoint} — use the managed backend or loopback for local testing");}- log::debug!(- "[providers][openhuman-backend] authenticating managed inference with api-key"- );+ log::debug!("[providers][openhuman-backend] authenticating managed inference with api-key");return Ok(key);}resolve_session_bearer(&self.state_dir(),- self.options.secrets_encrypt,- self.options.auth_profile_override.as_deref(),+ self.secrets_encrypt,+ self.auth_profile_override.as_deref(),"openhuman_backend_model.resolve_bearer",)}-- /// The managed OpenAI-compatible endpoint, from the configured `api_url`- /// override or this crate's backend URL defaults.- pub(super) fn base_url(&self) -> anyhow::Result<String> {- let base = effective_api_url(&self.api_url);- Ok(format!("{}/openai/v1", base.trim_end_matches('/')))- }-- /// Resolve the current JWT + base URL and build a fresh crate `OpenAiModel`- /// (Bearer). Rebuilt per call because the session JWT rotates, but every- /// build shares one pooled HTTP client ([`managed_inference_http_client`]),- /// so consecutive calls reuse a warm TLS connection to the backend.- pub(super) fn build_wire_model(&self) -> tinyinference_llm::Result<OpenAiModel> {- let token = self- .resolve_bearer()- .map_err(|e| TiError::Model(e.to_string()))?;- let base_url = self.base_url().map_err(|e| TiError::Model(e.to_string()))?;- // The hosted API is chat-completions only (no `/v1/responses`); auth is a- // plain bearer JWT. The tier/model rides `request.model`, which the backend- // resolves — the baked default only applies when a request omits it.- Ok(- OpenAiModel::compatible_provider(PROVIDER_LABEL, token, base_url, &self.default_model)- .with_native_tool_calling(self.native_tool_calling)- .with_request_options(ProviderRequestOptions {- http: Some(managed_inference_http_client()),- ..ProviderRequestOptions::default()- }),- )- }-- /// Probe whether the managed backend account actually has a working- /// inference provider configured, cheaply and without inflating usage- /// (issue B45 — flows provider-connectivity author gate).- ///- /// [`build_wire_model`](Self::build_wire_model) only resolves the session- /// JWT and builds the request client — it says nothing about whether the- /// account has a provider API key configured server-side. That only- /// surfaces on a real completion attempt, as an HTTP 400- /// `{"success":false,"error":"API key not configured for provider","errorCode":"BAD_REQUEST"}`.- /// Previously the first time a flows author found this out was mid-run,- /// deep inside a tinyflows `agent` node. This probe moves that discovery- /// to author time by issuing one minimal completion (`"ping"`,- /// `max_tokens: 1`) and classifying the result.- ///- /// Fails OPEN on everything except a definitive client-configuration- /// error: a 5-second timeout, a transport failure, a 5xx, or any other- /// non-matching provider error all return `Ok(())` so a flaky backend or- /// slow network never blocks authoring. Only a backend-confirmed "no- /// provider configured for this account" response returns `Err` —- /// carrying the backend's own error string so the author sees exactly- /// what run time would have shown them.- pub async fn probe_readiness(&self) -> Result<(), String> {- log::debug!(- "[flows][inference-probe] entering probe_readiness model={}",- self.default_model- );-- let model = match self.build_wire_model() {- Ok(model) => model,- Err(e) => {- // The flows readiness gate's Layer 1 (sign-in / session- // checks) is responsible for catching a genuinely- // absent/expired session before this ever runs — a- // construction failure reaching here is a race, not a- // provider-configuration problem, so fail open rather than- // duplicate or contradict that gate's message.- log::debug!(- "[flows][inference-probe] wire model construction failed, failing open: {e}"- );- return Ok(());- }- };-- let request = ModelRequest::new(vec![Message::user("ping")]).with_max_tokens(1);-- let outcome =- match tokio::time::timeout(Duration::from_secs(5), model.invoke(&(), request)).await {- Ok(result) => result,- Err(_) => {- log::debug!(- "[flows][inference-probe] model={} timed out after 5s, failing open",- self.default_model- );- return Ok(());- }- };-- match outcome {- Ok(_) => {- log::debug!(- "[flows][inference-probe] model={} probe completion succeeded — provider ready",- self.default_model- );- Ok(())- }- Err(TiError::Provider(err)) => {- if is_provider_not_configured_error(&err) {- log::warn!(- "[flows][inference-probe] model={} backend reports no provider configured: {}",- self.default_model,- err.message- );- Err(err.message.clone())- } else if err.status.is_some_and(|status| status >= 500) {- log::debug!(- "[flows][inference-probe] model={} backend {:?}, failing open: {}",- self.default_model,- err.status,- err.message- );- Ok(())- } else {- // Any other structured provider failure (401, 429, a- // malformed request, …) is not the definitive "provider- // not configured" signal this probe exists to catch —- // fail open rather than risk a false-positive- // author-time block.- log::debug!(- "[flows][inference-probe] model={} non-definitive provider error {:?}, \- failing open: {}",- self.default_model,- err.status,- err.message- );- Ok(())- }- }- Err(e) => {- log::debug!(- "[flows][inference-probe] model={} transport/model error, failing open: {e}",- self.default_model- );- Ok(())- }- }- }-}--/// Whether `err` is the definitive "no inference provider configured for this-/// account" signal the managed backend returns as an HTTP 400 with body-/// `{"success":false,"error":"API key not configured for provider","errorCode":"BAD_REQUEST"}`.-///-/// Deliberately narrow: matches ONLY a 400 whose message contains the specific-/// `"api key not configured for provider"` phrasing, or (as a `BAD_REQUEST`--/// coded tolerance for message wording drift) the narrower `"not configured-/// for provider"` substring — never a bare `"not configured"`, which an-/// unrelated 400 (a malformed request naming some other unconfigured field,-/// a validation error, …) could also contain. Every other 4xx/5xx/transport-/// failure fails open (see [`OpenHumanBackendModel::probe_readiness`]'s doc).-fn is_provider_not_configured_error(err: &ProviderError) -> bool {- if err.status != Some(400) {- return false;- }- let message = err.message.to_ascii_lowercase();- let code_is_bad_request = err- .code- .as_deref()- .is_some_and(|c| c.eq_ignore_ascii_case("BAD_REQUEST"));- message.contains("api key not configured for provider")- || (code_is_bad_request && message.contains("not configured for provider"))}-fn resolve_model(model: &str) -> String {- let trimmed = model.trim();- if trimmed.is_empty() {- log::debug!(- "[providers][openhuman-backend] empty model passed to OpenHuman backend; \- substituting default `{}` (TAURI-RUST-RS)",- super::tiers::MODEL_MANAGED_DEFAULT- );- super::tiers::MODEL_MANAGED_DEFAULT.to_string()- } else {- trimmed.to_string()+/// Constructs the owning TinyInference model with OpenHuman's rotating+/// credential source and host-specific connection policy.+pub(super) fn build_chat_model(+ api_url: Option<&str>,+ options: &ManagedOptions,+ model: &str,+ native_tool_calling: bool,+ thread_id: Option<&str>,+ observe_auth_failures: bool,+) -> anyhow::Result<TinyHumansModel> {+ let endpoint = effective_api_url(&api_url.map(str::to_owned));+ let resolver_options = options.clone();+ let resolver_endpoint = endpoint.clone();+ let mut managed = TinyHumansModel::new(+ endpoint,+ model,+ model,+ move || {+ resolver_options+ .resolve_bearer(&resolver_endpoint)+ .map_err(|error| tinyinference_llm::Error::Model(error.to_string()))+ },+ )?+ .with_cache_namespace("openhuman")+ .with_native_tool_calling(native_tool_calling)+ .with_thread_id(thread_id)+ .with_http_client(managed_inference_http_client());++ if observe_auth_failures {+ managed = managed.with_managed_auth_failure_handler(|error, context| {+ publish_managed_auth_failure(error, context);+ });}+ Ok(managed)}-/// Connect timeout for the shared managed-inference client; matches the-/// adapter's own default.+/// Connect timeout for managed inference; matches the previous host adapter.const MANAGED_INFERENCE_CONNECT_TIMEOUT: Duration = Duration::from_secs(30);--/// How long an idle pooled connection is kept. Below the AWS ALB's default 60s-/// idle timeout in front of the backend, so the pool never hands out a-/// connection the load balancer has already closed.+/// Keep pooled connections below the backend ALB's idle timeout.const MANAGED_INFERENCE_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(50);-/// The one HTTP client every managed-inference call in the process shares.-///-/// `OpenAiModel` otherwise builds its own `reqwest::Client` per instance, and-/// [`OpenHumanBackendModel::build_wire_model`] builds an instance per model-/// call, so each call paid a fresh TCP + TLS handshake to the backend (about-/// 150-300 ms measured to `api.tinyhumans.ai`) before its first token. A-/// `reqwest::Client` is a cheap handle over a shared connection pool, so-/// cloning it keeps connections warm across calls, turns and threads.+/// Shared HTTP client retains TLS connections across managed turns.fn managed_inference_http_client() -> reqwest::Client {static CLIENT: std::sync::OnceLock<reqwest::Client> = std::sync::OnceLock::new();CLIENT@@ -367,6 +123,3 @@ fn managed_inference_http_client() -> reqwest::Client {.clone()}-#[cfg(test)]-#[path = "managed_model_tests.rs"]-mod tests;diff --git a/crates/openhuman-tinyhumans/src/inference/managed_session.rs b/crates/openhuman-tinyhumans/src/inference/managed_session.rsindex 4a164f2152..8bf99e051f 100644--- a/crates/openhuman-tinyhumans/src/inference/managed_session.rs+++ b/crates/openhuman-tinyhumans/src/inference/managed_session.rs@@ -3,6 +3,7 @@//! rejected session.use tinyinference_llm::model::ProviderError;+use tinyinference_llm::providers::tinyhumans::{TinyHumansAuthFailure, TinyHumansAuthFailureKind, TinyHumansOperation};use tinyinference_llm::Error as TiError;/// Resolve the persisted app-session token for a managed inference caller.@@ -91,30 +92,32 @@ fn maybe_publish_local_session_expiry(operation: &str) {);}-/// Publish a `SessionExpired` event when the backend rejects a crate-native-/// model call with `401`/`403` Unauthorized — mirroring the check in-/// the core `CrateBackedProvider::invoke` which the-/// crate-native path bypasses.-pub(super) fn maybe_publish_session_expired(err: &TiError, operation: &str) {- if let TiError::Provider(pe) = err {- maybe_publish_provider_session_expired(pe, operation);- }-}--pub(super) fn maybe_publish_provider_session_expired(pe: &ProviderError, operation: &str) {- if pe.provider.as_str() == "OpenHuman" && matches!(pe.status, Some(401 | 403)) {- let reason = tinyinference_core::sanitize::sanitize_api_error(&pe.message);- openhuman_embed::__host::core::bus::BUS.publish(- openhuman_embed::__host::core::events::DomainEvent::SessionExpired {- source: format!(- "openhuman_backend_model.{}({})",- operation,- pe.status.unwrap_or(0)- ),- reason,- },- );- }+/// Publish expiry only from TinyInference's explicit managed-endpoint+/// provenance. The provider label is a wire diagnostic and is not an auth+/// ownership signal.+pub(super) fn publish_managed_auth_failure(+ error: &ProviderError,+ context: TinyHumansAuthFailure,+) {+ let status = match context.kind() {+ TinyHumansAuthFailureKind::Unauthorized => 401,+ TinyHumansAuthFailureKind::Forbidden => 403,+ TinyHumansAuthFailureKind::Other => return,+ };+ let operation = match context.operation() {+ TinyHumansOperation::Invoke => "invoke",+ TinyHumansOperation::Stream => "stream",+ // Readiness deliberately remains fail-open and does not sign the user+ // out, matching its previous host behavior.+ TinyHumansOperation::Readiness => return,+ };+ let reason = tinyinference_core::sanitize::sanitize_api_error(&error.message);+ openhuman_embed::__host::core::bus::BUS.publish(+ openhuman_embed::__host::core::events::DomainEvent::SessionExpired {+ source: format!("openhuman_backend_model.{operation}({status})"),+ reason,+ },+ );}#[cfg(test)]