The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(provider): remove model implementation details
Steven Enamakelb08fab76457b
diff --git a/crates/openhuman-core/src/inference/provider/mod.rs b/crates/openhuman-core/src/inference/provider/mod.rsindex 4e28d99432..cd4751bb16 100644--- a/crates/openhuman-core/src/inference/provider/mod.rs+++ b/crates/openhuman-core/src/inference/provider/mod.rs@@ -8,13 +8,8 @@pub mod factory;pub(crate) mod fallback_diagnostics;pub mod http_error;-pub(crate) mod model;pub mod models;pub(crate) mod openai_codex;-pub mod types;--#[allow(unused_imports)]-pub use types::{BilledUsage, ChatResponse, ProviderDelta, AGENT_TURN_MAX_OUTPUT_TOKENS};pub use factory::{create_chat_model_from_string, create_chat_model_from_string_with_model_id,diff --git a/crates/openhuman-core/src/inference/provider/model.rs b/crates/openhuman-core/src/inference/provider/model.rsdeleted file mode 100644index c02b3b9500..0000000000--- a/crates/openhuman-core/src/inference/provider/model.rs+++ /dev/null@@ -1,204 +0,0 @@-//! OpenHuman helpers and wrappers for native TinyAgents [`ChatModel`] values.--use std::sync::Arc;--use tinyinference_llm::message::{AssistantMessage, ContentBlock};-use tinyinference_llm::model::{ChatModel, ModelRequest, ModelResponse};-use tinyinference_llm::tool::ToolCall as TaToolCall;-use tinyinference_llm::usage::{recover_usage, ProviderUsageReport};-use tinytools_agent::dialect::DialectResponse;--use crate::inference::provider::{BilledUsage, ChatResponse};--pub(crate) type TurnChatModel = Arc<dyn ChatModel<()>>;-pub(crate) type TierRoutes = Vec<(String, TurnChatModel)>;-pub(crate) type BuiltTurnModels = (TurnChatModel, TierRoutes, TurnChatModel);--/// Convert a host response into the canonical dialect response at the inference boundary.-pub(crate) fn dialect_response_from_provider(response: &ChatResponse) -> DialectResponse {- DialectResponse {- text: response.text.clone(),- tool_calls: response.tool_calls.clone(),- }-}--/// Build a [`PFormatRegistry`](tinytools_agent::PFormatRegistry)-/// from the tool schemas advertised on a [`ModelRequest`] (issue #4465).-///-/// The text-mode fallback parse needs each tool's positional parameter layout-/// to reconstruct named JSON arguments from a P-Format `name[a|b]` body. The-/// harness populates `request.tools` when tools are available (schemas are-/// rendered into the prompt for prompt-guided providers, or advertised natively-/// otherwise), so the registry is available in both modes. Tool-less requests-/// skip fallback parsing entirely; this empty registry is therefore consulted-/// only alongside a non-empty advertised tool list.-fn pformat_registry_from_request(request: &ModelRequest) -> tinytools_agent::PFormatRegistry {- request- .tools- .iter()- .map(|t| {- (- t.name.clone(),- tinytools_agent::PFormatToolParams::from_schema(&t.parameters),- )- })- .collect()-}--/// Translate an openhuman [`ChatResponse`] into a harness [`ModelResponse`]-/// (visible text + tool calls + token usage).-///-/// Native `tool_calls` take precedence; when absent and the request advertised-/// tools, the response text is parsed for prompt-guided (`<tool_call>…` /-/// p-format) calls — matching the legacy dispatcher — so text-mode models drive-/// the tinyagents loop too. Tool-less requests preserve response text verbatim,-/// including literal tool-call examples. When parsing is enabled, visible text-/// is the prose with any parsed tool-call markup stripped.-///-/// `pformat_registry` carries the advertised tools' positional layouts so the-/// text-mode fallback can recover P-Format (`name[a|b]`) calls that ~10 builtin-/// prompts still teach — the migrated parse path had dropped that grammar and-/// silently lost those calls (issue #4465). It is empty for the native-tool-/// path, where `response.tool_calls` is used directly.-///-/// Unknown-tool recovery is handled by `RunPolicy::unknown_tool`, so the model-/// adapter preserves the provider-requested tool name.-fn response_to_model_response(- response: &ChatResponse,- pformat_registry: &tinytools_agent::PFormatRegistry,- parse_text_tool_calls: bool,-) -> ModelResponse {- let (visible_text, tool_calls): (String, Vec<TaToolCall>) = if !response.tool_calls.is_empty() {- let calls = response- .tool_calls- .iter()- .map(|tc| TaToolCall {- id: tc.id.clone(),- name: tc.name.clone(),- arguments: serde_json::from_str(&tc.arguments).unwrap_or(serde_json::Value::Null),- invalid: None,- })- .collect();- (response.text.clone().unwrap_or_default(), calls)- } else if parse_text_tool_calls {- let text = response.text.as_deref().unwrap_or_default();- let (prose, parsed) =- tinytools_agent::parse_tool_calls_with_pformat(text, pformat_registry);- if parsed.is_empty() {- (text.to_string(), Vec::new())- } else {- let calls = parsed- .into_iter()- .enumerate()- .map(|(i, p)| TaToolCall {- // Prompt-guided calls carry no provider id; synthesize a- // stable one so tool results correlate in the harness.- id: p.id.unwrap_or_else(|| format!("call_{i}")),- name: p.name,- arguments: p.arguments,- invalid: None,- })- .collect();- (prose, calls)- }- } else {- (response.text.clone().unwrap_or_default(), Vec::new())- };-- let mut content = Vec::new();- if !visible_text.is_empty() {- content.push(ContentBlock::Text(visible_text));- }- // Thinking models return `reasoning_content` separately from the visible- // reply. Preserve it as a typed thinking block so it stays out of- // `Message::text()` but survives persistence and the next turn's request,- // where thinking-mode providers require it back.- if let Some(block) = tinyagents_runtime::transcript::reasoning_content_block(- response.reasoning_content.as_deref(),- ) {- content.push(block);- }- let report = response.usage.as_ref().map(billed_usage_report);- let usage = report.as_ref().map(ProviderUsageReport::to_usage);- let finish_reason = if tool_calls.is_empty() {- "stop"- } else {- "tool_calls"- };- ModelResponse {- message: AssistantMessage {- id: None,- content,- tool_calls,- usage,- origin: None,- },- usage,- finish_reason: Some(finish_reason.to_string()),- // The crate `Usage` has no field for the provider's **charged USD** or the- // model's **context window**, but `ModelResponse.raw` is exactly "raw- // provider metadata preserved for callers who need it" — so stash them- // there (gap G1). A standalone `invoke` then round-trips the OpenHuman- // managed backend's charged amount + window via- // [`usage_info_from_response`], no crate change required. Omitted when the- // provider reported neither (keeps non-managed responses byte-clean).- raw: report.as_ref().and_then(ProviderUsageReport::raw_meta),- resolved_model: None,- continue_turn: None,- served_from_cache: false,- correlation: None,- resolved_route: None,- }-}--/// Convert a host response while preserving the legacy text-tool fallback for-/// a request that advertised tools. This remains available to migration-/// fixtures that exercise the old XML/P-Format recovery contract without-/// constructing a [`native model adapter`].-pub(crate) fn native_model_response_for_request(- response: &ChatResponse,- request: &ModelRequest,-) -> ModelResponse {- response_to_model_response(- response,- &pformat_registry_from_request(request),- !request.tools.is_empty(),- )-}--/// Build the `ModelResponse.raw` value carrying charged-USD + context-window-/// metadata, or `None` when the provider reported neither.-fn billed_usage_report(u: &BilledUsage) -> ProviderUsageReport {- ProviderUsageReport {- input_tokens: u.input_tokens,- output_tokens: u.output_tokens,- cached_input_tokens: u.cached_input_tokens(),- cache_creation_tokens: u.cache_creation_tokens,- reasoning_tokens: u.reasoning_tokens,- charged_amount_usd: u.charged_amount_usd,- charge_reported: u.charge_reported,- cost_is_estimate: u.cost_is_estimate,- context_window: u.context_window(),- }-}--/// Reconstruct a host [`BilledUsage`] from a crate [`ModelResponse`], recovering-/// the provider-charged USD + context window stashed in [`ModelResponse::raw`]-/// by `tinyinference_llm::usage::recover_usage`. `None` when the response-/// carried no usage at all.-pub(crate) fn usage_info_from_response(response: &ModelResponse) -> Option<BilledUsage> {- let recovered = recover_usage(response)?;- let usage = recovered.usage;- Some(- BilledUsage::from_counts(usage.input_tokens, usage.output_tokens)- .with_context_window(recovered.context_window)- .with_cached_input_tokens(usage.cache_read_tokens)- .with_cache_creation_tokens(usage.cache_creation_tokens)- .with_reasoning_tokens(usage.reasoning_tokens)- .with_reported_charge(recovered.charged_amount_usd),- )-}--#[cfg(test)]-#[path = "model_g1_usage_tests_tests.rs"]-mod g1_usage_tests;diff --git a/crates/openhuman-core/src/inference/provider/model_g1_usage_tests_tests.rs b/crates/openhuman-core/src/inference/provider/model_g1_usage_tests_tests.rsdeleted file mode 100644index 7adc06f991..0000000000--- a/crates/openhuman-core/src/inference/provider/model_g1_usage_tests_tests.rs+++ /dev/null@@ -1,199 +0,0 @@-//! Gap G1: a standalone `invoke` must stay usage-faithful — token-//! breakdowns ride the crate `Usage`, and the two host fields with no crate-//! home (charged USD + context window) ride `ModelResponse.raw` and-//! reconstruct exactly via [`usage_info_from_response`].-use super::*;-use tinyinference_llm::usage::Usage;--fn empty_registry() -> tinytools_agent::PFormatRegistry {- tinytools_agent::PFormatRegistry::default()-}--#[test]-fn usage_round_trips_charged_usd_and_all_token_breakdowns() {- let chat = ChatResponse {- text: Some("hi".to_string()),- tool_calls: Vec::new(),- usage: Some(- BilledUsage::from_counts(100, 20)- .with_context_window(128_000)- .with_cached_input_tokens(40)- .with_cache_creation_tokens(10)- .with_reasoning_tokens(7)- .with_charged_usd(0.0123),- ),- reasoning_content: None,- };- let model_response = response_to_model_response(&chat, &empty_registry(), false);-- // Crate Usage carries every token breakdown natively.- let usage = model_response.usage.expect("usage present");- assert_eq!(usage.input_tokens, 100);- assert_eq!(usage.output_tokens, 20);- assert_eq!(usage.cache_read_tokens, 40);- assert_eq!(usage.cache_creation_tokens, 10);- assert_eq!(usage.reasoning_tokens, 7);- assert_eq!(- usage.charged_amount.map(|amount| amount.micros),- Some(12_300)- );- assert_eq!(usage.context_window_tokens, Some(128_000));-- // Charged USD + context window ride raw and reconstruct exactly.- let recovered = usage_info_from_response(&model_response).expect("usage info");- assert_eq!(recovered.input_tokens, 100);- assert_eq!(recovered.output_tokens, 20);- assert_eq!(recovered.context_window(), 128_000);- assert_eq!(recovered.cached_input_tokens(), 40);- assert_eq!(recovered.cache_creation_tokens, 10);- assert_eq!(recovered.reasoning_tokens, 7);- assert!((recovered.charged_amount_usd - 0.0123).abs() < 1e-9);-}--#[test]-fn no_billing_metadata_leaves_raw_clean() {- let chat = ChatResponse {- text: Some("hi".to_string()),- tool_calls: Vec::new(),- usage: Some(BilledUsage::from_counts(5, 3)),- reasoning_content: None,- };- let model_response = response_to_model_response(&chat, &empty_registry(), false);- assert!(- model_response.raw.is_none(),- "no charged USD / window ⇒ raw stays None"- );- let recovered = usage_info_from_response(&model_response).expect("usage info");- assert_eq!(recovered.charged_amount_usd, 0.0);- assert_eq!(recovered.context_window(), 0);- assert_eq!(recovered.input_tokens, 5);-}--#[test]-fn explicitly_reported_zero_charge_survives_in_native_usage() {- let chat = ChatResponse {- text: Some("free route".to_string()),- usage: Some(BilledUsage::from_counts(5, 3).with_charged_usd(0.0)),- ..Default::default()- };-- let response = response_to_model_response(&chat, &empty_registry(), false);- assert_eq!(- response- .usage- .and_then(|usage| usage.charged_amount)- .map(|charge| charge.micros),- Some(0),- "native journal usage must retain known-free versus unknown"- );-}--#[test]-fn provider_neutral_total_cost_metadata_is_recovered() {- let model_response = ModelResponse {- usage: Some(Usage::new(8, 3)),- raw: Some(serde_json::json!({"total_cost_usd": 0.0042})),- ..ModelResponse::assistant("done")- };-- let recovered = usage_info_from_response(&model_response).expect("usage info");- assert!((recovered.charged_amount_usd - 0.0042).abs() < 1e-9);-}--#[test]-fn no_usage_reconstructs_to_none() {- let chat = ChatResponse {- text: Some("hi".to_string()),- tool_calls: Vec::new(),- usage: None,- reasoning_content: None,- };- let model_response = response_to_model_response(&chat, &empty_registry(), false);- assert!(usage_info_from_response(&model_response).is_none());-}--#[test]-fn tool_less_response_preserves_literal_tool_call_markup() {- let text = r#"Example: <tool_call>{"name":"lookup","arguments":{}}</tool_call>"#;- let chat = ChatResponse {- text: Some(text.to_string()),- ..Default::default()- };-- let response = response_to_model_response(&chat, &empty_registry(), false);-- assert_eq!(response.text(), text);- assert!(response.message.tool_calls.is_empty());-}--#[test]-fn tool_enabled_response_still_extracts_tool_call_markup() {- let chat = ChatResponse {- text: Some(r#"<tool_call>{"name":"lookup","arguments":{}}</tool_call>"#.to_string()),- ..Default::default()- };-- let response = response_to_model_response(&chat, &empty_registry(), true);-- assert_eq!(response.text(), "");- assert_eq!(response.message.tool_calls.len(), 1);- assert_eq!(response.message.tool_calls[0].name, "lookup");-}--fn tool_request() -> ModelRequest {- ModelRequest {- tools: vec![tinyinference_llm::tool::ToolSchema::new(- "lookup",- "looks up a record",- serde_json::json!({- "type": "object",- "properties": {- "id": { "type": "integer" },- "query": { "type": "string" }- }- }),- )],- ..Default::default()- }-}--/// Wire shape of the billing/context metadata stashed in `ModelResponse.raw`:-/// a literal-JSON pin so the key and field names cannot drift, and a response-/// written by the current release (with the old key set) still reconstructs.-#[test]-fn usage_meta_raw_wire_shape_is_stable_and_old_payloads_load() {- let chat = ChatResponse {- text: Some("hi".to_string()),- tool_calls: Vec::new(),- usage: Some(- BilledUsage::from_counts(100, 20)- .with_context_window(128_000)- .with_charged_usd(0.0123),- ),- reasoning_content: None,- };- let written = response_to_model_response(&chat, &empty_registry(), false);- assert_eq!(- written.raw,- Some(serde_json::json!({- "openhuman_usage_meta": {"charged_amount_usd": 0.0123, "context_window": 128000}- }))- );-- let mut old = response_to_model_response(- &ChatResponse {- text: Some("x".into()),- usage: Some(BilledUsage::from_counts(7, 2)),- ..Default::default()- },- &empty_registry(),- false,- );- old.raw = Some(serde_json::json!({- "openhuman_usage_meta": {"charged_amount_usd": 0.5, "context_window": 32000}- }));- let recovered = usage_info_from_response(&old).expect("usage");- assert_eq!(recovered.charged_amount_usd, 0.5);- assert_eq!(recovered.context_window(), 32_000);- assert_eq!(recovered.input_tokens, 7);-}diff --git a/crates/openhuman-core/src/inference/provider/types.rs b/crates/openhuman-core/src/inference/provider/types.rsdeleted file mode 100644index bd53b5fb00..0000000000--- a/crates/openhuman-core/src/inference/provider/types.rs+++ /dev/null@@ -1,170 +0,0 @@-use tinyinference_llm::usage::Usage;-use tinytools_agent::dialect::NativeToolCall;--/// Token usage returned by a provider: the vendor [`Usage`] token counts plus-/// the host-owned billing the vendor type deliberately does not carry.-///-/// `usage` is the single token-count representation threaded everywhere-/// (cache reads/writes, reasoning tokens and the context window all have-/// homes on it). Only the provider-charged amount stays host data: it is kept-/// as an exact `f64` USD value (the vendor `ChargedAmount` is integer micro-/// units and would round the persisted cost rows). `Deref` exposes the-/// vendor fields (`input_tokens`, `output_tokens`, `cache_creation_tokens`,-/// `reasoning_tokens`, ...) directly.-#[derive(Debug, Clone, Copy, Default, PartialEq)]-pub struct BilledUsage {- /// Vendor token counts for the call.- pub usage: Usage,- /// Amount billed for this request in USD (from- /// `openhuman.billing.charged_amount_usd`). Zero when unavailable.- pub charged_amount_usd: f64,- /// `charged_amount_usd` is a local catalog estimate, not a provider charge- /// (see [`BilledUsage::with_estimated_usd`]).- pub cost_is_estimate: bool,- /// The provider reported what it billed for this call, so- /// `charged_amount_usd` is known even when it is zero (a free route).- /// `false` means no charge arrived; zero then means "unknown", not free.- pub charge_reported: bool,-}--impl BilledUsage {- /// Plain token counts with no cache breakdown, context window or charge.- pub fn from_counts(input_tokens: u64, output_tokens: u64) -> Self {- Self {- usage: Usage::new(input_tokens, output_tokens),- charged_amount_usd: 0.0,- cost_is_estimate: false,- charge_reported: false,- }- }-- /// Input tokens served from the provider prompt/KV cache.- pub fn cached_input_tokens(&self) -> u64 {- self.usage.cache_read_tokens- }-- /// Model context window in tokens; `0` when unknown.- pub fn context_window(&self) -> u64 {- self.usage.context_window_tokens.unwrap_or(0)- }-- /// Sets the cache-read token count.- pub fn with_cached_input_tokens(mut self, tokens: u64) -> Self {- self.usage.cache_read_tokens = tokens;- self- }-- /// Sets the context window (`0` means unknown).- pub fn with_context_window(mut self, tokens: u64) -> Self {- self.usage.context_window_tokens = (tokens > 0).then_some(tokens);- self- }-- /// Sets the cache-creation (write) token count.- pub fn with_cache_creation_tokens(mut self, tokens: u64) -> Self {- self.usage.cache_creation_tokens = tokens;- self- }-- /// Sets the reasoning/thinking token count.- pub fn with_reasoning_tokens(mut self, tokens: u64) -> Self {- self.usage.reasoning_tokens = tokens;- self- }-- /// Records what the provider billed for this call. Zero is a real charge- /// (a free route), not "unknown".- pub fn with_charged_usd(mut self, usd: f64) -> Self {- self.charged_amount_usd = usd;- self.cost_is_estimate = false;- self.charge_reported = true;- self- }-- /// Records a charge only when the provider reported one; `None` leaves- /// the cost unknown.- pub fn with_reported_charge(self, usd: Option<f64>) -> Self {- match usd {- Some(usd) => self.with_charged_usd(usd),- None => self,- }- }-- /// Sets a locally estimated USD cost, kept apart from provider charges so- /// usage reports can split charged from estimated spend.- pub fn with_estimated_usd(mut self, usd: f64) -> Self {- self.charged_amount_usd = usd;- self.cost_is_estimate = true;- self.charge_reported = false;- self- }-}--impl std::ops::Deref for BilledUsage {- type Target = Usage;- fn deref(&self) -> &Usage {- &self.usage- }-}--/// An LLM response that may contain text, tool calls, or both.-#[derive(Debug, Clone, Default)]-pub struct ChatResponse {- /// Text content of the response (may be empty if only tool calls).- pub text: Option<String>,- /// Tool calls requested by the LLM.- pub tool_calls: Vec<NativeToolCall>,- /// Token usage info from the provider (if available).- pub usage: Option<BilledUsage>,- /// Raw reasoning/thinking content returned by thinking models (e.g.- /// DeepSeek-R1, Qwen3) in the `reasoning_content` field. This must be- /// passed back verbatim on the next turn — the API returns HTTP 400- /// ("reasoning_content in thinking mode must be passed back") if it is- /// omitted from the assistant message in a multi-turn conversation.- ///- /// Stored separately from `text` so callers can preserve it through- /// the conversation history without merging it into the visible reply.- pub reasoning_content: Option<String>,-}--impl ChatResponse {}--/// A fine-grained streaming event emitted by a provider while serving a-/// `chat()` call. Providers that support SSE/streaming forward these to-/// the optional sender on [`ChatRequest::stream`]; the final aggregated-/// response is still returned from `chat()` so callers that ignore the-/// stream keep working unchanged.-#[derive(Debug, Clone)]-pub enum ProviderDelta {- /// A chunk of the assistant's visible text output.- TextDelta { delta: String },- /// A chunk of the model's reasoning/thinking output (for models- /// that emit `reasoning_content` or an equivalent). Consumers should- /// render this in a separate UI affordance from the visible output.- ThinkingDelta { delta: String },- /// The start of a new native tool call. `call_id` is the- /// provider-assigned id that later appears on the result message.- ToolCallStart { call_id: String, tool_name: String },- /// A chunk of argument JSON text for an in-flight tool call.- /// Streamed verbatim; may arrive as partial JSON that only becomes- /// valid once the stream completes.- ToolCallArgsDelta { call_id: String, delta: String },-}--/// Upper bound on output tokens requested for an agent chat turn.-///-/// The agent loop used to leave `ChatRequest::max_tokens` `None` ("open-ended-/// generation"), but an unset cap makes reservation-pricing providers (e.g.-/// OpenRouter) reserve credit against the model's *entire* output window-/// (64k+) during their pre-flight balance check — so a modest-balance BYO user-/// can hit a `402` purely from the oversized reservation, a **preventable**-/// condition. Capping every agent turn at a realistic ceiling prices the-/// pre-flight against a budget the user can actually afford; a residual `402`-/// is then the genuine flat-balance case the insufficient-credits demote arm-/// is meant for (TAURI-RUST-C62).-///-/// `16384` sits comfortably above any realistic single agent turn — `max_tokens`-/// is an upper bound, not a forced length, so the model still stops at its-/// natural end well below the cap on normal turns — while cutting the-/// reservation 4× versus a 64k window.-pub const AGENT_TURN_MAX_OUTPUT_TOKENS: u32 = 16384;diff --git a/crates/openhuman-core/src/inference/turn_models.rs b/crates/openhuman-core/src/inference/turn_models.rsindex 3a357db070..dc3aea29bc 100644--- a/crates/openhuman-core/src/inference/turn_models.rs+++ b/crates/openhuman-core/src/inference/turn_models.rs@@ -3,9 +3,12 @@use std::sync::Arc;+pub(crate) type TurnChatModel = Arc<dyn tinyinference_llm::model::ChatModel<()>>;+pub(crate) type TierRoutes = Vec<(String, TurnChatModel)>;+pub(crate) type BuiltTurnModels = (TurnChatModel, TierRoutes, TurnChatModel);+use crate::core::runtime::DispatchCtx;-use crate::inference::provider::model::{BuiltTurnModels, TierRoutes, TurnChatModel};use tinyinference_llm::model::ProfileOverrideModel;use tinyinference_llm::model::{ResolvedModelRoute, RouteRecordingModel};