A high-performance inference engine for AI models
Bindings migration 3/n (#355)
Aleksei Savin6139794b9fb2
diff --git a/.env.sample b/.env.sampleindex 42ed54a35..06e02c764 100644--- a/.env.sample+++ b/.env.sample@@ -1,6 +1,9 @@MIRAI_API_KEY=+LALAMO_PATH=HF_TOKEN=OPENAI_API_KEY=-BASETEN_API_KEY=ANTHROPIC_API_KEY=+GEMINI_API_KEY=+XAI_API_KEY=+BASETEN_API_KEY=OPENROUTER_API_KEY=\ No newline at end of filediff --git a/Cargo.lock b/Cargo.lockindex d45c984f8..22747da7f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -256,6 +256,46 @@ dependencies = ["tokio",]+[[package]]+name = "async-openai"+version = "0.35.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "583553a05a8cbf55bbd67d6a824a206c21030556fdc5f43de387959f888fb6be"+dependencies = [+ "async-openai-macros",+ "backoff",+ "base64 0.22.1",+ "bytes",+ "derive_builder",+ "eventsource-stream",+ "futures",+ "getrandom 0.3.4",+ "rand 0.9.4",+ "reqwest 0.12.28",+ "reqwest-eventsource",+ "secrecy",+ "serde",+ "serde_json",+ "serde_urlencoded",+ "thiserror 2.0.18",+ "tokio",+ "tokio-stream",+ "tokio-util",+ "tracing",+ "url",+]++[[package]]+name = "async-openai-macros"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "81872a8e595e8ceceab71c6ba1f9078e313b452a1e31934e6763ef5d308705e4"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.117",+]+[[package]]name = "async-shutdown"version = "0.2.2"@@ -502,6 +542,21 @@ dependencies = ["fs_extra",]+[[package]]+name = "backend-remote"+version = "0.1.9"+dependencies = [+ "async-openai",+ "futures",+ "indexmap 2.14.0",+ "reqwest 0.13.2",+ "serde",+ "serde_json",+ "shoji",+ "thiserror 2.0.18",+ "tokio-util",+]+[[package]]name = "backend-uzu"version = "0.1.9"@@ -555,12 +610,27 @@ dependencies = ["thiserror 2.0.18","tokenizers","tokio",+ "tokio-util","walkdir","xgrammar-rs","xxhash-rust","zerocopy",]+[[package]]+name = "backoff"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"+dependencies = [+ "futures-core",+ "getrandom 0.2.17",+ "instant",+ "pin-project-lite",+ "rand 0.8.5",+ "tokio",+]+[[package]]name = "base64"version = "0.13.1"@@ -2026,6 +2096,17 @@ dependencies = ["num-traits",]+[[package]]+name = "eventsource-stream"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab"+dependencies = [+ "futures-core",+ "nom 7.1.3",+ "pin-project-lite",+]+[[package]]name = "exr"version = "1.74.0"@@ -2332,6 +2413,12 @@ version = "0.3.32"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"+[[package]]+name = "futures-timer"+version = "3.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"+[[package]]name = "futures-util"version = "0.3.32"@@ -2753,6 +2840,7 @@ dependencies = ["hyper 1.9.0","hyper-util","rustls",+ "rustls-native-certs","tokio","tokio-rustls","tower-service",@@ -3073,6 +3161,15 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "instant"+version = "0.1.13"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"+dependencies = [+ "cfg-if 1.0.4",+]+[[package]]name = "interpolate_name"version = "0.2.4"@@ -3784,6 +3881,7 @@ dependencies = [name = "nagare"version = "0.1.9"dependencies = [+ "futures","hanashi","indexmap 2.14.0","reqwest 0.13.2",@@ -3791,6 +3889,9 @@ dependencies = ["serde_json","shoji","thiserror 2.0.18",+ "tokio",+ "tokio-stream",+ "tokio-util",][[package]]@@ -4232,22 +4333,6 @@ version = "11.1.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"-[[package]]-name = "openai-api-rs"-version = "10.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ba538da8c96b5f486ec8706a1b4663fa9c46be33cf12f8bf43e82e4c7da8acb5"-dependencies = [- "bytes",- "futures-util",- "reqwest 0.12.28",- "serde",- "serde_json",- "tokio",- "tokio-tungstenite",- "url",-]-[[package]]name = "openai-harmony"version = "0.0.8"@@ -5331,6 +5416,7 @@ dependencies = ["pin-project-lite","quinn","rustls",+ "rustls-native-certs","rustls-pki-types","serde","serde_json",@@ -5396,6 +5482,22 @@ dependencies = ["web-sys",]+[[package]]+name = "reqwest-eventsource"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "632c55746dbb44275691640e7b40c907c16a2dc1a5842aa98aaec90da6ec6bde"+dependencies = [+ "eventsource-stream",+ "futures-core",+ "futures-timer",+ "mime",+ "nom 7.1.3",+ "pin-project-lite",+ "reqwest 0.12.28",+ "thiserror 1.0.69",+]+[[package]]name = "rgb"version = "0.8.53"@@ -5754,6 +5856,16 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "secrecy"+version = "0.10.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a"+dependencies = [+ "serde",+ "zeroize",+]+[[package]]name = "security-framework"version = "3.7.0"@@ -5973,6 +6085,7 @@ name = "shoji"version = "0.1.9"dependencies = ["bindings",+ "futures","indexmap 2.14.0","napi","napi-build",@@ -5982,6 +6095,7 @@ dependencies = ["serde-wasm-bindgen","serde_json","thiserror 2.0.18",+ "tokio-util","tsify","uniffi","wasm-bindgen",@@ -6626,20 +6740,6 @@ dependencies = ["tokio-util",]-[[package]]-name = "tokio-tungstenite"-version = "0.28.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857"-dependencies = [- "futures-util",- "log",- "native-tls",- "tokio",- "tokio-native-tls",- "tungstenite",-]-[[package]]name = "tokio-util"version = "0.7.18"@@ -6649,6 +6749,7 @@ dependencies = ["bytes","futures-core","futures-sink",+ "futures-util","pin-project-lite","tokio",]@@ -6905,24 +7006,6 @@ dependencies = ["syn 2.0.117",]-[[package]]-name = "tungstenite"-version = "0.28.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442"-dependencies = [- "bytes",- "data-encoding",- "http 1.4.0",- "httparse",- "log",- "native-tls",- "rand 0.9.4",- "sha1",- "thiserror 2.0.18",- "utf-8",-]-[[package]]name = "typenum"version = "1.19.0"@@ -7207,12 +7290,6 @@ dependencies = ["serde",]-[[package]]-name = "utf-8"-version = "0.7.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"-[[package]]name = "utf8-zero"version = "0.8.1"@@ -7258,12 +7335,15 @@ dependencies = [name = "uzu"version = "0.1.9"dependencies = [+ "async-openai",+ "backend-remote","backend-uzu","bindings","chrono","dirs","dotenvy","download-manager",+ "fancy-regex 0.17.0","futures","futures-util","indexmap 2.14.0",@@ -7275,9 +7355,7 @@ dependencies = ["objc2","objc2-foundation","objc2-security",- "openai-api-rs","pyo3",- "regex","reqwest 0.13.2","serde","serde-wasm-bindgen",diff --git a/Cargo.toml b/Cargo.tomlindex 450193eef..3eb8f5043 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -3,12 +3,13 @@ members = ["crates/dsl","crates/bindings","crates/shoji",+ "crates/backend-uzu",+ "crates/backend-remote","crates/json-transform","crates/token-stream-parser","crates/hanashi","crates/nagare","crates/download-manager",- "crates/backend-uzu","crates/uzu","crates/benchmarks","crates/cli",@@ -28,13 +29,14 @@ repository = "https://github.com/trymirai/uzu"dsl = { path = "crates/dsl" }bindings = { path = "crates/bindings", default-features = false }shoji = { path = "crates/shoji", default-features = false }+backend-uzu = { path = "crates/backend-uzu", default-features = false }+backend-remote = { path = "crates/backend-remote", default-features = false }json-transform = { path = "crates/json-transform", default-features = false }token-stream-parser = { path = "crates/token-stream-parser", default-features = false }hanashi = { path = "crates/hanashi", default-features = false }nagare = { path = "crates/nagare", default-features = false }download-manager = { path = "crates/download-manager", default-features = false }benchmarks = { path = "crates/benchmarks", default-features = false }-backend-uzu = { path = "crates/backend-uzu", default-features = false }uzu = { path = "crates/uzu", default-features = false }xgrammar = { package = "xgrammar-rs", version = "0.1", features = ["hf"] }half = { version = "2.7", features = ["bytemuck", "num-traits"] }@@ -118,7 +120,7 @@ reqwest = { version = "0.13.2", features = ["blocking","query",] }-tokio-util = "0.7.17"+tokio-util = { version = "0.7.17", default-features = false, features = ["rt"] }async-shutdown = "0.2.2"objc-sys = "0.3.5"crc32c = "0.6.8"@@ -139,7 +141,7 @@ dirs = "6.0.0"dotenvy = "0.15.7"ratatui = "0.30.0"crossterm = "0.29.0"-openai-api-rs = "10.0.0"+async-openai = { version = "0.35.0", features = ["chat-completion", "model", "byot", "responses"] }[profile.dev.build-override]opt-level = 3diff --git a/crates/backend-remote/Cargo.toml b/crates/backend-remote/Cargo.tomlnew file mode 100644index 000000000..c35ff571b--- /dev/null+++ b/crates/backend-remote/Cargo.toml@@ -0,0 +1,19 @@+[package]+name = "backend-remote"+version.workspace = true+edition.workspace = true+rust-version.workspace = true+license.workspace = true+authors.workspace = true+repository.workspace = true++[dependencies]+shoji.workspace = true+async-openai.workspace = true+futures.workspace = true+serde_json.workspace = true+thiserror.workspace = true+tokio-util.workspace = true+serde.workspace = true+reqwest.workspace = true+indexmap.workspace = truediff --git a/crates/backend-remote/src/lib.rs b/crates/backend-remote/src/lib.rsnew file mode 100644index 000000000..d8c308735--- /dev/null+++ b/crates/backend-remote/src/lib.rs@@ -0,0 +1 @@+pub mod openai;diff --git a/crates/backend-remote/src/openai/bridging/chat/api_stream.rs b/crates/backend-remote/src/openai/bridging/chat/api_stream.rsnew file mode 100644index 000000000..4c35a9aa2--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/chat/api_stream.rs@@ -0,0 +1,66 @@+use std::{pin::Pin, sync::Arc};++use async_openai::{Client, config::OpenAIConfig};+use futures::{FutureExt, Stream, StreamExt, stream};+use shoji::{+ traits::backend::{Error as BackendError, chat_message::Output},+ types::{encoding::Message, session::chat::StreamConfig},+};+use tokio_util::sync::CancellationToken;++use crate::openai::{+ Error,+ bridging::{+ self,+ chat::{request, stream_chunk},+ },+ stream_state::StreamState,+};++pub struct ApiStream;++impl bridging::ApiStream for ApiStream {+ fn stream(+ &self,+ client: Arc<Client<OpenAIConfig>>,+ model: String,+ config: StreamConfig,+ messages: Vec<Message>,+ cancel: CancellationToken,+ ) -> Pin<Box<dyn Stream<Item = Result<Output, BackendError>> + Send>> {+ let stream = async move {+ let request = match request::build(&model, &config, messages) {+ Ok(request) => request,+ Err(error) => {+ return stream::once(async move { Err(Box::new(error) as BackendError) }).boxed();+ },+ };+ match client.chat().create_stream_byot::<_, stream_chunk::Response>(request).await {+ Ok(upstream) => {+ let mut state = StreamState::new();+ upstream+ .filter_map(move |response| {+ let item = match response {+ Ok(response) => {+ stream_chunk::build(response).and_then(|chunk| state.process_chunk(chunk)).map(Ok)+ },+ Err(error) => Some(Err(Box::new(Error::Network {+ message: error.to_string(),+ }) as BackendError)),+ };+ futures::future::ready(item)+ })+ .boxed()+ },+ Err(error) => stream::once(async move {+ Err(Box::new(Error::Network {+ message: error.to_string(),+ }) as BackendError)+ })+ .boxed(),+ }+ };++ stream.flatten_stream().take_until(cancel.cancelled_owned()).boxed()+ }+}diff --git a/crates/backend-remote/src/openai/bridging/chat/finish_reason.rs b/crates/backend-remote/src/openai/bridging/chat/finish_reason.rsnew file mode 100644index 000000000..74e673661--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/chat/finish_reason.rs@@ -0,0 +1,12 @@+use async_openai::types::chat::FinishReason as ChatFinishReason;+use shoji::types::session::chat::FinishReason;++pub fn build(reason: ChatFinishReason) -> FinishReason {+ match reason {+ ChatFinishReason::Stop => FinishReason::Stop,+ ChatFinishReason::Length => FinishReason::Length,+ ChatFinishReason::ToolCalls => FinishReason::ToolCalls,+ ChatFinishReason::ContentFilter => FinishReason::Rejected,+ ChatFinishReason::FunctionCall => FinishReason::ToolCalls,+ }+}diff --git a/crates/backend-remote/src/openai/bridging/chat/message.rs b/crates/backend-remote/src/openai/bridging/chat/message.rsnew file mode 100644index 000000000..309cca8bf--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/chat/message.rs@@ -0,0 +1,86 @@+use async_openai::types::chat::{+ ChatCompletionMessageToolCall, ChatCompletionMessageToolCalls, ChatCompletionRequestAssistantMessage,+ ChatCompletionRequestAssistantMessageContent, ChatCompletionRequestMessage, ChatCompletionRequestSystemMessage,+ ChatCompletionRequestSystemMessageContent, ChatCompletionRequestToolMessage,+ ChatCompletionRequestToolMessageContent, ChatCompletionRequestUserMessage, ChatCompletionRequestUserMessageContent,+ FunctionCall,+};+use shoji::types::encoding::{Message, Role};++use crate::openai::Error;++enum ResolvedRole {+ System,+ User,+ Assistant,+ Tool,+}++pub fn build(message: &Message) -> Result<ChatCompletionRequestMessage, Error> {+ let (role, content, tool_calls, tool_call_id) = match message.role {+ Role::System {} => (ResolvedRole::System, message.text(), None, None),+ Role::User {} => (ResolvedRole::User, message.text(), None, None),+ Role::Assistant {} => (ResolvedRole::Assistant, message.text(), Some(message.tool_calls()), None),+ Role::Tool {} => {+ let tool_call_results = message.tool_call_results();+ if tool_call_results.len() != 1 {+ return Err(Error::ToolCallResultRequired);+ }+ let (tool_call_id, _, value) = tool_call_results.first().ok_or(Error::ToolCallResultRequired)?;+ let content = serde_json::to_string(&value).map_err(|error| Error::Serialization {+ message: error.to_string(),+ })?;+ (ResolvedRole::Tool, content, None, tool_call_id.clone())+ },+ Role::Developer {}+ | Role::Custom {+ ..+ } => return Err(Error::UnsupportedRole),+ };++ let tool_calls: Option<Vec<ChatCompletionMessageToolCalls>> = tool_calls.map(|calls| {+ calls+ .into_iter()+ .map(|tool_call| {+ ChatCompletionMessageToolCalls::Function(ChatCompletionMessageToolCall {+ id: tool_call.identifier.unwrap_or_default(),+ function: FunctionCall {+ name: tool_call.name.clone(),+ arguments: tool_call.arguments.json.clone(),+ },+ })+ })+ .collect()+ });++ Ok(match role {+ ResolvedRole::System => ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage {+ content: ChatCompletionRequestSystemMessageContent::Text(content),+ name: None,+ }),+ ResolvedRole::User => ChatCompletionRequestMessage::User(ChatCompletionRequestUserMessage {+ content: ChatCompletionRequestUserMessageContent::Text(content),+ name: None,+ }),+ ResolvedRole::Assistant => {+ let text_content = if content.is_empty() {+ None+ } else {+ Some(ChatCompletionRequestAssistantMessageContent::Text(content))+ };+ #[allow(deprecated)]+ ChatCompletionRequestMessage::Assistant(ChatCompletionRequestAssistantMessage {+ content: text_content,+ refusal: None,+ name: None,+ audio: None,+ tool_calls,+ function_call: None,+ })+ },+ ResolvedRole::Tool => ChatCompletionRequestMessage::Tool(ChatCompletionRequestToolMessage {+ content: ChatCompletionRequestToolMessageContent::Text(content),+ tool_call_id: tool_call_id.unwrap_or_default(),+ }),+ })+}diff --git a/crates/backend-remote/src/openai/bridging/chat/mod.rs b/crates/backend-remote/src/openai/bridging/chat/mod.rsnew file mode 100644index 000000000..0b53ca265--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/chat/mod.rs@@ -0,0 +1,6 @@+pub mod api_stream;+pub mod finish_reason;+pub mod message;+pub mod request;+pub mod stream_chunk;+pub mod tool;diff --git a/crates/backend-remote/src/openai/bridging/chat/request.rs b/crates/backend-remote/src/openai/bridging/chat/request.rsnew file mode 100644index 000000000..dfdefdec0--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/chat/request.rs@@ -0,0 +1,70 @@+use async_openai::types::chat::{+ ChatCompletionStreamOptions, ChatCompletionToolChoiceOption, CreateChatCompletionRequest, ToolChoiceOptions,+};+use shoji::types::{+ basic::{SamplingMethod, SamplingPolicy},+ encoding::{Message, MessageList},+ session::chat::StreamConfig,+};++use crate::openai::{+ Error,+ bridging::{chat, reasoning_effort},+};++pub fn build(+ model: &str,+ config: &StreamConfig,+ messages: Vec<Message>,+) -> Result<CreateChatCompletionRequest, Error> {+ let completion_messages = messages.iter().map(chat::message::build).collect::<Result<Vec<_>, _>>()?;++ let tools = messages+ .tool_namespaces()+ .into_iter()+ .flat_map(|namespace| namespace.tools.into_iter())+ .map(chat::tool::build)+ .collect::<Result<Vec<_>, _>>()?;++ let reasoning_effort = messages.reasoning_effort().map(reasoning_effort::build);++ let mut request = CreateChatCompletionRequest {+ messages: completion_messages,+ model: model.to_string(),+ stream: Some(true),+ stream_options: Some(ChatCompletionStreamOptions {+ include_usage: Some(true),+ include_obfuscation: None,+ }),+ ..Default::default()+ };+ if !tools.is_empty() {+ request.tools = Some(tools);+ request.tool_choice = Some(ChatCompletionToolChoiceOption::Mode(ToolChoiceOptions::Auto));+ }+ if let Some(effort) = reasoning_effort {+ request.reasoning_effort = Some(effort);+ }+ if let Some(token_limit) = config.token_limit {+ request.max_completion_tokens = Some(token_limit as u32);+ }+ match &config.sampling_policy {+ SamplingPolicy::Default {} => {},+ SamplingPolicy::Custom {+ method,+ } => match method {+ SamplingMethod::Greedy {} => {},+ SamplingMethod::Stochastic {+ temperature,+ top_k: _,+ top_p,+ min_p: _,+ } => {+ request.temperature = temperature.map(|v| v as f32);+ request.top_p = top_p.map(|v| v as f32);+ },+ },+ }++ Ok(request)+}diff --git a/crates/backend-remote/src/openai/bridging/chat/stream_chunk.rs b/crates/backend-remote/src/openai/bridging/chat/stream_chunk.rsnew file mode 100644index 000000000..21304056d--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/chat/stream_chunk.rs@@ -0,0 +1,104 @@+use async_openai::types::chat::FinishReason;+use serde::Deserialize;++use crate::openai::{+ bridging::chat::finish_reason,+ stream_state::{StreamChunk, ToolCallChunk},+};++#[derive(Debug, Deserialize)]+pub struct Response {+ #[serde(default)]+ choices: Vec<Choice>,+ #[serde(default)]+ usage: Option<Usage>,+}++#[derive(Debug, Deserialize)]+struct Choice {+ delta: Delta,+ #[serde(default)]+ finish_reason: Option<FinishReason>,+}++#[derive(Debug, Deserialize)]+struct Delta {+ #[serde(default)]+ content: Option<String>,+ #[serde(default)]+ tool_calls: Option<Vec<ToolCall>>,+}++#[derive(Debug, Deserialize)]+struct ToolCall {+ #[serde(default)]+ index: Option<u32>,+ #[serde(default)]+ id: Option<String>,+ #[serde(default)]+ function: Option<FunctionCallStream>,+}++#[derive(Debug, Deserialize)]+struct FunctionCallStream {+ #[serde(default)]+ name: Option<String>,+ #[serde(default)]+ arguments: Option<String>,+}++#[derive(Debug, Deserialize)]+struct Usage {+ #[serde(default)]+ prompt_tokens: Option<u32>,+ #[serde(default)]+ completion_tokens: Option<u32>,+}++pub fn build(response: Response) -> Option<StreamChunk> {+ let usage = response.usage;+ let choice = response.choices.into_iter().next();+ let tokens_input = usage.as_ref().and_then(|usage| usage.prompt_tokens);+ let tokens_output = usage.as_ref().and_then(|usage| usage.completion_tokens);++ let (content, tool_calls, finish_reason) = match choice {+ Some(choice) => {+ let delta = choice.delta;+ let tool_calls = delta+ .tool_calls+ .unwrap_or_default()+ .into_iter()+ .enumerate()+ .map(|(position, call)| {+ let (name, arguments) = call.function.map_or((None, None), |f| (f.name, f.arguments));+ ToolCallChunk {+ index: call.index.unwrap_or(position as u32),+ id: call.id,+ name,+ arguments,+ }+ })+ .collect();+ (delta.content, tool_calls, choice.finish_reason.map(finish_reason::build))+ },+ None => (None, Vec::new(), None),+ };++ if content.is_none()+ && tool_calls.is_empty()+ && finish_reason.is_none()+ && tokens_input.is_none()+ && tokens_output.is_none()+ {+ return None;+ }++ Some(StreamChunk {+ content,+ reasoning: None,+ tool_calls,+ finish_reason,+ tokens_input,+ tokens_output,+ })+}diff --git a/crates/backend-remote/src/openai/bridging/chat/tool.rs b/crates/backend-remote/src/openai/bridging/chat/tool.rsnew file mode 100644index 000000000..555b1bfdd--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/chat/tool.rs@@ -0,0 +1,26 @@+use async_openai::types::chat::{ChatCompletionTool, ChatCompletionTools, FunctionObject};+use shoji::types::encoding::ToolDescription;++use crate::openai::Error;++pub fn build(description: ToolDescription) -> Result<ChatCompletionTools, Error> {+ let ToolDescription::Function {+ function,+ } = description;+ let parameters = match function.parameters {+ Some(value) => {+ Some(serde_json::from_str::<serde_json::Value>(&value.json).map_err(|error| Error::Serialization {+ message: error.to_string(),+ })?)+ },+ None => None,+ };+ Ok(ChatCompletionTools::Function(ChatCompletionTool {+ function: FunctionObject {+ name: function.name,+ description: Some(function.description),+ parameters,+ strict: None,+ },+ }))+}diff --git a/crates/backend-remote/src/openai/bridging/mod.rs b/crates/backend-remote/src/openai/bridging/mod.rsnew file mode 100644index 000000000..0af2cf9d7--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/mod.rs@@ -0,0 +1,24 @@+pub mod chat;+pub mod reasoning_effort;+pub mod responses;++use std::{pin::Pin, sync::Arc};++use async_openai::{Client, config::OpenAIConfig};+use futures::Stream;+use shoji::{+ traits::backend::{Error as BackendError, chat_message::Output},+ types::{encoding::Message, session::chat::StreamConfig},+};+use tokio_util::sync::CancellationToken;++pub trait ApiStream: Send + Sync {+ fn stream(+ &self,+ client: Arc<Client<OpenAIConfig>>,+ model: String,+ config: StreamConfig,+ messages: Vec<Message>,+ cancel: CancellationToken,+ ) -> Pin<Box<dyn Stream<Item = Result<Output, BackendError>> + Send>>;+}diff --git a/crates/backend-remote/src/openai/bridging/reasoning_effort.rs b/crates/backend-remote/src/openai/bridging/reasoning_effort.rsnew file mode 100644index 000000000..bbaa5d96d--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/reasoning_effort.rs@@ -0,0 +1,12 @@+use async_openai::types::chat::ReasoningEffort as OpenAIReasoningEffort;+use shoji::types::encoding::ReasoningEffort;++pub fn build(effort: ReasoningEffort) -> OpenAIReasoningEffort {+ match effort {+ ReasoningEffort::Disabled => OpenAIReasoningEffort::None,+ ReasoningEffort::Default => OpenAIReasoningEffort::Medium,+ ReasoningEffort::Low => OpenAIReasoningEffort::Low,+ ReasoningEffort::Medium => OpenAIReasoningEffort::Medium,+ ReasoningEffort::High => OpenAIReasoningEffort::High,+ }+}diff --git a/crates/backend-remote/src/openai/bridging/responses/api_stream.rs b/crates/backend-remote/src/openai/bridging/responses/api_stream.rsnew file mode 100644index 000000000..1b7ab79b5--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/responses/api_stream.rs@@ -0,0 +1,66 @@+use std::{pin::Pin, sync::Arc};++use async_openai::{Client, config::OpenAIConfig};+use futures::{FutureExt, Stream, StreamExt, stream};+use shoji::{+ traits::backend::{Error as BackendError, chat_message::Output},+ types::{encoding::Message, session::chat::StreamConfig},+};+use tokio_util::sync::CancellationToken;++use crate::openai::{+ Error,+ bridging::{+ self,+ responses::{request, stream_chunk},+ },+ stream_state::StreamState,+};++pub struct ApiStream;++impl bridging::ApiStream for ApiStream {+ fn stream(+ &self,+ client: Arc<Client<OpenAIConfig>>,+ model: String,+ config: StreamConfig,+ messages: Vec<Message>,+ cancel: CancellationToken,+ ) -> Pin<Box<dyn Stream<Item = Result<Output, BackendError>> + Send>> {+ let stream = async move {+ let request = match request::build(&model, &config, messages) {+ Ok(request) => request,+ Err(error) => {+ return stream::once(async move { Err(Box::new(error) as BackendError) }).boxed();+ },+ };+ match client.responses().create_stream(request).await {+ Ok(upstream) => {+ let mut state = StreamState::new();+ upstream+ .filter_map(move |event| {+ let item = match event {+ Ok(event) => {+ stream_chunk::build(event).and_then(|chunk| state.process_chunk(chunk)).map(Ok)+ },+ Err(error) => Some(Err(Box::new(Error::Network {+ message: error.to_string(),+ }) as BackendError)),+ };+ futures::future::ready(item)+ })+ .boxed()+ },+ Err(error) => stream::once(async move {+ Err(Box::new(Error::Network {+ message: error.to_string(),+ }) as BackendError)+ })+ .boxed(),+ }+ };++ stream.flatten_stream().take_until(cancel.cancelled_owned()).boxed()+ }+}diff --git a/crates/backend-remote/src/openai/bridging/responses/finish_reason.rs b/crates/backend-remote/src/openai/bridging/responses/finish_reason.rsnew file mode 100644index 000000000..564137639--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/responses/finish_reason.rs@@ -0,0 +1,10 @@+use async_openai::types::responses::Status;+use shoji::types::session::chat::FinishReason;++pub fn build(status: &Status) -> Option<FinishReason> {+ match status {+ Status::Completed | Status::Failed => Some(FinishReason::Stop),+ Status::Cancelled => Some(FinishReason::Cancelled),+ _ => None,+ }+}diff --git a/crates/backend-remote/src/openai/bridging/responses/message.rs b/crates/backend-remote/src/openai/bridging/responses/message.rsnew file mode 100644index 000000000..da95db218--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/responses/message.rs@@ -0,0 +1,60 @@+use async_openai::types::responses::{+ EasyInputContent, EasyInputMessage, FunctionCallOutput, FunctionCallOutputItemParam, FunctionToolCall, InputItem,+ Item, MessageType, Role as ResponseRole,+};+use shoji::types::encoding::{Message, Role};++use crate::openai::Error;++pub fn build(message: &Message) -> Result<Vec<InputItem>, Error> {+ let mut items = Vec::new();+ match message.role {+ Role::System {} | Role::Developer {} | Role::User {} | Role::Assistant {} => {+ let role = match message.role {+ Role::System {} => ResponseRole::System,+ Role::Developer {} => ResponseRole::Developer,+ Role::User {} => ResponseRole::User,+ Role::Assistant {} => ResponseRole::Assistant,+ _ => return Err(Error::UnsupportedRole),+ };+ let text = message.text();+ if !text.is_empty() {+ items.push(InputItem::EasyMessage(EasyInputMessage {+ r#type: MessageType::default(),+ role,+ content: EasyInputContent::Text(text),+ phase: None,+ }));+ }+ if matches!(message.role, Role::Assistant {}) {+ for tool_call in message.tool_calls() {+ items.push(InputItem::Item(Item::FunctionCall(FunctionToolCall {+ arguments: tool_call.arguments.json.clone(),+ call_id: tool_call.identifier.unwrap_or_default(),+ namespace: None,+ name: tool_call.name,+ id: None,+ status: None,+ })));+ }+ }+ },+ Role::Tool {} => {+ for (call_id, _name, value) in message.tool_call_results() {+ let content = serde_json::to_string(&value).map_err(|error| Error::Serialization {+ message: error.to_string(),+ })?;+ items.push(InputItem::Item(Item::FunctionCallOutput(FunctionCallOutputItemParam {+ call_id: call_id.unwrap_or_default(),+ output: FunctionCallOutput::Text(content),+ id: None,+ status: None,+ })));+ }+ },+ Role::Custom {+ ..+ } => return Err(Error::UnsupportedRole),+ }+ Ok(items)+}diff --git a/crates/backend-remote/src/openai/bridging/responses/mod.rs b/crates/backend-remote/src/openai/bridging/responses/mod.rsnew file mode 100644index 000000000..0b53ca265--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/responses/mod.rs@@ -0,0 +1,6 @@+pub mod api_stream;+pub mod finish_reason;+pub mod message;+pub mod request;+pub mod stream_chunk;+pub mod tool;diff --git a/crates/backend-remote/src/openai/bridging/responses/request.rs b/crates/backend-remote/src/openai/bridging/responses/request.rsnew file mode 100644index 000000000..44bf6533e--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/responses/request.rs@@ -0,0 +1,71 @@+use async_openai::types::responses::{+ CreateResponse, InputParam, Reasoning, ReasoningSummary, Tool as ResponseTool, ToolChoiceOptions, ToolChoiceParam,+};+use shoji::types::{+ basic::{SamplingMethod, SamplingPolicy},+ encoding::{Message, MessageList},+ session::chat::StreamConfig,+};++use crate::openai::{+ Error,+ bridging::{reasoning_effort, responses},+};++pub fn build(+ model: &str,+ config: &StreamConfig,+ messages: Vec<Message>,+) -> Result<CreateResponse, Error> {+ let input_items = messages+ .iter()+ .map(responses::message::build)+ .collect::<Result<Vec<_>, _>>()?+ .into_iter()+ .flatten()+ .collect::<Vec<_>>();++ let tools: Vec<ResponseTool> = messages+ .tool_namespaces()+ .into_iter()+ .flat_map(|namespace| namespace.tools.into_iter())+ .map(responses::tool::build)+ .collect::<Result<Vec<_>, _>>()?;++ let reasoning = messages.reasoning_effort().map(|effort| Reasoning {+ effort: Some(reasoning_effort::build(effort)),+ summary: Some(ReasoningSummary::Auto),+ });++ let (temperature, top_p) = match &config.sampling_policy {+ SamplingPolicy::Default {} => (None, None),+ SamplingPolicy::Custom {+ method,+ } => match method {+ SamplingMethod::Greedy {} => (None, None),+ SamplingMethod::Stochastic {+ temperature,+ top_k: _,+ top_p,+ min_p: _,+ } => (temperature.map(|value| value as f32), top_p.map(|value| value as f32)),+ },+ };++ let mut request = CreateResponse {+ input: InputParam::Items(input_items),+ model: Some(model.to_string()),+ stream: Some(true),+ max_output_tokens: config.token_limit.map(|value| value as u32),+ temperature,+ top_p,+ reasoning,+ ..Default::default()+ };+ if !tools.is_empty() {+ request.tools = Some(tools);+ request.tool_choice = Some(ToolChoiceParam::Mode(ToolChoiceOptions::Auto));+ }++ Ok(request)+}diff --git a/crates/backend-remote/src/openai/bridging/responses/stream_chunk.rs b/crates/backend-remote/src/openai/bridging/responses/stream_chunk.rsnew file mode 100644index 000000000..d7357fbd7--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/responses/stream_chunk.rs@@ -0,0 +1,63 @@+use async_openai::types::responses::{OutputItem, ResponseStreamEvent};++use crate::openai::{+ bridging::responses::finish_reason,+ stream_state::{StreamChunk, ToolCallChunk},+};++pub fn build(event: ResponseStreamEvent) -> Option<StreamChunk> {+ match event {+ ResponseStreamEvent::ResponseOutputTextDelta(event) => Some(StreamChunk {+ content: Some(event.delta),+ ..StreamChunk::default()+ }),+ ResponseStreamEvent::ResponseReasoningTextDelta(event) => Some(StreamChunk {+ reasoning: Some(event.delta),+ ..StreamChunk::default()+ }),+ ResponseStreamEvent::ResponseReasoningSummaryTextDelta(event) => Some(StreamChunk {+ reasoning: Some(event.delta),+ ..StreamChunk::default()+ }),+ ResponseStreamEvent::ResponseOutputItemAdded(event) => match event.item {+ OutputItem::FunctionCall(call) => Some(StreamChunk {+ tool_calls: vec![ToolCallChunk {+ index: event.output_index,+ id: Some(call.call_id),+ name: Some(call.name),+ arguments: None,+ }],+ ..StreamChunk::default()+ }),+ _ => None,+ },+ ResponseStreamEvent::ResponseFunctionCallArgumentsDelta(event) => Some(StreamChunk {+ tool_calls: vec![ToolCallChunk {+ index: event.output_index,+ id: None,+ name: None,+ arguments: Some(event.delta),+ }],+ ..StreamChunk::default()+ }),+ ResponseStreamEvent::ResponseCompleted(event) => Some(StreamChunk {+ finish_reason: finish_reason::build(&event.response.status),+ tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),+ tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),+ ..StreamChunk::default()+ }),+ ResponseStreamEvent::ResponseFailed(event) => Some(StreamChunk {+ finish_reason: finish_reason::build(&event.response.status),+ tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),+ tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),+ ..StreamChunk::default()+ }),+ ResponseStreamEvent::ResponseIncomplete(event) => Some(StreamChunk {+ finish_reason: finish_reason::build(&event.response.status),+ tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),+ tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),+ ..StreamChunk::default()+ }),+ _ => None,+ }+}diff --git a/crates/backend-remote/src/openai/bridging/responses/tool.rs b/crates/backend-remote/src/openai/bridging/responses/tool.rsnew file mode 100644index 000000000..a068cd9e6--- /dev/null+++ b/crates/backend-remote/src/openai/bridging/responses/tool.rs@@ -0,0 +1,25 @@+use async_openai::types::responses::{FunctionTool, Tool};+use shoji::types::encoding::ToolDescription;++use crate::openai::Error;++pub fn build(description: ToolDescription) -> Result<Tool, Error> {+ let ToolDescription::Function {+ function,+ } = description;+ let parameters = match function.parameters {+ Some(value) => {+ Some(serde_json::from_str::<serde_json::Value>(&value.json).map_err(|error| Error::Serialization {+ message: error.to_string(),+ })?)+ },+ None => None,+ };+ Ok(Tool::Function(FunctionTool {+ name: function.name,+ parameters,+ strict: None,+ description: Some(function.description),+ defer_loading: None,+ }))+}diff --git a/crates/backend-remote/src/openai/config.rs b/crates/backend-remote/src/openai/config.rsnew file mode 100644index 000000000..69d856b4b--- /dev/null+++ b/crates/backend-remote/src/openai/config.rs@@ -0,0 +1,20 @@+use std::collections::HashMap;++use serde::{Deserialize, Serialize};++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum ApiType {+ Completions,+ Responses,+}++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct Config {+ pub identifier: String,+ pub api_endpoint: String,+ pub api_key: Option<String>,+ pub api_type: ApiType,+ pub headers: Option<HashMap<String, String>>,+}diff --git a/crates/backend-remote/src/openai/error.rs b/crates/backend-remote/src/openai/error.rsnew file mode 100644index 000000000..16c62ba98--- /dev/null+++ b/crates/backend-remote/src/openai/error.rs@@ -0,0 +1,19 @@+#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error("Unable to create: {message}")]+ UnableToCreate {+ message: String,+ },+ #[error("Network error: {message}")]+ Network {+ message: String,+ },+ #[error("Serialization error: {message}")]+ Serialization {+ message: String,+ },+ #[error("Unsupported role")]+ UnsupportedRole,+ #[error("Tool call result is required")]+ ToolCallResultRequired,+}diff --git a/crates/backend-remote/src/openai/instance.rs b/crates/backend-remote/src/openai/instance.rsnew file mode 100644index 000000000..12be33fe7--- /dev/null+++ b/crates/backend-remote/src/openai/instance.rs@@ -0,0 +1,73 @@+use std::{pin::Pin, sync::Arc};++use async_openai::{Client, config::OpenAIConfig};+use futures::Stream;+use shoji::{+ traits::{+ Instance as InstanceTrait, State as StateTrait,+ backend::{+ Error as BackendError,+ chat_message::{StreamInput, StreamOutput},+ },+ },+ types::session::chat::StreamConfig,+};+use tokio_util::sync::CancellationToken;++use crate::openai::{+ ApiType,+ bridging::{self, ApiStream},+};++#[derive(Debug, Clone)]+pub struct State;++impl StateTrait for State {+ fn clone_boxed(&self) -> Box<dyn StateTrait> {+ Box::new(self.clone())+ }+}++pub struct Instance {+ client: Arc<Client<OpenAIConfig>>,+ model_identifier: String,+ api_stream: Box<dyn ApiStream>,+}++impl Instance {+ pub fn new(+ client: Arc<Client<OpenAIConfig>>,+ model_identifier: String,+ api_type: ApiType,+ ) -> Self {+ let api_stream: Box<dyn ApiStream> = match api_type {+ ApiType::Completions => Box::new(bridging::chat::api_stream::ApiStream),+ ApiType::Responses => Box::new(bridging::responses::api_stream::ApiStream),+ };+ Self {+ client,+ model_identifier,+ api_stream,+ }+ }+}++impl InstanceTrait for Instance {+ type StreamConfig = StreamConfig;+ type StreamInput = StreamInput;+ type StreamOutput = StreamOutput;++ fn state(&self) -> Pin<Box<dyn Future<Output = Result<Box<dyn StateTrait>, BackendError>> + Send + '_>> {+ Box::pin(async move { Ok(Box::new(State) as Box<dyn StateTrait>) })+ }++ fn stream<'a>(+ &'a self,+ input: &'a Self::StreamInput,+ _state: &'a mut dyn StateTrait,+ config: Self::StreamConfig,+ cancel: CancellationToken,+ ) -> Pin<Box<dyn Stream<Item = Result<Self::StreamOutput, BackendError>> + Send + 'a>> {+ self.api_stream.stream(self.client.clone(), self.model_identifier.clone(), config, input.clone(), cancel)+ }+}diff --git a/crates/backend-remote/src/openai/mod.rs b/crates/backend-remote/src/openai/mod.rsnew file mode 100644index 000000000..da932284a--- /dev/null+++ b/crates/backend-remote/src/openai/mod.rs@@ -0,0 +1,79 @@+mod bridging;+mod config;+mod error;+mod instance;+mod stream_state;+mod tool_call_state;++use std::{pin::Pin, sync::Arc};++use async_openai::{Client, config::OpenAIConfig};+pub use config::{ApiType, Config};+pub use error::Error;+pub use instance::Instance;+use shoji::{+ traits::{+ Backend as BackendTrait,+ backend::{Error as BackendError, chat_message},+ },+ types::session::chat::Config as ChatConfig,+};++pub struct Backend {+ config: Config,+ client: Arc<Client<OpenAIConfig>>,+}++impl Backend {+ pub fn new(config: Config) -> Result<Self, Error> {+ let mut openai_config = OpenAIConfig::new().with_api_base(&config.api_endpoint);+ if let Some(api_key) = config.api_key.as_ref() {+ openai_config = openai_config.with_api_key(api_key);+ }+ if let Some(headers) = config.headers.as_ref() {+ for (key, value) in headers {+ let name =+ reqwest::header::HeaderName::from_bytes(key.as_bytes()).map_err(|error| Error::UnableToCreate {+ message: error.to_string(),+ })?;+ openai_config =+ openai_config.with_header(name, value.as_str()).map_err(|error| Error::UnableToCreate {+ message: error.to_string(),+ })?;+ }+ }++ Ok(Self {+ config,+ client: Arc::new(Client::with_config(openai_config)),+ })+ }+}++impl BackendTrait for Backend {+ fn identifier(&self) -> String {+ self.config.identifier.clone()+ }++ fn version(&self) -> String {+ "default".to_string()+ }++ fn as_chat_via_message_capable(&self) -> Option<&dyn chat_message::Backend> {+ Some(self)+ }+}++impl chat_message::Backend for Backend {+ fn instance(+ &self,+ reference: String,+ _config: ChatConfig,+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn chat_message::Instance>, BackendError>> + Send + '_>> {+ let client = self.client.clone();+ let api_type = self.config.api_type.clone();+ Box::pin(+ async move { Ok(Box::new(Instance::new(client, reference, api_type)) as Box<dyn chat_message::Instance>) },+ )+ }+}diff --git a/crates/backend-remote/src/openai/stream_state.rs b/crates/backend-remote/src/openai/stream_state.rsnew file mode 100644index 000000000..b1cf8bc82--- /dev/null+++ b/crates/backend-remote/src/openai/stream_state.rs@@ -0,0 +1,143 @@+use std::time::Instant;++use indexmap::IndexMap;+use shoji::{+ traits::backend::chat_message::Output,+ types::session::chat::{FinishReason, Stats},+};++use crate::openai::tool_call_state::ToolCallState;++#[derive(Debug)]+pub struct ToolCallChunk {+ pub index: u32,+ pub id: Option<String>,+ pub name: Option<String>,+ pub arguments: Option<String>,+}++#[derive(Debug, Default)]+pub struct StreamChunk {+ pub content: Option<String>,+ pub reasoning: Option<String>,+ pub tool_calls: Vec<ToolCallChunk>,+ pub finish_reason: Option<FinishReason>,+ pub tokens_input: Option<u32>,+ pub tokens_output: Option<u32>,+}++#[derive(Debug)]+pub struct StreamState {+ text: Option<String>,+ reasoning: Option<String>,+ tool_calls: IndexMap<u32, ToolCallState>,+ finish_reason: Option<FinishReason>,+ start_moment: Instant,+ first_token_moment: Option<Instant>,+ tokens_input: Option<u32>,+ tokens_output: Option<u32>,+}++impl StreamState {+ pub fn new() -> Self {+ Self {+ text: None,+ reasoning: None,+ tool_calls: IndexMap::new(),+ finish_reason: None,+ start_moment: Instant::now(),+ first_token_moment: None,+ tokens_input: None,+ tokens_output: None,+ }+ }++ pub fn process_chunk(+ &mut self,+ chunk: StreamChunk,+ ) -> Option<Output> {+ let mut processed = false;+ let produced_output = chunk.content.is_some() || chunk.reasoning.is_some() || !chunk.tool_calls.is_empty();++ if let Some(content) = chunk.content {+ self.text.get_or_insert_with(String::new).push_str(&content);+ processed = true;+ }++ if let Some(reasoning) = chunk.reasoning {+ self.reasoning.get_or_insert_with(String::new).push_str(&reasoning);+ processed = true;+ }++ if !chunk.tool_calls.is_empty() {+ for call in chunk.tool_calls {+ self.tool_calls.entry(call.index).or_default().merge(call);+ }+ processed = true;+ }++ if produced_output && self.first_token_moment.is_none() {+ self.first_token_moment = Some(Instant::now());+ }++ if let Some(tokens) = chunk.tokens_input {+ self.tokens_input = Some(tokens);+ processed = true;+ }++ if let Some(tokens) = chunk.tokens_output {+ self.tokens_output = Some(tokens);+ processed = true;+ }++ if let Some(reason) = chunk.finish_reason {+ self.finish_reason = Some(reason);+ processed = true;+ }++ if processed {+ Some(self.build())+ } else {+ None+ }+ }++ fn build(&self) -> Output {+ let finalize = self.finish_reason.is_some();+ Output {+ reasoning: self.reasoning.clone(),+ text: self.text.clone(),+ tool_calls: self.tool_calls.values().map(|state| state.build(finalize)).collect(),+ finish_reason: self.finish_reason.clone(),+ stats: self.stats(),+ }+ }++ fn stats(&self) -> Stats {+ let duration = self.start_moment.elapsed().as_secs_f64();+ let time_to_first_token =+ self.first_token_moment.map(|first| first.duration_since(self.start_moment).as_secs_f64());+ let prefill_tokens_per_second = match (self.tokens_input, time_to_first_token) {+ (Some(tokens_input), Some(time_to_first_token)) if time_to_first_token > 0.0 => {+ Some(tokens_input as f64 / time_to_first_token)+ },+ _ => None,+ };+ let generate_tokens_per_second = match (self.tokens_output, time_to_first_token) {+ (Some(tokens_output), Some(time_to_first_token))+ if (tokens_output > 0) && (duration > time_to_first_token) =>+ {+ Some((tokens_output - 1) as f64 / (duration - time_to_first_token))+ },+ _ => None,+ };+ Stats {+ duration,+ time_to_first_token,+ prefill_tokens_per_second,+ generate_tokens_per_second,+ tokens_count_input: self.tokens_input,+ tokens_count_output: self.tokens_output,+ }+ }+}diff --git a/crates/backend-remote/src/openai/tool_call_state.rs b/crates/backend-remote/src/openai/tool_call_state.rsnew file mode 100644index 000000000..ad3f557b2--- /dev/null+++ b/crates/backend-remote/src/openai/tool_call_state.rs@@ -0,0 +1,48 @@+use shoji::{+ traits::backend::chat_message::ToolCallState as OutputToolCallState,+ types::{basic::Value, encoding::ToolCall},+};++use crate::openai::stream_state::ToolCallChunk;++#[derive(Debug, Default)]+pub struct ToolCallState {+ id: String,+ name: String,+ arguments: String,+}++impl ToolCallState {+ pub fn merge(+ &mut self,+ chunk: ToolCallChunk,+ ) {+ if let Some(id) = chunk.id {+ self.id.push_str(&id);+ }+ if let Some(name) = chunk.name {+ self.name.push_str(&name);+ }+ if let Some(arguments) = chunk.arguments {+ self.arguments.push_str(&arguments);+ }+ }++ pub fn build(+ &self,+ finalize: bool,+ ) -> OutputToolCallState {+ if finalize {+ if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(&self.arguments) {+ return OutputToolCallState::Finished(ToolCall {+ identifier: Some(self.id.clone()),+ name: self.name.clone(),+ arguments: Value {+ json: parsed.to_string(),+ },+ });+ }+ }+ OutputToolCallState::Candidate(self.arguments.clone())+ }+}diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.tomlindex 1874c0edc..9f61bf8f1 100644--- a/crates/backend-uzu/Cargo.toml+++ b/crates/backend-uzu/Cargo.toml@@ -32,6 +32,8 @@ sysinfo.workspace = trueitertools.workspace = trueschemars.workspace = truerustix.workspace = true+futures.workspace = true+tokio-util.workspace = true# metal on macos requires objc2, bytesize, metal[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]diff --git a/crates/backend-uzu/src/inference_backend.rs b/crates/backend-uzu/src/inference_backend.rsindex ff12d9692..6d1bb5482 100644--- a/crates/backend-uzu/src/inference_backend.rs+++ b/crates/backend-uzu/src/inference_backend.rs@@ -1,20 +1,46 @@-use serde::{Deserialize, Serialize};-use shoji::traits::BackendInstance as BackendInstanceTrait;+use std::{+ path::PathBuf,+ pin::Pin,+ sync::{Arc, Mutex},+};-use crate::TOOLCHAIN_VERSION;+use futures::{Stream, stream};+use shoji::{+ traits::{+ Backend as BackendTrait, Instance as InstanceTrait, State as StateTrait,+ backend::{+ Error as BackendError,+ chat_token::{self, StreamInput, StreamOutput},+ },+ },+ types::session::chat::{Config, StreamConfig},+};+use tokio_util::sync::CancellationToken;-#[derive(Debug, thiserror::Error)]-pub enum Error {}+use crate::{+ TOOLCHAIN_VERSION,+ session::{ChatSession, config::DecodingConfig},+};-#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]-pub struct Config;+#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error("Unable to load model")]+ UnableToLoad,+ #[error("Unable to create state")]+ UnableToCreateState,+ #[error("Stream failed")]+ StreamFailed,+}pub struct Backend;-impl BackendInstanceTrait for Backend {- type Error = Error;- type Config = Config;+impl Backend {+ pub fn new() -> Result<Self, Error> {+ Ok(Self)+ }+}+impl BackendTrait for Backend {fn identifier(&self) -> String {"uzu".to_string()}@@ -23,8 +49,67 @@ impl BackendInstanceTrait for Backend {TOOLCHAIN_VERSION.to_string()}- #[allow(unused_variables)]- fn new(config: Self::Config) -> Result<Self, Self::Error> {- Ok(Self)+ fn as_chat_via_token_capable(&self) -> Option<&dyn chat_token::Backend> {+ Some(self)+ }+}++impl chat_token::Backend for Backend {+ fn instance(+ &self,+ reference: String,+ _config: Config,+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn chat_token::Instance>, BackendError>> + Send + '_>> {+ Box::pin(async move {+ let path = PathBuf::from(&reference);+ let session = ChatSession::new(path, DecodingConfig::default())+ .map_err(|_| Box::new(Error::UnableToLoad) as BackendError)?;+ Ok(Box::new(Instance {+ session: Arc::new(Mutex::new(session)),+ }) as Box<dyn chat_token::Instance>)+ })+ }+}++pub struct Instance {+ session: Arc<Mutex<ChatSession>>,+}+unsafe impl Send for Instance {}+unsafe impl Sync for Instance {}++#[derive(Debug, Clone)]+pub struct State;++impl StateTrait for State {+ fn clone_boxed(&self) -> Box<dyn StateTrait> {+ Box::new(self.clone())+ }+}++impl InstanceTrait for Instance {+ type StreamConfig = StreamConfig;+ type StreamInput = StreamInput;+ type StreamOutput = StreamOutput;++ fn state(&self) -> Pin<Box<dyn Future<Output = Result<Box<dyn StateTrait>, BackendError>> + Send + '_>> {+ let reset = self+ .session+ .lock()+ .map_err(|_| Error::UnableToCreateState)+ .and_then(|mut guard| guard.reset().map_err(|_| Error::UnableToCreateState));+ Box::pin(async move {+ reset.map_err(|error| Box::new(error) as BackendError)?;+ Ok(Box::new(State) as Box<dyn StateTrait>)+ })+ }++ fn stream<'a>(+ &'a self,+ _input: &'a Self::StreamInput,+ _state: &'a mut dyn StateTrait,+ _config: Self::StreamConfig,+ _cancel_token: CancellationToken,+ ) -> Pin<Box<dyn Stream<Item = Result<Self::StreamOutput, BackendError>> + Send + 'a>> {+ Box::pin(stream::once(async { Err(Box::new(Error::StreamFailed) as BackendError) }))}}diff --git a/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs b/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rsindex f200c7271..c278e8b7d 100644--- a/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs+++ b/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs@@ -11,7 +11,7 @@ use backend_uzu::{types::{Input, Message, Output},},};-use shoji::types::ClassificationFeature;+use shoji::types::session::chat::Feature;use test_tag::tag;use tokenizers::Tokenizer;@@ -53,7 +53,7 @@ fn test_text_session_with_prompt_lookup_speculator() {fn test_text_session_with_fixed_speculator() {let tokenizer = Tokenizer::from_file(get_test_model_path().join("tokenizer.json")).unwrap();- let feature = ClassificationFeature {+ let feature = Feature {name: String::from("sentiment"),values: vec!["Happy", "Sad", "Angry", "Fearful", "Surprised", "Disgusted"].into_iter()diff --git a/crates/cli-storage/src/app.rs b/crates/cli-storage/src/app.rsindex 6d98832db..7ca987a93 100644--- a/crates/cli-storage/src/app.rs+++ b/crates/cli-storage/src/app.rs@@ -3,7 +3,7 @@ use std::{collections::HashMap, sync::Arc};use crossterm::event::KeyCode;use futures_util::StreamExt;use ratatui::widgets::ListState;-use shoji::types::Model;+use shoji::types::model::Model;use tokio::{sync::Mutex as TokioMutex, task::JoinHandle};use uzu::{engine::Engine, storage::types::DownloadState};@@ -31,7 +31,8 @@ impl App {engine: Arc<Engine>,tokio_handle: tokio::runtime::Handle,) -> Self {- let models = engine.models().await.unwrap().into_iter().filter(|model| model.is_local()).collect::<Vec<_>>();+ let models =+ engine.models().await.unwrap().into_iter().filter(|model| model.is_downloadable()).collect::<Vec<_>>();// Fetch initial state for all modelslet mut models_with_state = HashMap::new();diff --git a/crates/hanashi/src/chat/config.rs b/crates/hanashi/src/chat/config.rsindex a2fcbd402..a59c1deca 100644--- a/crates/hanashi/src/chat/config.rs+++ b/crates/hanashi/src/chat/config.rs@@ -1,5 +1,5 @@use serde::{Deserialize, Serialize};-use shoji::types::Capabilities;+use shoji::types::encoding::Capabilities;use crate::chat::{Error, hanashi::Config as HanashiConfig, harmony::Config as HarmonyConfig};diff --git a/crates/hanashi/src/chat/hanashi/config/mod.rs b/crates/hanashi/src/chat/hanashi/config/mod.rsindex 7b5ab509e..641270ce0 100644--- a/crates/hanashi/src/chat/hanashi/config/mod.rs+++ b/crates/hanashi/src/chat/hanashi/config/mod.rs@@ -2,7 +2,7 @@ mod bundled;mod tokens;use serde::{Deserialize, Serialize};-use shoji::types::{Capabilities, ContentBlockType, ReasoningEffort};+use shoji::types::encoding::{Capabilities, ContentBlockType, ReasoningEffort};use token_stream_parser::token_stream::TokenStreamParserConfig;pub use tokens::TokensConfig;diff --git a/crates/hanashi/src/chat/hanashi/config/tokens.rs b/crates/hanashi/src/chat/hanashi/config/tokens.rsindex 6ad3d76bf..bc7ad2075 100644--- a/crates/hanashi/src/chat/hanashi/config/tokens.rs+++ b/crates/hanashi/src/chat/hanashi/config/tokens.rs@@ -1,5 +1,5 @@use serde::{Deserialize, Serialize};-use shoji::types::TokenId;+use shoji::types::basic::TokenId;#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]pub struct TokensConfig {diff --git a/crates/hanashi/src/chat/hanashi/messages/canonical/content_block.rs b/crates/hanashi/src/chat/hanashi/messages/canonical/content_block.rsindex 8225db393..d42a21de7 100644--- a/crates/hanashi/src/chat/hanashi/messages/canonical/content_block.rs+++ b/crates/hanashi/src/chat/hanashi/messages/canonical/content_block.rs@@ -1,5 +1,5 @@use serde_json::Value;-use shoji::types::ContentBlockType;+use shoji::types::encoding::ContentBlockType;pub struct ContentBlock {pub r#type: ContentBlockType,diff --git a/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs b/crates/hanashi/src/chat/hanashi/messages/canonical/message.rsindex 5a6ba48df..ea8c7a213 100644--- a/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs+++ b/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs@@ -2,7 +2,7 @@ use std::collections::HashMap;use indexmap::IndexMap;use serde_json::Value;-use shoji::types::{+use shoji::types::encoding::{ContentBlock as OriginalContentBlock, ContentBlockType, Message as OriginalMessage, Role, TranslationInput,};@@ -127,6 +127,7 @@ impl Message {value,} => serde_json::to_value(value.clone()),OriginalContentBlock::ToolCallResult {+ identifier: _,name: _,value,} => serde_json::to_value(value.clone()),diff --git a/crates/hanashi/src/chat/hanashi/messages/error.rs b/crates/hanashi/src/chat/hanashi/messages/error.rsindex c18f69f40..e8adfe05f 100644--- a/crates/hanashi/src/chat/hanashi/messages/error.rs+++ b/crates/hanashi/src/chat/hanashi/messages/error.rs@@ -1,4 +1,4 @@-use shoji::types::{ContentBlockType, Role};+use shoji::types::encoding::{ContentBlockType, Role};#[derive(Debug, thiserror::Error)]#[non_exhaustive]diff --git a/crates/hanashi/src/chat/hanashi/messages/rendered/config.rs b/crates/hanashi/src/chat/hanashi/messages/rendered/config.rsindex 5e7b1b148..c68c00f10 100644--- a/crates/hanashi/src/chat/hanashi/messages/rendered/config.rs+++ b/crates/hanashi/src/chat/hanashi/messages/rendered/config.rs@@ -1,7 +1,7 @@use indexmap::IndexMap;use serde::{Deserialize, Serialize};use serde_json::Value;-use shoji::types::ContentBlockType;+use shoji::types::encoding::ContentBlockType;#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]pub struct Field {diff --git a/crates/hanashi/src/chat/hanashi/messages/rendered/message.rs b/crates/hanashi/src/chat/hanashi/messages/rendered/message.rsindex bb9312144..9ae81f1b8 100644--- a/crates/hanashi/src/chat/hanashi/messages/rendered/message.rs+++ b/crates/hanashi/src/chat/hanashi/messages/rendered/message.rs@@ -2,7 +2,7 @@ use std::collections::HashMap;use indexmap::IndexMap;use serde_json::{Map, Value};-use shoji::types::{ContentBlockType, Message as OriginalMessage, Role};+use shoji::types::encoding::{ContentBlockType, Message as OriginalMessage, Role};use crate::chat::hanashi::messages::{Error,diff --git a/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs b/crates/hanashi/src/chat/hanashi/messages/streamed/content.rsindex 153da9c78..13308a949 100644--- a/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs+++ b/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};use serde_json::Value;-use shoji::types::{ContentBlock, Role, ToolCall};+use shoji::types::encoding::{ContentBlock, Role, ToolCall};use crate::chat::hanashi::messages::streamed::Section;@@ -23,6 +23,7 @@ impl Content {}],Content::Value(value) => match role {Role::Tool {} => vec![ContentBlock::ToolCallResult {+ identifier: None,name: None,value: value.into(),}],diff --git a/crates/hanashi/src/chat/hanashi/messages/streamed/mod.rs b/crates/hanashi/src/chat/hanashi/messages/streamed/mod.rsindex 28ae48533..c3840a598 100644--- a/crates/hanashi/src/chat/hanashi/messages/streamed/mod.rs+++ b/crates/hanashi/src/chat/hanashi/messages/streamed/mod.rs@@ -6,7 +6,7 @@ use std::{collections::HashMap, str::FromStr};pub use content::Content;pub use section::Section;use serde::{Deserialize, Deserializer};-use shoji::types::{Message as CommonMessage, Role};+use shoji::types::encoding::{Message as CommonMessage, Role};#[derive(Deserialize)]pub struct Message {diff --git a/crates/hanashi/src/chat/hanashi/mod.rs b/crates/hanashi/src/chat/hanashi/mod.rsindex 488ab0eff..4af218f0b 100644--- a/crates/hanashi/src/chat/hanashi/mod.rs+++ b/crates/hanashi/src/chat/hanashi/mod.rs@@ -9,7 +9,10 @@ use std::path::PathBuf;pub use config::Config;pub use error::Error;-use shoji::types::{ContentBlock, Message, Token, TokenId};+use shoji::types::{+ basic::{Token, TokenId},+ encoding::{ContentBlock, Message},+};use token_stream_parser::{Parser as _, token_stream::TokenStreamParser};use tokenizers::{Tokenizer, step_decode_stream};diff --git a/crates/hanashi/src/chat/hanashi/ordering/config.rs b/crates/hanashi/src/chat/hanashi/ordering/config.rsindex 379658e5c..418cac9c0 100644--- a/crates/hanashi/src/chat/hanashi/ordering/config.rs+++ b/crates/hanashi/src/chat/hanashi/ordering/config.rs@@ -1,6 +1,6 @@use indexmap::IndexMap;use serde::{Deserialize, Serialize};-use shoji::types::Role;+use shoji::types::encoding::Role;#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]pub struct Config {diff --git a/crates/hanashi/src/chat/hanashi/ordering/error.rs b/crates/hanashi/src/chat/hanashi/ordering/error.rsindex 3cf8e2ffa..d79bf96fe 100644--- a/crates/hanashi/src/chat/hanashi/ordering/error.rs+++ b/crates/hanashi/src/chat/hanashi/ordering/error.rs@@ -1,4 +1,4 @@-use shoji::types::Role;+use shoji::types::encoding::Role;#[derive(Debug, thiserror::Error)]pub enum Error {diff --git a/crates/hanashi/src/chat/hanashi/ordering/validator.rs b/crates/hanashi/src/chat/hanashi/ordering/validator.rsindex 729012385..1058abe98 100644--- a/crates/hanashi/src/chat/hanashi/ordering/validator.rs+++ b/crates/hanashi/src/chat/hanashi/ordering/validator.rs@@ -1,4 +1,4 @@-use shoji::types::Role;+use shoji::types::encoding::Role;use crate::chat::hanashi::ordering::{Config, Error};diff --git a/crates/hanashi/src/chat/hanashi/renderer/config/mod.rs b/crates/hanashi/src/chat/hanashi/renderer/config/mod.rsindex 32396622d..dc44f9bfd 100644--- a/crates/hanashi/src/chat/hanashi/renderer/config/mod.rs+++ b/crates/hanashi/src/chat/hanashi/renderer/config/mod.rs@@ -5,7 +5,7 @@ use indexmap::IndexMap;pub use jinja::JinjaConfig;pub use jinja_function::JinjaFunction;use serde::{Deserialize, Serialize};-use shoji::types::{ContentBlockType, Role};+use shoji::types::encoding::{ContentBlockType, Role};use crate::chat::hanashi::messages::{canonical::Config as CanonicalConfig,diff --git a/crates/hanashi/src/chat/hanashi/renderer/mod.rs b/crates/hanashi/src/chat/hanashi/renderer/mod.rsindex 1830392fb..c7518396d 100644--- a/crates/hanashi/src/chat/hanashi/renderer/mod.rs+++ b/crates/hanashi/src/chat/hanashi/renderer/mod.rs@@ -8,7 +8,7 @@ use functions::{raise_exception, strftime_now, to_json};use indexmap::IndexMap;use minijinja::Environment;use minijinja_contrib::pycompat::unknown_method_callback;-use shoji::types::{Message, Token};+use shoji::types::{basic::Token, encoding::Message};use crate::chat::hanashi::messages::rendered::Message as RenderedMessage;diff --git a/crates/hanashi/src/chat/hanashi/token.rs b/crates/hanashi/src/chat/hanashi/token.rsindex 2eeb93014..4b98a905b 100644--- a/crates/hanashi/src/chat/hanashi/token.rs+++ b/crates/hanashi/src/chat/hanashi/token.rs@@ -1,4 +1,4 @@-use shoji::types::Token;+use shoji::types::basic::Token;use token_stream_parser::types::Token as ParserToken;pub trait ToParserToken {diff --git a/crates/hanashi/src/chat/harmony/bridging/message.rs b/crates/hanashi/src/chat/harmony/bridging/message.rsindex b25deafc9..3fb892529 100644--- a/crates/hanashi/src/chat/harmony/bridging/message.rs+++ b/crates/hanashi/src/chat/harmony/bridging/message.rs@@ -4,7 +4,10 @@ use openai_harmony::chat::{Author as ExternalAuthor, Content as ExternalContent, DeveloperContent as ExternalDeveloperContent,Message as ExternalMessage, Role as ExternalRole, SystemContent as ExternalSystemContent, TextContent,};-use shoji::types::{ContentBlock, Message, ReasoningEffort, Role, ToolCall, ToolNamespace, Value};+use shoji::types::{+ basic::Value,+ encoding::{ContentBlock, Message, ReasoningEffort, Role, ToolCall, ToolNamespace},+};use crate::chat::harmony::bridging::{Error, FromHarmony, ToHarmony};@@ -248,6 +251,7 @@ pub fn bridge_messages_to_harmony(messages: &[Message]) -> Result<Vec<ExternalMelet block = &message.content[0];let ContentBlock::ToolCallResult {+ identifier: _,name,value,} = block@@ -429,6 +433,7 @@ pub fn bridge_messages_from_harmony(messages: &[ExternalMessage]) -> Result<Vec<Ok(json_value) => {content.push(ContentBlock::ToolCall {value: ToolCall {+ identifier: None,name: name.to_string(),arguments: Value::from(json_value),},@@ -471,6 +476,7 @@ pub fn bridge_messages_from_harmony(messages: &[ExternalMessage]) -> Result<Vec<result.push(Message {role: Role::Tool {},content: vec![ContentBlock::ToolCallResult {+ identifier: None,name: Some(name.to_string()),value,}],diff --git a/crates/hanashi/src/chat/harmony/bridging/mod.rs b/crates/hanashi/src/chat/harmony/bridging/mod.rsindex 8b22da73f..c264fbb4d 100644--- a/crates/hanashi/src/chat/harmony/bridging/mod.rs+++ b/crates/hanashi/src/chat/harmony/bridging/mod.rs@@ -5,7 +5,7 @@ mod role;mod tool_namespace;pub use message::{bridge_messages_from_harmony, bridge_messages_to_harmony};-use shoji::types::{ContentBlockType, ReasoningEffort, Role};+use shoji::types::encoding::{ContentBlockType, ReasoningEffort, Role};#[derive(Debug, thiserror::Error)]#[non_exhaustive]diff --git a/crates/hanashi/src/chat/harmony/bridging/reasoning_effort.rs b/crates/hanashi/src/chat/harmony/bridging/reasoning_effort.rsindex 930f7403f..ef8187978 100644--- a/crates/hanashi/src/chat/harmony/bridging/reasoning_effort.rs+++ b/crates/hanashi/src/chat/harmony/bridging/reasoning_effort.rs@@ -1,5 +1,5 @@use openai_harmony::chat::ReasoningEffort as ExternalReasoningEffort;-use shoji::types::ReasoningEffort;+use shoji::types::encoding::ReasoningEffort;use crate::chat::harmony::bridging::{Error, FromHarmony, ToHarmony};diff --git a/crates/hanashi/src/chat/harmony/bridging/role.rs b/crates/hanashi/src/chat/harmony/bridging/role.rsindex 58485e84b..64b4fa908 100644--- a/crates/hanashi/src/chat/harmony/bridging/role.rs+++ b/crates/hanashi/src/chat/harmony/bridging/role.rs@@ -1,5 +1,5 @@use openai_harmony::chat::Role as ExternalRole;-use shoji::types::Role;+use shoji::types::encoding::Role;use crate::chat::harmony::bridging::{Error, FromHarmony, ToHarmony};diff --git a/crates/hanashi/src/chat/harmony/bridging/tool_namespace.rs b/crates/hanashi/src/chat/harmony/bridging/tool_namespace.rsindex b983ddefa..992fbdd70 100644--- a/crates/hanashi/src/chat/harmony/bridging/tool_namespace.rs+++ b/crates/hanashi/src/chat/harmony/bridging/tool_namespace.rs@@ -1,7 +1,10 @@use openai_harmony::chat::{ToolDescription as ExternalToolDescription, ToolNamespaceConfig as ExternalToolNamespaceConfig,};-use shoji::types::{ToolDescription, ToolFunction, ToolNamespace, Value};+use shoji::types::{+ basic::Value,+ encoding::{ToolDescription, ToolFunction, ToolNamespace},+};use crate::chat::harmony::bridging::{Error, FromHarmony, ToHarmony};diff --git a/crates/hanashi/src/chat/harmony/config.rs b/crates/hanashi/src/chat/harmony/config.rsindex 314dba09d..3af85990f 100644--- a/crates/hanashi/src/chat/harmony/config.rs+++ b/crates/hanashi/src/chat/harmony/config.rs@@ -1,5 +1,5 @@use serde::{Deserialize, Serialize};-use shoji::types::Capabilities;+use shoji::types::encoding::Capabilities;use crate::chat::harmony::EncodingName;diff --git a/crates/hanashi/src/chat/harmony/mod.rs b/crates/hanashi/src/chat/harmony/mod.rsindex 1661a4080..760b56eeb 100644--- a/crates/hanashi/src/chat/harmony/mod.rs+++ b/crates/hanashi/src/chat/harmony/mod.rs@@ -15,7 +15,10 @@ use openai_harmony::{},load_harmony_encoding,};-use shoji::types::{Message, Token, TokenId};+use shoji::types::{+ basic::{Token, TokenId},+ encoding::Message,+};use crate::{Encoding as EncodingTrait,diff --git a/crates/hanashi/src/chat/mod.rs b/crates/hanashi/src/chat/mod.rsindex 39dad86df..9bbb9e398 100644--- a/crates/hanashi/src/chat/mod.rs+++ b/crates/hanashi/src/chat/mod.rs@@ -8,7 +8,7 @@ mod state;pub use config::Config;pub use context::{Context, TokenizerLocation};pub use error::Error;-use shoji::types::{Message, TokenId};+use shoji::types::{basic::TokenId, encoding::Message};pub use state::{State, SynchronizationError, SynchronizationResult};use crate::{diff --git a/crates/hanashi/src/chat/state.rs b/crates/hanashi/src/chat/state.rsindex 8d3e1f7e9..837c88c00 100644--- a/crates/hanashi/src/chat/state.rs+++ b/crates/hanashi/src/chat/state.rs@@ -1,5 +1,5 @@use serde::{Deserialize, Serialize};-use shoji::types::{Message, Token};+use shoji::types::{basic::Token, encoding::Message};#[derive(Debug, thiserror::Error)]#[non_exhaustive]diff --git a/crates/hanashi/tests/helpers/mod.rs b/crates/hanashi/tests/helpers/mod.rsindex f374d7870..4c90fca0e 100644--- a/crates/hanashi/tests/helpers/mod.rs+++ b/crates/hanashi/tests/helpers/mod.rs@@ -3,7 +3,9 @@use std::{collections::HashMap, path::PathBuf};use serde::Deserialize;-use shoji::types::{ContentBlock, Message, ReasoningEffort, Role, ToolDescription, ToolNamespace, TranslationInput};+use shoji::types::encoding::{+ ContentBlock, Message, ReasoningEffort, Role, ToolDescription, ToolNamespace, TranslationInput,+};use tokenizers::Tokenizer;fn test_data_path() -> PathBuf {diff --git a/crates/hanashi/tests/test_encoding.rs b/crates/hanashi/tests/test_encoding.rsindex f32d04702..dd63fb25d 100644--- a/crates/hanashi/tests/test_encoding.rs+++ b/crates/hanashi/tests/test_encoding.rs@@ -14,7 +14,7 @@ use helpers::{build_messages, load_registry, load_response_test_data, load_tokenizer, normalize_pattern, response_path,tokenizer_directory,};-use shoji::types::{ContentBlock, Message, ReasoningEffort, Role};+use shoji::types::encoding::{ContentBlock, Message, ReasoningEffort, Role};fn print_warning(message: &str) {println!("\x1b[33m{}\x1b[0m", message);diff --git a/crates/nagare/Cargo.toml b/crates/nagare/Cargo.tomlindex 3b8e049a1..42ed30bb2 100644--- a/crates/nagare/Cargo.toml+++ b/crates/nagare/Cargo.toml@@ -15,3 +15,7 @@ serde_json.workspace = truethiserror.workspace = trueindexmap.workspace = truereqwest.workspace = true+futures.workspace = true+tokio-util.workspace = true+tokio.workspace = true+tokio-stream.workspace = truediff --git a/crates/nagare/src/chat/error.rs b/crates/nagare/src/chat/error.rsnew file mode 100644index 000000000..9894084b4--- /dev/null+++ b/crates/nagare/src/chat/error.rs@@ -0,0 +1,12 @@+#[derive(Debug, thiserror::Error)]+#[non_exhaustive]+pub enum Error {+ #[error("Backend error: {message}")]+ Backend {+ message: String,+ },+ #[error("Unsupported model")]+ UnsupportedModel,+ #[error("Unable to perform operation in current state")]+ UnableToPerformOperationInCurrentState,+}diff --git a/crates/nagare/src/chat/message.rs b/crates/nagare/src/chat/message.rsnew file mode 100644index 000000000..42fb2bdc1--- /dev/null+++ b/crates/nagare/src/chat/message.rs@@ -0,0 +1,63 @@+use std::pin::Pin;++use futures::{Stream, StreamExt};+use shoji::{+ traits::{+ State,+ backend::chat_message::{Backend, Instance, Output},+ },+ types::{+ encoding::Message,+ session::chat::{Config, StreamConfig},+ },+};+use tokio_util::sync::CancellationToken;++use super::Error;++pub struct Session {+ instance: Box<dyn Instance>,+ state: Box<dyn State>,+}++impl Session {+ pub async fn new(+ backend: &dyn Backend,+ config: Config,+ reference: String,+ ) -> Result<Self, Error> {+ let instance = backend.instance(reference, config).await.map_err(|error| Error::Backend {+ message: error.to_string(),+ })?;+ let state = instance.state().await.map_err(|error| Error::Backend {+ message: error.to_string(),+ })?;+ Ok(Self {+ instance,+ state,+ })+ }++ pub async fn reset(&mut self) -> Result<(), Error> {+ self.state = self.instance.state().await.map_err(|error| Error::Backend {+ message: error.to_string(),+ })?;+ Ok(())+ }++ pub fn stream<'a>(+ &'a mut self,+ input: &'a Vec<Message>,+ config: StreamConfig,+ cancel_token: CancellationToken,+ ) -> Pin<Box<dyn Stream<Item = Result<Output, Error>> + Send + 'a>> {+ self.instance+ .stream(input, self.state.as_mut(), config, cancel_token)+ .map(|event| {+ event.map_err(|error| Error::Backend {+ message: error.to_string(),+ })+ })+ .boxed()+ }+}diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rsnew file mode 100644index 000000000..6d7d3accd--- /dev/null+++ b/crates/nagare/src/chat/mod.rs@@ -0,0 +1,214 @@+mod error;+pub mod message;+pub mod token;++use std::{pin::Pin, sync::Arc};++pub use error::Error;+use futures::{Stream, StreamExt};+use indexmap::IndexMap;+use shoji::{+ traits::{+ Backend,+ backend::chat_message::{Output as BackendOutput, ToolCallState},+ },+ types::{+ basic::{CancellationToken, Value},+ encoding::{Message, ToolCall},+ model::{Model, Specialization},+ session::chat::{Config, Output, StreamConfig},+ },+};+use tokio::sync::{Mutex, mpsc};+use tokio_stream::wrappers::UnboundedReceiverStream;++#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]+pub enum State {+ Idle,+ Generation,+ ToolCalling,+ Resetting,+}++enum Instance {+ Token(token::Session),+ Message(message::Session),+}++pub struct Session {+ instance: Arc<Mutex<Instance>>,+ state: Arc<Mutex<State>>,+ messages: Arc<Mutex<Vec<Message>>>,+}++impl Session {+ pub async fn new(+ backend: &dyn Backend,+ config: Config,+ model: Model,+ path: Option<String>,+ ) -> Result<Self, Error> {+ if !model.specializations.contains(&Specialization::Chat) {+ return Err(Error::UnsupportedModel);+ }+ let reference = path.unwrap_or_else(|| model.identifier.clone());++ let instance = if let Some(token_backend) = backend.as_chat_via_token_capable() {+ Instance::Token(token::Session::new(token_backend, config, reference).await?)+ } else if let Some(message_backend) = backend.as_chat_via_message_capable() {+ Instance::Message(message::Session::new(message_backend, config, reference).await?)+ } else {+ return Err(Error::UnsupportedModel);+ };++ Ok(Self {+ instance: Arc::new(Mutex::new(instance)),+ state: Arc::new(Mutex::new(State::Idle)),+ messages: Arc::new(Mutex::new(Vec::new())),+ })+ }++ pub async fn state(&self) -> State {+ *self.state.lock().await+ }++ pub async fn messages(&self) -> Vec<Message> {+ self.messages.lock().await.clone()+ }++ pub async fn reset(&self) -> Result<(), Error> {+ {+ let mut state = self.state.lock().await;+ match *state {+ State::Idle | State::ToolCalling => {+ *state = State::Resetting;+ },+ State::Generation | State::Resetting => {+ return Err(Error::UnableToPerformOperationInCurrentState);+ },+ }+ }++ let result = {+ let mut guard = self.instance.lock().await;+ match &mut *guard {+ Instance::Token(session) => session.reset().await,+ Instance::Message(session) => session.reset().await,+ }+ };++ self.messages.lock().await.clear();+ *self.state.lock().await = State::Idle;++ result+ }++ pub fn stream(+ &self,+ input: Vec<Message>,+ config: StreamConfig,+ ) -> (Pin<Box<dyn Stream<Item = Result<Vec<Output>, Error>> + Send>>, CancellationToken) {+ let cancel_token_to_return = CancellationToken::new();+ let (sender, receiver) = mpsc::unbounded_channel::<Result<Vec<Output>, Error>>();++ let instance = self.instance.clone();+ let state = self.state.clone();+ let messages = self.messages.clone();+ let cancel_token = cancel_token_to_return.inner().clone();++ tokio::spawn(async move {+ {+ let mut state = state.lock().await;+ match *state {+ State::Idle => {+ *state = State::Generation;+ },+ State::Generation | State::Resetting | State::ToolCalling => {+ let _ = sender.send(Err(Error::UnableToPerformOperationInCurrentState));+ return;+ },+ }+ drop(state);+ }++ let all_messages = {+ let mut messages = messages.lock().await;+ messages.extend(input);+ messages.clone()+ };++ let mut outputs: IndexMap<u32, Output> = IndexMap::new();++ let mut instance = instance.lock().await;+ let mut stream = match &mut *instance {+ Instance::Token(session) => session.stream(&all_messages, config, cancel_token),+ Instance::Message(session) => session.stream(&all_messages, config, cancel_token),+ };++ let turn_index: u32 = 0;+ while let Some(partial_output) = stream.next().await {+ match partial_output {+ Ok(backend_output) => {+ let message = build_message(&backend_output);+ let output = Output {+ message: message.clone(),+ stats: backend_output.stats.clone(),+ finish_reason: backend_output.finish_reason.clone(),+ };+ let is_new = outputs.insert(turn_index, output).is_none();++ {+ let mut messages = messages.lock().await;+ if is_new {+ messages.push(message.clone());+ } else if let Some(last) = messages.last_mut() {+ *last = message.clone();+ }+ }++ if sender.send(Ok(outputs.values().cloned().collect())).is_err() {+ break;+ }+ },+ Err(error) => {+ let _ = sender.send(Err(error));+ break;+ },+ }+ }++ drop(stream);+ drop(instance);++ {+ let mut state = state.lock().await;+ *state = State::Idle;+ }+ });++ (Box::pin(UnboundedReceiverStream::new(receiver)), cancel_token_to_return)+ }+}++fn build_message(output: &BackendOutput) -> Message {+ let mut message = Message::assistant();+ if let Some(reasoning) = &output.reasoning {+ message = message.with_reasoning(reasoning.clone());+ }+ if let Some(text) = &output.text {+ message = message.with_text(text.clone());+ }+ for tool_call in &output.tool_calls {+ message = match tool_call {+ ToolCallState::Candidate(candidate) => {+ message.with_tool_call_candidate(Value::from(serde_json::Value::String(candidate.clone())))+ },+ ToolCallState::Finished(tool_call) => message.with_tool_call(ToolCall {+ identifier: tool_call.identifier.clone(),+ name: tool_call.name.clone(),+ arguments: tool_call.arguments.clone(),+ }),+ };+ }+ message+}diff --git a/crates/nagare/src/chat/token.rs b/crates/nagare/src/chat/token.rsnew file mode 100644index 000000000..a5e26be15--- /dev/null+++ b/crates/nagare/src/chat/token.rs@@ -0,0 +1,62 @@+use std::pin::Pin;++use futures::{Stream, stream};+use shoji::{+ traits::{+ State,+ backend::{+ chat_message::Output,+ chat_token::{Backend, Instance},+ },+ },+ types::{+ encoding::Message,+ session::chat::{Config, StreamConfig},+ },+};+use tokio_util::sync::CancellationToken;++use super::Error;++pub struct Session {+ _instance: Box<dyn Instance>,+ _state: Box<dyn State>,+}++impl Session {+ pub async fn new(+ backend: &dyn Backend,+ config: Config,+ reference: String,+ ) -> Result<Self, Error> {+ let instance = backend.instance(reference, config).await.map_err(|error| Error::Backend {+ message: error.to_string(),+ })?;+ let state = instance.state().await.map_err(|error| Error::Backend {+ message: error.to_string(),+ })?;+ Ok(Self {+ _instance: instance,+ _state: state,+ })+ }++ pub async fn reset(&mut self) -> Result<(), Error> {+ Err(Error::Backend {+ message: "token chat session reset not implemented".to_string(),+ })+ }++ pub fn stream<'a>(+ &'a mut self,+ _input: &'a Vec<Message>,+ _config: StreamConfig,+ _cancel_token: CancellationToken,+ ) -> Pin<Box<dyn Stream<Item = Result<Output, Error>> + Send + 'a>> {+ Box::pin(stream::once(async {+ Err(Error::Backend {+ message: "Token chat session stream not implemented".to_string(),+ })+ }))+ }+}diff --git a/crates/nagare/src/lib.rs b/crates/nagare/src/lib.rsindex e5fdf85ee..38dfdaeba 100644--- a/crates/nagare/src/lib.rs+++ b/crates/nagare/src/lib.rs@@ -1 +1,2 @@pub mod api;+pub mod chat;diff --git a/crates/shoji/Cargo.toml b/crates/shoji/Cargo.tomlindex 7be0cd8e6..85b9b5a8f 100644--- a/crates/shoji/Cargo.toml+++ b/crates/shoji/Cargo.toml@@ -34,6 +34,8 @@ serde.workspace = trueserde_json.workspace = truethiserror.workspace = trueindexmap.workspace = true+futures.workspace = true+tokio-util.workspace = trueuniffi = { workspace = true, optional = true }napi = { workspace = true, optional = true }napi-derive = { workspace = true, optional = true }diff --git a/crates/shoji/src/traits/backend.rs b/crates/shoji/src/traits/backend.rsdeleted file mode 100644index c864f8a46..000000000--- a/crates/shoji/src/traits/backend.rs+++ /dev/null@@ -1,30 +0,0 @@-use std::error::Error;--use serde::{Serialize, de::DeserializeOwned};--pub trait BackendInstance: Send + Sync {- type Error: Error;- type Config: Serialize + DeserializeOwned;-- fn identifier(&self) -> String;- fn version(&self) -> String;-- fn new(config: Self::Config) -> Result<Self, Self::Error>- where- Self: Sized;-}--pub trait Backend: Send + Sync {- fn identifier(&self) -> String;- fn version(&self) -> String;-}--impl<B: BackendInstance> Backend for B {- fn identifier(&self) -> String {- self.identifier()- }-- fn version(&self) -> String {- self.version()- }-}diff --git a/crates/shoji/src/traits/backend/chat_message.rs b/crates/shoji/src/traits/backend/chat_message.rsnew file mode 100644index 000000000..a0e599d5e--- /dev/null+++ b/crates/shoji/src/traits/backend/chat_message.rs@@ -0,0 +1,48 @@+use std::pin::Pin;++use serde::{Deserialize, Serialize};++use crate::{+ traits::backend::{Error, Instance as InstanceTrait},+ types::{+ encoding::{Message, ToolCall},+ session::chat::{Config, FinishReason, Stats, StreamConfig},+ },+};++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(tag = "state", rename_all = "snake_case")]+pub enum ToolCallState {+ Candidate(String),+ Finished(ToolCall),+}++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]+pub struct Output {+ pub reasoning: Option<String>,+ pub text: Option<String>,+ pub tool_calls: Vec<ToolCallState>,+ pub finish_reason: Option<FinishReason>,+ pub stats: Stats,+}++pub type StreamInput = Vec<Message>;+pub type StreamOutput = Output;++pub trait Backend: Send + Sync {+ fn instance(+ &self,+ reference: String,+ config: Config,+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + '_>>;+}++pub trait Instance:+ InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>+{+}++impl<T> Instance for T where+ T: InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>+{+}diff --git a/crates/shoji/src/traits/backend/chat_token.rs b/crates/shoji/src/traits/backend/chat_token.rsnew file mode 100644index 000000000..8b09279d3--- /dev/null+++ b/crates/shoji/src/traits/backend/chat_token.rs@@ -0,0 +1,27 @@+use std::pin::Pin;++use crate::{+ traits::backend::{Error, Instance as InstanceTrait},+ types::session::chat::{Config, StreamConfig},+};++pub type StreamInput = Vec<u64>;+pub type StreamOutput = u64;++pub trait Backend: Send + Sync {+ fn instance(+ &self,+ reference: String,+ config: Config,+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + '_>>;+}++pub trait Instance:+ InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>+{+}++impl<T> Instance for T where+ T: InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>+{+}diff --git a/crates/shoji/src/traits/backend/classification.rs b/crates/shoji/src/traits/backend/classification.rsnew file mode 100644index 000000000..7762f3e15--- /dev/null+++ b/crates/shoji/src/traits/backend/classification.rs@@ -0,0 +1,29 @@+use std::pin::Pin;++use crate::{+ traits::backend::{Error, Instance as InstanceTrait},+ types::{encoding::Message, session::classification::Output},+};++pub type Config = ();+pub type StreamConfig = ();+pub type StreamInput = Vec<Message>;+pub type StreamOutput = Output;++pub trait Backend: Send + Sync {+ fn instance(+ &self,+ reference: String,+ config: Config,+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + '_>>;+}++pub trait Instance:+ InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>+{+}++impl<T> Instance for T where+ T: InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>+{+}diff --git a/crates/shoji/src/traits/backend/mod.rs b/crates/shoji/src/traits/backend/mod.rsnew file mode 100644index 000000000..db41a35a3--- /dev/null+++ b/crates/shoji/src/traits/backend/mod.rs@@ -0,0 +1,52 @@+pub mod chat_message;+pub mod chat_token;+pub mod classification;+pub mod text_to_speech;++use std::pin::Pin;++use futures::Stream;+use tokio_util::sync::CancellationToken;++pub type Error = Box<dyn std::error::Error + Send + Sync>;++pub trait Backend: Send + Sync {+ fn identifier(&self) -> String;+ fn version(&self) -> String;++ fn as_chat_via_token_capable(&self) -> Option<&dyn chat_token::Backend> {+ None+ }++ fn as_chat_via_message_capable(&self) -> Option<&dyn chat_message::Backend> {+ None+ }++ fn as_classification_capable(&self) -> Option<&dyn classification::Backend> {+ None+ }++ fn as_text_to_speech_capable(&self) -> Option<&dyn text_to_speech::Backend> {+ None+ }+}++pub trait Instance: Send + Sync {+ type StreamConfig;+ type StreamInput;+ type StreamOutput;++ fn state(&self) -> Pin<Box<dyn Future<Output = Result<Box<dyn State>, Error>> + Send + '_>>;++ fn stream<'a>(+ &'a self,+ input: &'a Self::StreamInput,+ state: &'a mut dyn State,+ config: Self::StreamConfig,+ cancel_token: CancellationToken,+ ) -> Pin<Box<dyn Stream<Item = Result<Self::StreamOutput, Error>> + Send + 'a>>;+}++pub trait State: Send + Sync + 'static {+ fn clone_boxed(&self) -> Box<dyn State>;+}diff --git a/crates/shoji/src/traits/backend/text_to_speech.rs b/crates/shoji/src/traits/backend/text_to_speech.rsnew file mode 100644index 000000000..d79213df4--- /dev/null+++ b/crates/shoji/src/traits/backend/text_to_speech.rs@@ -0,0 +1,29 @@+use std::pin::Pin;++use crate::{+ traits::backend::{Error, Instance as InstanceTrait},+ types::{encoding::Message, session::text_to_speech::PcmBatch},+};++pub type Config = ();+pub type StreamConfig = ();+pub type StreamInput = Vec<Message>;+pub type StreamOutput = PcmBatch;++pub trait Backend: Send + Sync {+ fn instance(+ &self,+ reference: String,+ config: Config,+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + '_>>;+}++pub trait Instance:+ InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>+{+}++impl<T> Instance for T where+ T: InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>+{+}diff --git a/crates/shoji/src/traits/mod.rs b/crates/shoji/src/traits/mod.rsindex bfc9cfba3..80dd54bd3 100644--- a/crates/shoji/src/traits/mod.rs+++ b/crates/shoji/src/traits/mod.rs@@ -1,5 +1,5 @@-mod backend;+pub mod backend;mod registry;-pub use backend::{Backend, BackendInstance};+pub use backend::{Backend, Instance, State};pub use registry::Registry;diff --git a/crates/shoji/src/traits/registry.rs b/crates/shoji/src/traits/registry.rsindex 9ba5a3192..57a3bbe44 100644--- a/crates/shoji/src/traits/registry.rs+++ b/crates/shoji/src/traits/registry.rs@@ -1,6 +1,6 @@use std::{error::Error, future::Future, pin::Pin};-use crate::types::Model;+use crate::types::model::Model;pub trait Registry: Send + Sync {type Error: Error;diff --git a/crates/shoji/src/types/basic/cancellation_token.rs b/crates/shoji/src/types/basic/cancellation_token.rsnew file mode 100644index 000000000..1f2b24688--- /dev/null+++ b/crates/shoji/src/types/basic/cancellation_token.rs@@ -0,0 +1,52 @@+use tokio_util::sync::CancellationToken as TokioCancellationToken;++#[derive(Debug, Clone, Default)]+pub struct CancellationToken {+ inner: TokioCancellationToken,+}++impl CancellationToken {+ pub fn new() -> Self {+ Self {+ inner: TokioCancellationToken::new(),+ }+ }++ pub fn cancel(&self) {+ self.inner.cancel();+ }++ pub fn is_cancelled(&self) -> bool {+ self.inner.is_cancelled()+ }++ pub fn child(&self) -> Self {+ Self {+ inner: self.inner.child_token(),+ }+ }+}++impl CancellationToken {+ pub fn inner(&self) -> &TokioCancellationToken {+ &self.inner+ }++ pub fn into_inner(self) -> TokioCancellationToken {+ self.inner+ }+}++impl From<TokioCancellationToken> for CancellationToken {+ fn from(inner: TokioCancellationToken) -> Self {+ Self {+ inner,+ }+ }+}++impl From<CancellationToken> for TokioCancellationToken {+ fn from(token: CancellationToken) -> Self {+ token.inner+ }+}diff --git a/crates/shoji/src/types/basic/mod.rs b/crates/shoji/src/types/basic/mod.rsindex 9bfa2d446..738faef4e 100644--- a/crates/shoji/src/types/basic/mod.rs+++ b/crates/shoji/src/types/basic/mod.rs@@ -1,5 +1,13 @@+mod cancellation_token;+mod sampling_method;+mod sampling_policy;+mod sampling_seed;mod token;mod value;+pub use cancellation_token::CancellationToken;+pub use sampling_method::SamplingMethod;+pub use sampling_policy::SamplingPolicy;+pub use sampling_seed::SamplingSeed;pub use token::{Token, TokenId, TokenValue};pub use value::Value;diff --git a/crates/shoji/src/types/basic/sampling_method.rs b/crates/shoji/src/types/basic/sampling_method.rsnew file mode 100644index 000000000..3ea9f933c--- /dev/null+++ b/crates/shoji/src/types/basic/sampling_method.rs@@ -0,0 +1,13 @@+use serde::{Deserialize, Serialize};++#[bindings::export(Enum, name = "SamplingMethod")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub enum SamplingMethod {+ Greedy {},+ Stochastic {+ temperature: Option<f64>,+ top_k: Option<i64>,+ top_p: Option<f64>,+ min_p: Option<f64>,+ },+}diff --git a/crates/shoji/src/types/basic/sampling_policy.rs b/crates/shoji/src/types/basic/sampling_policy.rsnew file mode 100644index 000000000..d82a28a1a--- /dev/null+++ b/crates/shoji/src/types/basic/sampling_policy.rs@@ -0,0 +1,18 @@+use serde::{Deserialize, Serialize};++use crate::types::basic::SamplingMethod;++#[bindings::export(Enum, name = "SamplingPolicy")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub enum SamplingPolicy {+ Default {},+ Custom {+ method: SamplingMethod,+ },+}++impl Default for SamplingPolicy {+ fn default() -> Self {+ SamplingPolicy::Default {}+ }+}diff --git a/crates/shoji/src/types/basic/sampling_seed.rs b/crates/shoji/src/types/basic/sampling_seed.rsnew file mode 100644index 000000000..de03c47c0--- /dev/null+++ b/crates/shoji/src/types/basic/sampling_seed.rs@@ -0,0 +1,16 @@+use serde::{Deserialize, Serialize};++#[bindings::export(Enum, name = "SamplingSeed")]+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+pub enum SamplingSeed {+ Default {},+ Custom {+ seed: i64,+ },+}++impl Default for SamplingSeed {+ fn default() -> Self {+ SamplingSeed::Default {}+ }+}diff --git a/crates/shoji/src/types/basic/token.rs b/crates/shoji/src/types/basic/token.rsindex 5311ebb8b..0ac0ea47b 100644--- a/crates/shoji/src/types/basic/token.rs+++ b/crates/shoji/src/types/basic/token.rs@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};pub type TokenId = u32;pub type TokenValue = String;-#[bindings::export(Struct)]+#[bindings::export(Struct, name = "Token")]#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]#[serde(rename_all = "snake_case")]pub struct Token {diff --git a/crates/shoji/src/types/basic/value.rs b/crates/shoji/src/types/basic/value.rsindex 2a42887cd..6bee593d2 100644--- a/crates/shoji/src/types/basic/value.rs+++ b/crates/shoji/src/types/basic/value.rs@@ -2,7 +2,7 @@ use std::fmt;use serde::{Deserialize, Deserializer, Serialize, Serializer};-#[bindings::export(Struct)]+#[bindings::export(Struct, name = "Value")]#[derive(Clone, PartialEq)]pub struct Value {pub json: String,diff --git a/crates/shoji/src/types/chat/message.rs b/crates/shoji/src/types/chat/message.rsdeleted file mode 100644index 743842b3b..000000000--- a/crates/shoji/src/types/chat/message.rs+++ /dev/null@@ -1,14 +0,0 @@-use std::collections::HashMap;--use serde::{Deserialize, Serialize};--use crate::types::{ContentBlock, Role, Value};--#[bindings::export(Struct)]-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]-#[serde(rename_all = "snake_case")]-pub struct Message {- pub role: Role,- pub content: Vec<ContentBlock>,- pub metadata: HashMap<String, Value>,-}diff --git a/crates/shoji/src/types/chat/capabilities.rs b/crates/shoji/src/types/encoding/capabilities.rssimilarity index 100%rename from crates/shoji/src/types/chat/capabilities.rsrename to crates/shoji/src/types/encoding/capabilities.rsdiff --git a/crates/shoji/src/types/chat/content_block.rs b/crates/shoji/src/types/encoding/content_block.rssimilarity index 97%rename from crates/shoji/src/types/chat/content_block.rsrename to crates/shoji/src/types/encoding/content_block.rsindex b70bebc66..2085f0849 100644--- a/crates/shoji/src/types/chat/content_block.rs+++ b/crates/shoji/src/types/encoding/content_block.rs@@ -5,7 +5,10 @@ use std::{use serde::{Deserialize, Deserializer, Serialize, Serializer, de};-use crate::types::{ReasoningEffort, ToolCall, ToolNamespace, TranslationInput, Value};+use crate::types::{+ basic::Value,+ encoding::{ReasoningEffort, ToolCall, ToolNamespace, TranslationInput},+};#[bindings::export(Enum)]#[derive(Debug, Clone, PartialEq, Eq, Hash)]@@ -147,6 +150,8 @@ pub enum ContentBlock {value: Value,},ToolCallResult {+ #[serde(rename = "id")]+ identifier: Option<String>,name: Option<String>,value: Value,},diff --git a/crates/shoji/src/types/encoding/message.rs b/crates/shoji/src/types/encoding/message.rsnew file mode 100644index 000000000..34c9d99a8--- /dev/null+++ b/crates/shoji/src/types/encoding/message.rs@@ -0,0 +1,177 @@+use std::collections::HashMap;++use serde::{Deserialize, Serialize};++use crate::types::{+ basic::Value,+ encoding::{ContentBlock, ReasoningEffort, Role, ToolCall, ToolNamespace},+};++#[bindings::export(Struct)]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct Message {+ pub role: Role,+ pub content: Vec<ContentBlock>,+ pub metadata: HashMap<String, Value>,+}++impl Message {+ pub fn for_role(role: Role) -> Self {+ Self {+ role,+ content: vec![],+ metadata: HashMap::new(),+ }+ }++ pub fn system() -> Self {+ Self::for_role(Role::System {})+ }++ pub fn developer() -> Self {+ Self::for_role(Role::Developer {})+ }++ pub fn user() -> Self {+ Self::for_role(Role::User {})+ }++ pub fn assistant() -> Self {+ Self::for_role(Role::Assistant {})+ }++ pub fn tool() -> Self {+ Self::for_role(Role::Tool {})+ }+}++impl Message {+ fn with_block(+ self,+ block: ContentBlock,+ ) -> Self {+ let mut content = self.content;+ content.push(block);+ Self {+ content,+ ..self+ }+ }++ pub fn with_reasoning_effort(+ self,+ reasoning_effort: ReasoningEffort,+ ) -> Self {+ self.with_block(ContentBlock::ReasoningEffort {+ value: reasoning_effort,+ })+ }++ pub fn with_tool_namespaces(+ self,+ tool_namespaces: Vec<ToolNamespace>,+ ) -> Self {+ self.with_block(ContentBlock::Tools {+ namespaces: tool_namespaces,+ })+ }++ pub fn with_text(+ self,+ text: String,+ ) -> Self {+ self.with_block(ContentBlock::Text {+ value: text,+ })+ }++ pub fn with_reasoning(+ self,+ reasoning: String,+ ) -> Self {+ self.with_block(ContentBlock::Reasoning {+ value: reasoning,+ })+ }++ pub fn with_tool_call(+ self,+ tool_call: ToolCall,+ ) -> Self {+ self.with_block(ContentBlock::ToolCall {+ value: tool_call,+ })+ }++ pub fn with_tool_call_candidate(+ self,+ value: Value,+ ) -> Self {+ self.with_block(ContentBlock::ToolCallCandidate {+ value,+ })+ }+}++macro_rules! blocks_by_type {+ ($self:expr, $variant:ident, $field:ident) => {+ $self.content.iter().filter_map(|block| match block {+ ContentBlock::$variant {+ $field,+ } => Some($field.clone()),+ _ => None,+ })+ };+}++impl Message {+ pub fn reasoning_effort(&self) -> Option<ReasoningEffort> {+ blocks_by_type!(self, ReasoningEffort, value).next()+ }++ pub fn text(&self) -> String {+ blocks_by_type!(self, Text, value).collect::<String>()+ }++ pub fn reasoning(&self) -> String {+ blocks_by_type!(self, Reasoning, value).collect::<String>()+ }++ pub fn tool_namespaces(&self) -> Vec<ToolNamespace> {+ blocks_by_type!(self, Tools, namespaces).flatten().collect()+ }++ pub fn tool_calls(&self) -> Vec<ToolCall> {+ blocks_by_type!(self, ToolCall, value).collect()+ }++ pub fn tool_call_results(&self) -> Vec<(Option<String>, Option<String>, Value)> {+ self.content+ .iter()+ .filter_map(|block| match block {+ ContentBlock::ToolCallResult {+ identifier,+ name,+ value,+ } => Some((identifier.clone(), name.clone(), value.clone())),+ _ => None,+ })+ .collect()+ }+}++pub trait MessageList {+ fn reasoning_effort(&self) -> Option<ReasoningEffort>;+ fn tool_namespaces(&self) -> Vec<ToolNamespace>;+}++impl MessageList for Vec<Message> {+ fn reasoning_effort(&self) -> Option<ReasoningEffort> {+ self.iter().find_map(|message| message.reasoning_effort())+ }++ fn tool_namespaces(&self) -> Vec<ToolNamespace> {+ self.iter().flat_map(|message| message.tool_namespaces()).collect()+ }+}diff --git a/crates/shoji/src/types/chat/mod.rs b/crates/shoji/src/types/encoding/mod.rssimilarity index 92%rename from crates/shoji/src/types/chat/mod.rsrename to crates/shoji/src/types/encoding/mod.rsindex 32b790ae2..3814d6d42 100644--- a/crates/shoji/src/types/chat/mod.rs+++ b/crates/shoji/src/types/encoding/mod.rs@@ -11,7 +11,7 @@ mod translation_input;pub use capabilities::Capabilities;pub use content_block::{ContentBlock, ContentBlockType};-pub use message::Message;+pub use message::{Message, MessageList};pub use reasoning_effort::ReasoningEffort;pub use role::Role;pub use tool_call::ToolCall;diff --git a/crates/shoji/src/types/chat/reasoning_effort.rs b/crates/shoji/src/types/encoding/reasoning_effort.rssimilarity index 100%rename from crates/shoji/src/types/chat/reasoning_effort.rsrename to crates/shoji/src/types/encoding/reasoning_effort.rsdiff --git a/crates/shoji/src/types/chat/role.rs b/crates/shoji/src/types/encoding/role.rssimilarity index 100%rename from crates/shoji/src/types/chat/role.rsrename to crates/shoji/src/types/encoding/role.rsdiff --git a/crates/shoji/src/types/chat/tool_call.rs b/crates/shoji/src/types/encoding/tool_call.rssimilarity index 70%rename from crates/shoji/src/types/chat/tool_call.rsrename to crates/shoji/src/types/encoding/tool_call.rsindex d0dd4519d..360355b27 100644--- a/crates/shoji/src/types/chat/tool_call.rs+++ b/crates/shoji/src/types/encoding/tool_call.rs@@ -1,11 +1,13 @@use serde::{Deserialize, Serialize};-use crate::types::Value;+use crate::types::basic::Value;#[bindings::export(Struct)]#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]#[serde(rename_all = "snake_case")]pub struct ToolCall {+ #[serde(rename = "id")]+ pub identifier: Option<String>,pub name: String,pub arguments: Value,}diff --git a/crates/shoji/src/types/chat/tool_description.rs b/crates/shoji/src/types/encoding/tool_description.rssimilarity index 85%rename from crates/shoji/src/types/chat/tool_description.rsrename to crates/shoji/src/types/encoding/tool_description.rsindex a2777cc20..71cb8c485 100644--- a/crates/shoji/src/types/chat/tool_description.rs+++ b/crates/shoji/src/types/encoding/tool_description.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::types::ToolFunction;+use crate::types::encoding::ToolFunction;#[bindings::export(Enum)]#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]diff --git a/crates/shoji/src/types/chat/tool_function.rs b/crates/shoji/src/types/encoding/tool_function.rssimilarity index 90%rename from crates/shoji/src/types/chat/tool_function.rsrename to crates/shoji/src/types/encoding/tool_function.rsindex 9f86aff28..bbddcfd20 100644--- a/crates/shoji/src/types/chat/tool_function.rs+++ b/crates/shoji/src/types/encoding/tool_function.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::types::Value;+use crate::types::basic::Value;#[bindings::export(Struct)]#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]diff --git a/crates/shoji/src/types/chat/tool_namespace.rs b/crates/shoji/src/types/encoding/tool_namespace.rssimilarity index 86%rename from crates/shoji/src/types/chat/tool_namespace.rsrename to crates/shoji/src/types/encoding/tool_namespace.rsindex 45868863f..1143571dc 100644--- a/crates/shoji/src/types/chat/tool_namespace.rs+++ b/crates/shoji/src/types/encoding/tool_namespace.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::types::ToolDescription;+use crate::types::encoding::ToolDescription;#[bindings::export(Struct)]#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]diff --git a/crates/shoji/src/types/chat/translation_input.rs b/crates/shoji/src/types/encoding/translation_input.rssimilarity index 100%rename from crates/shoji/src/types/chat/translation_input.rsrename to crates/shoji/src/types/encoding/translation_input.rsdiff --git a/crates/shoji/src/types/mod.rs b/crates/shoji/src/types/mod.rsindex 800f82e3c..34d1e0f50 100644--- a/crates/shoji/src/types/mod.rs+++ b/crates/shoji/src/types/mod.rs@@ -1,9 +1,4 @@-mod basic;-mod chat;-mod model;-mod session;--pub use basic::*;-pub use chat::*;-pub use model::*;-pub use session::*;+pub mod basic;+pub mod encoding;+pub mod model;+pub mod session;diff --git a/crates/shoji/src/types/model/accessibility.rs b/crates/shoji/src/types/model/accessibility.rsindex 4cd5372aa..b620faa0a 100644--- a/crates/shoji/src/types/model/accessibility.rs+++ b/crates/shoji/src/types/model/accessibility.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::types::{Reference, Repository};+use crate::types::model::{Reference, Repository};#[bindings::export(Enum)]#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]diff --git a/crates/shoji/src/types/model/entity.rs b/crates/shoji/src/types/model/entity.rsindex e4a9807c3..4a66341f3 100644--- a/crates/shoji/src/types/model/entity.rs+++ b/crates/shoji/src/types/model/entity.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::types::Image;+use crate::types::model::Image;#[bindings::export(Enum)]#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]diff --git a/crates/shoji/src/types/model/file.rs b/crates/shoji/src/types/model/file.rsindex 8ac7457d8..fd71a3c3a 100644--- a/crates/shoji/src/types/model/file.rs+++ b/crates/shoji/src/types/model/file.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::types::{Hash, HashMethod};+use crate::types::model::{Hash, HashMethod};#[bindings::export(Struct)]#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]diff --git a/crates/shoji/src/types/model/model.rs b/crates/shoji/src/types/model/model.rsindex 0e7ecd18a..a16d397f4 100644--- a/crates/shoji/src/types/model/model.rs+++ b/crates/shoji/src/types/model/model.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::types::{Accessibility, Entity, EntityType, Quantization, Reference, Specialization};+use crate::types::model::{Accessibility, Entity, EntityType, Quantization, Reference, Specialization};#[bindings::export(Struct)]#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]@@ -20,7 +20,7 @@ impl Model {}pub fn cache_identifier(&self) -> String {- self.identifier.replace(":", "-")+ self.identifier.replace(":", "-").replace("/", "-")}pub fn is_local(&self) -> bool {@@ -31,6 +31,15 @@ impl Model {matches!(self.accessibility, Accessibility::Remote { .. })}+ pub fn is_downloadable(&self) -> bool {+ matches!(+ self.accessibility,+ Accessibility::Local {+ reference: Reference::Mirai { .. } | Reference::HuggingFace { .. }+ }+ )+ }+pub fn repo_ids(&self) -> Vec<String> {match &self.accessibility {Accessibility::Local {@@ -54,6 +63,9 @@ impl Model {Reference::HuggingFace {repository,} => vec![repository.identifier.clone()],+ Reference::Local {+ ..+ } => vec![],},Accessibility::Remote {repository,@@ -68,12 +80,34 @@ impl Model {}}- pub fn reference_type(&self) -> Option<String> {+ pub fn local_external_path(&self) -> Option<String> {match &self.accessibility {Accessibility::Local {reference,..- } => Some(reference.r#type()),+ } => match reference {+ Reference::Mirai {+ ..+ } => None,+ Reference::HuggingFace {+ ..+ } => None,+ Reference::Local {+ path,+ } => Some(path.clone()),+ },+ Accessibility::Remote {+ ..+ } => None,+ }+ }++ pub fn reference_name(&self) -> Option<String> {+ match &self.accessibility {+ Accessibility::Local {+ reference,+ ..+ } => Some(reference.name()),Accessibility::Remote {..} => None,@@ -93,6 +127,9 @@ impl Model {Reference::HuggingFace {repository,} => Some(repository.commit_hash.clone()),+ Reference::Local {+ ..+ } => None,},Accessibility::Remote {..diff --git a/crates/shoji/src/types/model/reference.rs b/crates/shoji/src/types/model/reference.rsindex eb3f99d5d..25daa58e9 100644--- a/crates/shoji/src/types/model/reference.rs+++ b/crates/shoji/src/types/model/reference.rs@@ -1,31 +1,37 @@use serde::{Deserialize, Serialize};-use crate::types::{File, Repository};+use crate::types::model::{File, Repository};#[bindings::export(Enum)]#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]#[serde(tag = "type", rename_all = "snake_case")]pub enum Reference {- HuggingFace {- repository: Repository,- },Mirai {toolchain_version: String,repository: Option<Repository>,source_repository: Option<Repository>,files: Vec<File>,},+ HuggingFace {+ repository: Repository,+ },+ Local {+ path: String,+ },}impl Reference {- pub fn r#type(&self) -> String {+ pub fn name(&self) -> String {match self {+ Reference::Mirai {+ ..+ } => "mirai".to_string(),Reference::HuggingFace {..} => "huggingface".to_string(),- Reference::Mirai {+ Reference::Local {..- } => "mirai".to_string(),+ } => "local".to_string(),}}}diff --git a/crates/shoji/src/types/session/chat/config.rs b/crates/shoji/src/types/session/chat/config.rsnew file mode 100644index 000000000..b8ea8d0ef--- /dev/null+++ b/crates/shoji/src/types/session/chat/config.rs@@ -0,0 +1,60 @@+use serde::{Deserialize, Serialize};++use crate::types::{+ basic::SamplingSeed,+ session::chat::{ContextLength, SpeculationPreset},+};++#[bindings::export(Struct, name = "ChatConfig")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct Config {+ pub context_length: ContextLength,+ pub sampling_seed: SamplingSeed,+ pub speculation_preset: Option<SpeculationPreset>,+}++impl Default for Config {+ fn default() -> Self {+ Self {+ context_length: ContextLength::default(),+ sampling_seed: SamplingSeed::default(),+ speculation_preset: None,+ }+ }+}++impl Config {+ pub fn with_context_length(+ self,+ context_length: ContextLength,+ ) -> Self {+ Self {+ context_length,+ ..self+ }+ }+}++impl Config {+ pub fn with_sampling_seed(+ self,+ sampling_seed: SamplingSeed,+ ) -> Self {+ Self {+ sampling_seed,+ ..self+ }+ }+}++impl Config {+ pub fn with_speculation_preset(+ self,+ speculation_preset: Option<SpeculationPreset>,+ ) -> Self {+ Self {+ speculation_preset,+ ..self+ }+ }+}diff --git a/crates/shoji/src/types/session/chat/context_length.rs b/crates/shoji/src/types/session/chat/context_length.rsnew file mode 100644index 000000000..b3b75e617--- /dev/null+++ b/crates/shoji/src/types/session/chat/context_length.rs@@ -0,0 +1,17 @@+use serde::{Deserialize, Serialize};++#[bindings::export(Enum, name = "ChatContextLength")]+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+pub enum ContextLength {+ Default {},+ Maximal {},+ Custom {+ length: i64,+ },+}++impl Default for ContextLength {+ fn default() -> Self {+ ContextLength::Default {}+ }+}diff --git a/crates/shoji/src/types/session/classification_feature.rs b/crates/shoji/src/types/session/chat/feature.rssimilarity index 69%rename from crates/shoji/src/types/session/classification_feature.rsrename to crates/shoji/src/types/session/chat/feature.rsindex f60698022..bd330d6fa 100644--- a/crates/shoji/src/types/session/classification_feature.rs+++ b/crates/shoji/src/types/session/chat/feature.rs@@ -1,8 +1,8 @@use serde::{Deserialize, Serialize};-#[bindings::export(Struct)]+#[bindings::export(Struct, name = "ChatFeature")]#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]-pub struct ClassificationFeature {+pub struct Feature {pub name: String,pub values: Vec<String>,}diff --git a/crates/shoji/src/types/session/chat/finish_reason.rs b/crates/shoji/src/types/session/chat/finish_reason.rsnew file mode 100644index 000000000..694cb1ed8--- /dev/null+++ b/crates/shoji/src/types/session/chat/finish_reason.rs@@ -0,0 +1,12 @@+use serde::{Deserialize, Serialize};++#[bindings::export(Enum, name = "ChatFinishReason")]+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+pub enum FinishReason {+ Stop,+ Length,+ Cancelled,+ ContextLimitReached,+ ToolCalls,+ Rejected,+}diff --git a/crates/shoji/src/types/session/chat/grammar.rs b/crates/shoji/src/types/session/chat/grammar.rsnew file mode 100644index 000000000..7f7b060b3--- /dev/null+++ b/crates/shoji/src/types/session/chat/grammar.rs@@ -0,0 +1,13 @@+use serde::{Deserialize, Serialize};++#[bindings::export(Enum, name = "ChatGrammar")]+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+pub enum Grammar {+ JsonAny {},+ JsonSchema {+ schema: String,+ },+ Regex {+ pattern: String,+ },+}diff --git a/crates/shoji/src/types/session/chat/mod.rs b/crates/shoji/src/types/session/chat/mod.rsnew file mode 100644index 000000000..cfe4d3bea--- /dev/null+++ b/crates/shoji/src/types/session/chat/mod.rs@@ -0,0 +1,19 @@+mod config;+mod context_length;+mod feature;+mod finish_reason;+mod grammar;+mod output;+mod speculation_preset;+mod stats;+mod stream_config;++pub use config::Config;+pub use context_length::ContextLength;+pub use feature::Feature;+pub use finish_reason::FinishReason;+pub use grammar::Grammar;+pub use output::Output;+pub use speculation_preset::SpeculationPreset;+pub use stats::Stats;+pub use stream_config::StreamConfig;diff --git a/crates/shoji/src/types/session/chat/output.rs b/crates/shoji/src/types/session/chat/output.rsnew file mode 100644index 000000000..ba6fecb32--- /dev/null+++ b/crates/shoji/src/types/session/chat/output.rs@@ -0,0 +1,14 @@+use serde::{Deserialize, Serialize};++use crate::types::{+ encoding::Message,+ session::chat::{FinishReason, Stats},+};++#[bindings::export(Struct, name = "ChatOutput")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct Output {+ pub message: Message,+ pub stats: Stats,+ pub finish_reason: Option<FinishReason>,+}diff --git a/crates/shoji/src/types/session/chat/speculation_preset.rs b/crates/shoji/src/types/session/chat/speculation_preset.rsnew file mode 100644index 000000000..9ec68696c--- /dev/null+++ b/crates/shoji/src/types/session/chat/speculation_preset.rs@@ -0,0 +1,13 @@+use serde::{Deserialize, Serialize};++use crate::types::session::chat::Feature;++#[bindings::export(Enum, name = "ChatSpeculationPreset")]+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+pub enum SpeculationPreset {+ GeneralChat {},+ Summarization {},+ Classification {+ feature: Feature,+ },+}diff --git a/crates/shoji/src/types/session/chat/stats.rs b/crates/shoji/src/types/session/chat/stats.rsnew file mode 100644index 000000000..cb58ad58b--- /dev/null+++ b/crates/shoji/src/types/session/chat/stats.rs@@ -0,0 +1,18 @@+use serde::{Deserialize, Serialize};++#[bindings::export(Struct, name = "ChatStats")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]+pub struct Stats {+ pub duration: f64,+ pub time_to_first_token: Option<f64>,+ pub prefill_tokens_per_second: Option<f64>,+ pub generate_tokens_per_second: Option<f64>,+ pub tokens_count_input: Option<u32>,+ pub tokens_count_output: Option<u32>,+}++impl Stats {+ pub fn tokens_count(&self) -> Option<u32> {+ self.tokens_count_input.and_then(|input| self.tokens_count_output.map(|output| input + output))+ }+}diff --git a/crates/shoji/src/types/session/chat/stream_config.rs b/crates/shoji/src/types/session/chat/stream_config.rsnew file mode 100644index 000000000..1fffed2b1--- /dev/null+++ b/crates/shoji/src/types/session/chat/stream_config.rs@@ -0,0 +1,68 @@+use serde::{Deserialize, Serialize};++use crate::types::{+ basic::{SamplingMethod, SamplingPolicy},+ session::chat::Grammar,+};++#[bindings::export(Struct, name = "ChatStreamConfig")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct StreamConfig {+ pub token_limit: Option<u32>,+ pub sampling_policy: SamplingPolicy,+ pub grammar: Option<Grammar>,+}++impl Default for StreamConfig {+ fn default() -> Self {+ Self {+ token_limit: None,+ sampling_policy: SamplingPolicy::default(),+ grammar: None,+ }+ }+}++impl StreamConfig {+ pub fn with_token_limit(+ self,+ token_limit: Option<u32>,+ ) -> Self {+ Self {+ token_limit,+ ..self+ }+ }++ pub fn with_sampling_policy(+ self,+ sampling_policy: SamplingPolicy,+ ) -> Self {+ Self {+ sampling_policy,+ ..self+ }+ }++ pub fn with_sampling_method(+ self,+ sampling_method: SamplingMethod,+ ) -> Self {+ Self {+ sampling_policy: SamplingPolicy::Custom {+ method: sampling_method,+ },+ ..self+ }+ }++ pub fn with_grammar(+ self,+ grammar: Option<Grammar>,+ ) -> Self {+ Self {+ grammar,+ ..self+ }+ }+}diff --git a/crates/shoji/src/types/session/classification/label.rs b/crates/shoji/src/types/session/classification/label.rsnew file mode 100644index 000000000..ed0c90437--- /dev/null+++ b/crates/shoji/src/types/session/classification/label.rs@@ -0,0 +1,11 @@+use serde::{Deserialize, Serialize};++#[bindings::export(Struct)]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct Label {+ pub index: i64,+ pub label: String,+ pub title: String,+ pub description: String,+ pub recommended_threshold: f64,+}diff --git a/crates/shoji/src/types/session/classification/mod.rs b/crates/shoji/src/types/session/classification/mod.rsnew file mode 100644index 000000000..c0ea53c95--- /dev/null+++ b/crates/shoji/src/types/session/classification/mod.rs@@ -0,0 +1,7 @@+mod label;+mod output;+mod stats;++pub use label::Label;+pub use output::Output;+pub use stats::Stats;diff --git a/crates/shoji/src/types/session/classification/output.rs b/crates/shoji/src/types/session/classification/output.rsnew file mode 100644index 000000000..712d53dde--- /dev/null+++ b/crates/shoji/src/types/session/classification/output.rs@@ -0,0 +1,13 @@+use std::collections::HashMap;++use serde::{Deserialize, Serialize};++use crate::types::session::classification::Stats;++#[bindings::export(Struct, name = "ClassificationOutput")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct Output {+ pub logits: Vec<f64>,+ pub probabilities: HashMap<String, f64>,+ pub stats: Stats,+}diff --git a/crates/shoji/src/types/session/classification/stats.rs b/crates/shoji/src/types/session/classification/stats.rsnew file mode 100644index 000000000..f5a7401cb--- /dev/null+++ b/crates/shoji/src/types/session/classification/stats.rs@@ -0,0 +1,14 @@+use serde::{Deserialize, Serialize};++#[bindings::export(Struct, name = "ClassificationStats")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct Stats {+ pub preprocessing_duration: f64,+ pub forward_pass_duration: f64,+ pub postprocessing_duration: f64,+ pub total_duration: f64,+ pub tokens_count: i64,+ pub tokens_per_second: f64,+ pub predicted_label: String,+ pub confidence: f64,+}diff --git a/crates/shoji/src/types/session/mod.rs b/crates/shoji/src/types/session/mod.rsindex 83eefede0..6527a7365 100644--- a/crates/shoji/src/types/session/mod.rs+++ b/crates/shoji/src/types/session/mod.rs@@ -1,3 +1,3 @@-mod classification_feature;--pub use classification_feature::ClassificationFeature;+pub mod chat;+pub mod classification;+pub mod text_to_speech;diff --git a/crates/shoji/src/types/session/text_to_speech/mod.rs b/crates/shoji/src/types/session/text_to_speech/mod.rsnew file mode 100644index 000000000..bfcd648a0--- /dev/null+++ b/crates/shoji/src/types/session/text_to_speech/mod.rs@@ -0,0 +1,5 @@+mod pcm_batch;+mod stats;++pub use pcm_batch::PcmBatch;+pub use stats::Stats;diff --git a/crates/shoji/src/types/session/text_to_speech/pcm_batch.rs b/crates/shoji/src/types/session/text_to_speech/pcm_batch.rsnew file mode 100644index 000000000..c42564d96--- /dev/null+++ b/crates/shoji/src/types/session/text_to_speech/pcm_batch.rs@@ -0,0 +1,10 @@+use serde::{Deserialize, Serialize};++#[bindings::export(Struct)]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct PcmBatch {+ pub samples: Vec<f64>,+ pub sample_rate: u32,+ pub channels: u32,+ pub lengths: Vec<u32>,+}diff --git a/crates/shoji/src/types/session/text_to_speech/stats.rs b/crates/shoji/src/types/session/text_to_speech/stats.rsnew file mode 100644index 000000000..3bb8efa76--- /dev/null+++ b/crates/shoji/src/types/session/text_to_speech/stats.rs@@ -0,0 +1,19 @@+use serde::{Deserialize, Serialize};++#[bindings::export(Struct, name = "TextToSpeechStats")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct Stats {+ pub text_length: u32,+ pub first_chunk_seconds: f64,+ pub generation_duration: f64,+ pub audio_duration: f64,+}++impl Stats {+ pub fn real_time_factor(&self) -> f64 {+ if self.generation_duration <= 0.0 {+ return 0.0;+ }+ self.audio_duration / self.generation_duration+ }+}diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 121f1147b..967f8cedb 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -51,6 +51,7 @@ wasm-bindgen-futures = { workspace = true, optional = true }tsify = { workspace = true, optional = true }[target.'cfg(not(target_family = "wasm"))'.dependencies]+backend-remote.workspace = truenagare.workspace = truedownload-manager.workspace = trueserde.workspace = true@@ -68,8 +69,8 @@ tracing-subscriber.workspace = truetracing-appender.workspace = trueuuid.workspace = truefutures-util.workspace = true-openai-api-rs.workspace = true-regex.workspace = true+async-openai.workspace = true+fancy-regex.workspace = true[target.'cfg(target_vendor = "apple")'.dependencies]objc2.workspace = truediff --git a/crates/uzu/src/engine/config.rs b/crates/uzu/src/engine/config.rsindex 4eeaf207c..ac9927576 100644--- a/crates/uzu/src/engine/config.rs+++ b/crates/uzu/src/engine/config.rs@@ -7,10 +7,13 @@ use serde::{Deserialize, Serialize};#[serde(rename_all = "snake_case")]pub struct Config {pub mirai_api_key: Option<String>,+ pub lalamo_path: Option<String>,pub huggingface_api_key: Option<String>,pub openai_api_key: Option<String>,- pub baseten_api_key: Option<String>,pub anthropic_api_key: Option<String>,+ pub gemini_api_key: Option<String>,+ pub xai_api_key: Option<String>,+ pub baseten_api_key: Option<String>,pub openrouter_api_key: Option<String>,}@@ -18,10 +21,13 @@ impl Default for Config {fn default() -> Self {Self {mirai_api_key: env::var("MIRAI_API_KEY").ok(),+ lalamo_path: env::var("LALAMO_PATH").ok(),huggingface_api_key: env::var("HF_TOKEN").ok(),openai_api_key: env::var("OPENAI_API_KEY").ok(),- baseten_api_key: env::var("BASETEN_API_KEY").ok(),anthropic_api_key: env::var("ANTHROPIC_API_KEY").ok(),+ gemini_api_key: env::var("GEMINI_API_KEY").ok(),+ xai_api_key: env::var("XAI_API_KEY").ok(),+ baseten_api_key: env::var("BASETEN_API_KEY").ok(),openrouter_api_key: env::var("OPENROUTER_API_KEY").ok(),}}@@ -36,6 +42,14 @@ impl Config {self}+ pub fn with_lalamo_path(+ mut self,+ lalamo_path: String,+ ) -> Self {+ self.lalamo_path = Some(lalamo_path);+ self+ }+pub fn with_huggingface_api_key(mut self,huggingface_api_key: String,@@ -52,19 +66,35 @@ impl Config {self}- pub fn with_baseten_api_key(+ pub fn with_anthropic_api_key(mut self,- baseten_api_key: String,+ anthropic_api_key: String,) -> Self {- self.baseten_api_key = Some(baseten_api_key);+ self.anthropic_api_key = Some(anthropic_api_key);self}- pub fn with_anthropic_api_key(+ pub fn with_gemini_api_key(mut self,- anthropic_api_key: String,+ gemini_api_key: String,) -> Self {- self.anthropic_api_key = Some(anthropic_api_key);+ self.gemini_api_key = Some(gemini_api_key);+ self+ }++ pub fn with_xai_api_key(+ mut self,+ xai_api_key: String,+ ) -> Self {+ self.xai_api_key = Some(xai_api_key);+ self+ }++ pub fn with_baseten_api_key(+ mut self,+ baseten_api_key: String,+ ) -> Self {+ self.baseten_api_key = Some(baseten_api_key);self}diff --git a/crates/uzu/src/engine/error.rs b/crates/uzu/src/engine/error.rsindex 2f283f9ac..466beea22 100644--- a/crates/uzu/src/engine/error.rs+++ b/crates/uzu/src/engine/error.rs@@ -1,3 +1,5 @@+use nagare::chat::Error as ChatError;+#[derive(Debug, thiserror::Error)]#[non_exhaustive]pub enum Error {@@ -11,8 +13,10 @@ pub enum Error {Storage(#[from] crate::storage::Error),#[error(transparent)]Registry(#[from] crate::registry::Error),- #[error("Backend error: {message}")]- Backend {- message: String,- },+ #[error("Unable to create backend")]+ UnableToCreateBackend,+ #[error("Backend not found")]+ BackendNotFound,+ #[error(transparent)]+ Chat(#[from] ChatError),}diff --git a/crates/uzu/src/engine/mod.rs b/crates/uzu/src/engine/mod.rsindex aea1d399c..ebb32d55f 100644--- a/crates/uzu/src/engine/mod.rs+++ b/crates/uzu/src/engine/mod.rs@@ -2,15 +2,17 @@ mod config;mod downloader;mod error;-use std::collections::HashMap;+use std::{collections::HashMap, sync::Arc};-use backend_uzu::inference_backend::{Backend as UzuBackend, Config as UzuBackendConfig};+use backend_remote::openai::Backend as OpenAIBackend;+use backend_uzu::inference_backend::Backend as UzuBackend;pub use config::Config;pub use downloader::Downloader;pub use error::Error;+use nagare::chat::Session as ChatSession;use shoji::{- traits::{Backend, BackendInstance, Registry},- types::Model,+ traits::{Backend, Registry},+ types::{model::Model, session::chat::Config as ChatConfig},};use tokio::runtime::Handle;use tokio_stream::wrappers::BroadcastStream;@@ -21,16 +23,20 @@ use crate::{logs,registry::{CachedRegistry, Error as RegistryError, MergedRegistry,+ local::{Config as LocalRegistryConfig, Registry as LocalRegistry},mirai::{Backend as MiraiBackend, Config as MiraiRegistryConfig, Registry as MiraiRegistry},openai::{Config as OpenAIConfig, Registry as OpenAIRegistry},},- storage::{Config as StorageConfig, Storage, types::DownloadState},+ storage::{+ Config as StorageConfig, Storage,+ types::{DownloadPhase, DownloadState},+ },};pub struct Engine {registry: SharedAccess<MergedRegistry>,storage: SharedAccess<Storage>,- backends: HashMap<String, Box<dyn Backend>>,+ backends: HashMap<String, Arc<dyn Backend>>,}impl Engine {@@ -46,55 +52,74 @@ impl Engine {let storage = SharedAccess::new(Storage::new(tokio_handle.clone(), storage_config).await?);- let uzu_backend_config = UzuBackendConfig {};- let uzu_backend: Box<dyn Backend> =- Box::new(UzuBackend::new(uzu_backend_config).map_err(|error| Error::Backend {- message: error.to_string(),- })?);-- let mirai_registry_config = MiraiRegistryConfig {- api_key: config.mirai_api_key,- device: device.clone(),- backends: vec![MiraiBackend {- identifier: uzu_backend.identifier(),- version: uzu_backend.version(),- }],- include_traces: false,- };- let mirai_registry = Box::new(MiraiRegistry::new(mirai_registry_config)?);-let mut engine = Self {storage,registry,backends: HashMap::new(),};- engine.add_backend(uzu_backend);- engine.add_registry(mirai_registry).await?;- if let Some(openai_api_key) = config.openai_api_key {- let openai_registry = OpenAIRegistry::new(OpenAIConfig::openai(openai_api_key))?;- engine.add_registry(Box::new(openai_registry)).await?;- }+ {+ let uzu_backend = UzuBackend::new().map_err(|_| Error::UnableToCreateBackend)?;+ let uzu_backend_identifier = uzu_backend.identifier();+ let uzu_backend_version = uzu_backend.version();- let ollama_config = OpenAIConfig::ollama();- if is_endpoint_reachable(&ollama_config.api_endpoint).await {- let ollama_registry = OpenAIRegistry::new(ollama_config)?;- engine.add_registry(Box::new(ollama_registry)).await?;- }+ let mirai_registry_config = MiraiRegistryConfig {+ api_key: config.mirai_api_key,+ device: device.clone(),+ backends: vec![MiraiBackend {+ identifier: uzu_backend_identifier.clone(),+ version: uzu_backend_version,+ }],+ include_traces: false,+ };+ let mirai_registry = Box::new(MiraiRegistry::new(mirai_registry_config)?);- if let Some(baseten_api_key) = config.baseten_api_key {- let baseten_registry = OpenAIRegistry::new(OpenAIConfig::baseten(baseten_api_key))?;- engine.add_registry(Box::new(baseten_registry)).await?;+ engine.add_backend(Arc::new(uzu_backend) as Arc<dyn Backend>);+ engine.add_registry(mirai_registry).await?;++ if let Some(lalamo_path) = config.lalamo_path {+ let lalamo_registry =+ LocalRegistry::new(LocalRegistryConfig::lalamo(uzu_backend_identifier, lalamo_path))?;+ engine.add_registry(Box::new(lalamo_registry)).await?;+ }}+ let mut openai_configs: Vec<OpenAIConfig> = vec![];+ {+ let ollama_config = OpenAIConfig::ollama();+ if is_endpoint_reachable(&ollama_config.api_endpoint).await {+ openai_configs.push(ollama_config);+ }+ }+ {+ let lmstudio_config = OpenAIConfig::lmstudio();+ if is_endpoint_reachable(&lmstudio_config.api_endpoint).await {+ openai_configs.push(lmstudio_config);+ }+ }+ if let Some(openai_api_key) = config.openai_api_key {+ openai_configs.push(OpenAIConfig::openai(openai_api_key));+ }if let Some(anthropic_api_key) = config.anthropic_api_key {- let anthropic_registry = OpenAIRegistry::new(OpenAIConfig::anthropic(anthropic_api_key))?;- engine.add_registry(Box::new(anthropic_registry)).await?;+ openai_configs.push(OpenAIConfig::anthropic(anthropic_api_key));+ }+ if let Some(gemini_api_key) = config.gemini_api_key {+ openai_configs.push(OpenAIConfig::gemini(gemini_api_key));+ }+ if let Some(xai_api_key) = config.xai_api_key {+ openai_configs.push(OpenAIConfig::xai(xai_api_key));+ }+ if let Some(baseten_api_key) = config.baseten_api_key {+ openai_configs.push(OpenAIConfig::baseten(baseten_api_key));}-if let Some(openrouter_api_key) = config.openrouter_api_key {- let openrouter_registry = OpenAIRegistry::new(OpenAIConfig::openrouter(openrouter_api_key))?;- engine.add_registry(Box::new(openrouter_registry)).await?;+ openai_configs.push(OpenAIConfig::openrouter(openrouter_api_key));+ }+ for config in openai_configs {+ let registry = OpenAIRegistry::new(config.clone())?;+ let backend = OpenAIBackend::new(config.into()).map_err(|_| Error::UnableToCreateBackend)?;+ engine.add_registry(Box::new(registry)).await?;+ engine.add_backend(Arc::new(backend) as Arc<dyn Backend>);}Ok(engine)@@ -124,7 +149,7 @@ impl Engine {impl Engine {pub fn add_backend(&mut self,- backend: Box<dyn Backend>,+ backend: Arc<dyn Backend>,) {self.backends.insert(backend.identifier(), backend);}@@ -142,6 +167,19 @@ impl Engine {self.registry.lock().await.models().await.map_err(Error::from)}+ pub async fn model(+ &self,+ identifier: &str,+ ) -> Result<Option<Model>, Error> {+ if let Some(model) = self.model_by_identifier(identifier).await? {+ return Ok(Some(model));+ }+ if let Some(model) = self.model_by_repo_id(identifier).await? {+ return Ok(Some(model));+ }+ self.model_by_path(identifier).await+ }+pub async fn model_by_identifier(&self,identifier: &str,@@ -155,6 +193,21 @@ impl Engine {) -> Result<Option<Model>, Error> {self.registry.lock().await.model_by_repo_id(repo_id).await.map_err(Error::from)}++ pub async fn model_by_path(+ &self,+ path: &str,+ ) -> Result<Option<Model>, Error> {+ let models = self.models().await?;+ for model in models {+ if let Some(model_path) = self.model_path(&model).await {+ if model_path == path {+ return Ok(Some(model));+ }+ }+ }+ Ok(None)+ }}impl Engine {@@ -169,9 +222,53 @@ impl Engine {self.storage.lock().await.subscribe()}+ pub async fn model_path(+ &self,+ model: &Model,+ ) -> Option<String> {+ let path = if model.is_local() {+ if let Some(local_external_path) = model.local_external_path() {+ Some(local_external_path.clone())+ } else {+ let storage = self.storage.lock().await;+ let state = storage.state(&model.identifier()).await?;+ match state.phase {+ DownloadPhase::Downloaded => {+ storage.config.cache_model_path(model).map(|path| path.to_string_lossy().to_string())+ },+ DownloadPhase::NotDownloaded+ | DownloadPhase::Downloading+ | DownloadPhase::Paused+ | DownloadPhase::Locked+ | DownloadPhase::Error(_) => None,+ }+ }+ } else {+ None+ };+ path+ }+async fn handle_registry_resfresh(&self) -> Result<(), Error> {let models = self.registry.lock().await.models().await?;self.storage.lock().await.refresh(models).await?;Ok(())}}++impl Engine {+ pub async fn chat(+ &self,+ model: Model,+ config: ChatConfig,+ ) -> Result<ChatSession, Error> {+ let path = self.model_path(&model).await;+ if let Some(backend_entity) = model.backend_entity() {+ let backend = self.backends.get(&backend_entity.identifier).ok_or(Error::BackendNotFound)?;+ let session = ChatSession::new(backend.as_ref(), config, model, path).await?;+ Ok(session)+ } else {+ return Err(Error::BackendNotFound);+ }+ }+}diff --git a/crates/uzu/src/registry/cached.rs b/crates/uzu/src/registry/cached.rsindex ee8c1e4f8..08b2055c7 100644--- a/crates/uzu/src/registry/cached.rs+++ b/crates/uzu/src/registry/cached.rs@@ -1,6 +1,6 @@use std::{future::Future, pin::Pin};-use shoji::{traits::Registry, types::Model};+use shoji::{traits::Registry, types::model::Model};use tokio::sync::Mutex;use crate::registry::Error;diff --git a/crates/uzu/src/registry/fixed.rs b/crates/uzu/src/registry/fixed.rsindex cf08faa23..a7df1003a 100644--- a/crates/uzu/src/registry/fixed.rs+++ b/crates/uzu/src/registry/fixed.rs@@ -1,6 +1,6 @@use std::{future::Future, pin::Pin};-use shoji::{traits::Registry, types::Model};+use shoji::{traits::Registry, types::model::Model};use tokio::sync::Mutex;use crate::registry::Error;diff --git a/crates/uzu/src/registry/local/config.rs b/crates/uzu/src/registry/local/config.rsnew file mode 100644index 000000000..dcf3f590b--- /dev/null+++ b/crates/uzu/src/registry/local/config.rs@@ -0,0 +1,41 @@+use std::path::PathBuf;++use serde::{Deserialize, Serialize};++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct Config {+ pub identifier: String,+ pub backend_identifier: String,+ pub name: String,+ pub path: String,+}++impl Config {+ pub fn new(+ identifier: String,+ backend_identifier: String,+ name: String,+ path: String,+ ) -> Self {+ Self {+ identifier,+ backend_identifier,+ name,+ path,+ }+ }++ pub fn lalamo(+ backend_identifier: String,+ path: String,+ ) -> Self {+ let models_path = PathBuf::from(&path).join("models");+ Self::new(+ "lalamo".to_string(),+ backend_identifier,+ "Lalamo".to_string(),+ models_path.to_string_lossy().to_string(),+ )+ }+}diff --git a/crates/uzu/src/registry/local/mod.rs b/crates/uzu/src/registry/local/mod.rsnew file mode 100644index 000000000..11d522878--- /dev/null+++ b/crates/uzu/src/registry/local/mod.rs@@ -0,0 +1,108 @@+mod config;++use std::{fs, future::Future, path::Path, pin::Pin};++pub use config::Config;+use shoji::{+ traits::Registry as RegistryTrait,+ types::model::{Accessibility, Entity, EntityType, Model, Reference, Specialization},+};++use crate::registry::Error;++pub struct Registry {+ config: Config,+}++impl Registry {+ pub fn new(config: Config) -> Result<Self, Error> {+ Ok(Self {+ config,+ })+ }+}++impl RegistryTrait for Registry {+ type Error = Error;++ fn indentifier(&self) -> String {+ self.config.identifier.clone()+ }++ fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, Error>> + Send + '_>> {+ Box::pin(async {+ let path = Path::new(&self.config.path);+ if !path.exists() {+ Err(Error::UnableToGetModels {+ message: format!("Path not found: {}", path.display()),+ })?;+ }++ let entries = fs::read_dir(path).map_err(|error| Error::UnableToGetModels {+ message: error.to_string(),+ })?;++ let mut models = Vec::new();+ for entry in entries.flatten() {+ let path = entry.path();+ let Ok(file_type) = entry.file_type() else {+ continue;+ };+ if !file_type.is_dir() {+ continue;+ }+ let Some(name) = path.file_name().and_then(|name| name.to_str()) else {+ continue;+ };+ models.push(self.model(name));+ }++ Ok(models)+ })+ }+}++impl Registry {+ fn model(+ &self,+ name: &str,+ ) -> Model {+ let path = Path::new(&self.config.path).join(name);+ let identifier = name.to_string();+ Model {+ identifier: identifier.clone(),+ entities: vec![+ self.entity(EntityType::Registry, &self.config.identifier, &self.config.name),+ self.entity(EntityType::Backend, &self.config.backend_identifier, &self.config.backend_identifier),+ self.entity(EntityType::Vendor, &self.config.identifier, &self.config.name),+ self.entity(EntityType::Family, &self.config.identifier, &self.config.name),+ self.entity(EntityType::Variant, &identifier, &identifier),+ ],+ specializations: vec![Specialization::Chat],+ number_of_parameters: None,+ quantization: None,+ accessibility: Accessibility::Local {+ reference: Reference::Local {+ path: path.to_string_lossy().to_string(),+ },+ },+ }+ }++ fn entity(+ &self,+ r#type: EntityType,+ identifier: &str,+ name: &str,+ ) -> Entity {+ Entity {+ r#type,+ identifier: identifier.to_string(),+ parent_identifier: None,+ name: name.to_string(),+ description: None,+ version: None,+ icons: vec![],+ }+ }+}diff --git a/crates/uzu/src/registry/merged.rs b/crates/uzu/src/registry/merged.rsindex 4cbe8d694..b5feb8447 100644--- a/crates/uzu/src/registry/merged.rs+++ b/crates/uzu/src/registry/merged.rs@@ -1,6 +1,6 @@use std::{future::Future, pin::Pin};-use shoji::{traits::Registry, types::Model};+use shoji::{traits::Registry, types::model::Model};use crate::registry::Error;diff --git a/crates/uzu/src/registry/mirai/mod.rs b/crates/uzu/src/registry/mirai/mod.rsindex a022790b1..162fa1cb0 100644--- a/crates/uzu/src/registry/mirai/mod.rs+++ b/crates/uzu/src/registry/mirai/mod.rs@@ -9,7 +9,7 @@ pub use config::{Backend, Config};use indexmap::IndexMap;use nagare::api::{Client, Config as ClientConfig};use reqwest::header::AUTHORIZATION;-use shoji::{traits::Registry as RegistryTrait, types::Model};+use shoji::{traits::Registry as RegistryTrait, types::model::Model};pub use types::Response;use crate::registry::Error;diff --git a/crates/uzu/src/registry/mirai/types.rs b/crates/uzu/src/registry/mirai/types.rsindex 10f40e8e4..f721b3f9a 100644--- a/crates/uzu/src/registry/mirai/types.rs+++ b/crates/uzu/src/registry/mirai/types.rs@@ -1,5 +1,5 @@use serde::{Deserialize, Serialize};-use shoji::types::{Accessibility, Entity, Model, Quantization, Specialization};+use shoji::types::model::{Accessibility, Entity, Model, Quantization, Specialization};#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]#[serde(rename_all = "snake_case")]diff --git a/crates/uzu/src/registry/mod.rs b/crates/uzu/src/registry/mod.rsindex 8f11c8419..62a0a781e 100644--- a/crates/uzu/src/registry/mod.rs+++ b/crates/uzu/src/registry/mod.rs@@ -1,6 +1,7 @@mod cached;mod error;mod fixed;+pub mod local;mod merged;pub mod mirai;pub mod openai;diff --git a/crates/uzu/src/registry/openai/config.rs b/crates/uzu/src/registry/openai/config.rsindex d2c690ff4..f1ae0627f 100644--- a/crates/uzu/src/registry/openai/config.rs+++ b/crates/uzu/src/registry/openai/config.rs@@ -1,7 +1,15 @@use std::collections::HashMap;+use backend_remote::openai::{ApiType as BackendApiType, Config as BackendConfig};use serde::{Deserialize, Serialize};+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum ApiType {+ Completions,+ Responses,+}+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]#[serde(rename_all = "snake_case")]pub struct Config {@@ -9,6 +17,7 @@ pub struct Config {pub name: String,pub api_endpoint: String,pub api_key: Option<String>,+ pub api_type: ApiType,pub model_filter_pattern: Option<String>,pub headers: Option<HashMap<String, String>>,}@@ -19,6 +28,7 @@ impl Config {name: String,api_endpoint: String,api_key: Option<String>,+ api_type: ApiType,model_filter_pattern: Option<String>,headers: Option<HashMap<String, String>>,) -> Self {@@ -27,33 +37,44 @@ impl Config {name,api_endpoint,api_key,+ api_type,model_filter_pattern,headers,}}- pub fn openai(api_key: String) -> Self {+ pub fn ollama() -> Self {Self::new(- "openai".to_string(),- "OpenAI".to_string(),- "https://api.openai.com/v1".to_string(),- Some(api_key),- Some(r"^(?:gpt-\d[\d.]*[a-z]?|o\d+)(?:-(?:mini|nano|pro|turbo|codex(?:-mini|-max)?))*$".to_string()),+ "ollama".to_string(),+ "Ollama".to_string(),+ "http://localhost:11434/v1".to_string(),+ None,+ ApiType::Responses,+ None,None,)}- pub fn ollama() -> Self {- Self::new("ollama".to_string(), "Ollama".to_string(), "http://localhost:11434/v1".to_string(), None, None, None)+ pub fn lmstudio() -> Self {+ Self::new(+ "lmstudio".to_string(),+ "LM Studio".to_string(),+ "http://localhost:1234/v1".to_string(),+ None,+ ApiType::Responses,+ Some(r"^(?!.*embedding).+$".to_string()),+ None,+ )}- pub fn baseten(api_key: String) -> Self {+ pub fn openai(api_key: String) -> Self {Self::new(- "baseten".to_string(),- "Baseten".to_string(),- "https://inference.baseten.co/v1".to_string(),+ "openai".to_string(),+ "OpenAI".to_string(),+ "https://api.openai.com/v1".to_string(),Some(api_key),- None,+ ApiType::Responses,+ Some(r"^(?:gpt-\d[\d.]*[a-z]?|o\d+)(?:-(?:mini|nano|pro|turbo|codex(?:-mini|-max)?))*$".to_string()),None,)}@@ -62,8 +83,9 @@ impl Config {Self::new("anthropic".to_string(),"Anthropic".to_string(),- "https://api.anthropic.com/v1/".to_string(),+ "https://api.anthropic.com/v1".to_string(),Some(api_key.clone()),+ ApiType::Completions,Some(r"^claude-(?:opus|sonnet|haiku)-\d{1,2}(?:-\d{1,2})*$".to_string()),Some(HashMap::from([("X-Api-Key".to_string(), api_key.clone()),@@ -72,14 +94,72 @@ impl Config {)}+ pub fn gemini(api_key: String) -> Self {+ Self::new(+ "gemini".to_string(),+ "Gemini".to_string(),+ "https://generativelanguage.googleapis.com/v1beta/openai".to_string(),+ Some(api_key),+ ApiType::Completions,+ Some(r"^models/(?:gemini|gemma)-(?!.*(?:image|audio|tts|live|embedding|computer-use|robotics|preview|latest|-\d{3}$|-\d{2}-\d{4}$)).+$".to_string()),+ None,+ )+ }++ pub fn xai(api_key: String) -> Self {+ Self::new(+ "xai".to_string(),+ "xAI".to_string(),+ "https://api.x.ai/v1".to_string(),+ Some(api_key),+ ApiType::Responses,+ Some(r"^grok-(?!.*(?:imagine|-\d{4})).+$".to_string()),+ None,+ )+ }++ pub fn baseten(api_key: String) -> Self {+ Self::new(+ "baseten".to_string(),+ "Baseten".to_string(),+ "https://inference.baseten.co/v1".to_string(),+ Some(api_key),+ ApiType::Completions,+ None,+ None,+ )+ }+pub fn openrouter(api_key: String) -> Self {Self::new("openrouter".to_string(),"OpenRouter".to_string(),"https://openrouter.ai/api/v1".to_string(),Some(api_key.clone()),+ ApiType::Responses,None,None,)}}++impl From<Config> for BackendConfig {+ fn from(config: Config) -> Self {+ Self {+ identifier: config.identifier,+ api_endpoint: config.api_endpoint,+ api_key: config.api_key,+ api_type: config.api_type.into(),+ headers: config.headers,+ }+ }+}++impl From<ApiType> for BackendApiType {+ fn from(value: ApiType) -> Self {+ match value {+ ApiType::Completions => BackendApiType::Completions,+ ApiType::Responses => BackendApiType::Responses,+ }+ }+}diff --git a/crates/uzu/src/registry/openai/mod.rs b/crates/uzu/src/registry/openai/mod.rsindex fb357bb11..dae4442fa 100644--- a/crates/uzu/src/registry/openai/mod.rs+++ b/crates/uzu/src/registry/openai/mod.rs@@ -2,37 +2,53 @@ mod config;use std::{future::Future, pin::Pin};+use async_openai::{Client, config::OpenAIConfig};pub use config::Config;-use openai_api_rs::v1::api::OpenAIClient;-use regex::Regex;+use fancy_regex::Regex;+use serde::Deserialize;use shoji::{traits::Registry as RegistryTrait,- types::{Accessibility, Entity, EntityType, Model, Specialization},+ types::model::{Accessibility, Entity, EntityType, Model as ShojiModel, Specialization},};+#[derive(Debug, Deserialize)]+struct Model {+ id: String,+}++#[derive(Debug, Deserialize)]+struct ListModelsResponse {+ data: Vec<Model>,+}+use crate::registry::Error;pub struct Registry {config: Config,- client: OpenAIClient,+ client: Client<OpenAIConfig>,model_filter: Option<Regex>,}impl Registry {pub fn new(config: Config) -> Result<Self, Error> {- let mut client_builder = OpenAIClient::builder().with_endpoint(config.api_endpoint.clone());- if let Some(api_key) = config.api_key.clone() {- client_builder = client_builder.with_api_key(api_key);+ let mut openai_config = OpenAIConfig::new().with_api_base(&config.api_endpoint);+ if let Some(api_key) = config.api_key.as_ref() {+ openai_config = openai_config.with_api_key(api_key);}- if let Some(headers) = config.headers.clone() {- for (key, value) in headers.iter() {- client_builder = client_builder.with_header(key, value);+ if let Some(headers) = config.headers.as_ref() {+ for (key, value) in headers {+ let name =+ reqwest::header::HeaderName::from_bytes(key.as_bytes()).map_err(|error| Error::UnableToCreate {+ message: error.to_string(),+ })?;+ openai_config =+ openai_config.with_header(name, value.as_str()).map_err(|error| Error::UnableToCreate {+ message: error.to_string(),+ })?;}}- let client = client_builder.build().map_err(|error| Error::UnableToCreate {- message: error.to_string(),- })?;+ let client = Client::with_config(openai_config);let model_filter = config.model_filter_pattern.as_deref().map(Regex::new).transpose().map_err(|error| {Error::UnableToCreate {@@ -55,13 +71,13 @@ impl RegistryTrait for Registry {self.config.identifier.clone()}- fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, Error>> + Send + '_>> {+ fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<ShojiModel>, Error>> + Send + '_>> {Box::pin(async {- let response = self.client.list_models().await.map_err(|error| Error::UnableToGetModels {- message: error.to_string(),- })?;- let mut identifiers =- response.inner.data.into_iter().filter_map(|model| model.id.clone()).collect::<Vec<_>>();+ let response: ListModelsResponse =+ self.client.models().list_byot().await.map_err(|error| Error::UnableToGetModels {+ message: error.to_string(),+ })?;+ let mut identifiers = response.data.into_iter().map(|model| model.id).collect::<Vec<_>>();identifiers.sort();let registry_entity = self.create_entity(EntityType::Registry);@@ -71,7 +87,9 @@ impl RegistryTrait for Registry {let models = identifiers.into_iter()- .filter(|identifier| self.model_filter.as_ref().is_none_or(|regex| regex.is_match(identifier)))+ .filter(|identifier| {+ self.model_filter.as_ref().is_none_or(|regex| regex.is_match(identifier).unwrap_or(false))+ }).map(|identifier| {let variant_entity = Entity {r#type: EntityType::Variant,@@ -82,7 +100,7 @@ impl RegistryTrait for Registry {version: None,icons: vec![],};- let model = Model {+ let model = ShojiModel {identifier: identifier.clone(),entities: vec![registry_entity.clone(),diff --git a/crates/uzu/src/storage/config.rs b/crates/uzu/src/storage/config.rsindex 84b307da2..d13867d80 100644--- a/crates/uzu/src/storage/config.rs+++ b/crates/uzu/src/storage/config.rs@@ -1,7 +1,7 @@use std::path::PathBuf;use serde::{Deserialize, Serialize};-use shoji::types::Model;+use shoji::types::model::Model;use crate::device::Device;@@ -38,9 +38,9 @@ impl Config {&self,model: &Model,) -> Option<PathBuf> {- let reference_type = model.reference_type()?;+ let reference_name = model.reference_name()?;let checkpoint_version = model.checkpoint_version()?;- Some(self.cache_models_path().join(reference_type).join(model.cache_identifier()).join(checkpoint_version))+ Some(self.cache_models_path().join(reference_name).join(model.cache_identifier()).join(checkpoint_version))}pub fn log_name(&self) -> String {diff --git a/crates/uzu/src/storage/mod.rs b/crates/uzu/src/storage/mod.rsindex aa3c87818..2f9ad7f55 100644--- a/crates/uzu/src/storage/mod.rs+++ b/crates/uzu/src/storage/mod.rs@@ -13,7 +13,7 @@ use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase, create_download_manager,};pub use error::Error;-use shoji::types::{Accessibility, File, Model, Reference};+use shoji::types::model::{Accessibility, File, Model, Reference};use tokio::{runtime::Handle,sync::broadcast::{Sender, channel},@@ -26,7 +26,8 @@ use crate::{};pub struct Storage {- config: Config,+ pub config: Config,+download_manager: SharedAccess<Arc<dyn FileDownloadManager>>,items: SharedAccess<HashMap<String, Item>>,items_broadcast_sender: Sender<(String, DownloadState)>,@@ -74,7 +75,7 @@ impl Storage {&self,models: Vec<Model>,) -> Result<(), Error> {- let models = models.into_iter().filter(|model| model.is_local()).collect::<Vec<_>>();+ let models = models.into_iter().filter(|model| model.is_downloadable()).collect::<Vec<_>>();let actual_model_identifiers: HashSet<String> = models.iter().map(|model| model.identifier()).collect();let download_manager = { self.download_manager.lock().await.clone() };@@ -264,6 +265,11 @@ impl Storage {} => Err(Error::UnsupportedItem {identifier: model.identifier(),}),+ Reference::Local {+ ..+ } => Err(Error::UnsupportedItem {+ identifier: model.identifier(),+ }),},Accessibility::Remote {..diff --git a/crates/uzu/src/storage/types/item.rs b/crates/uzu/src/storage/types/item.rsindex 86abf1252..dfa73bf9f 100644--- a/crates/uzu/src/storage/types/item.rs+++ b/crates/uzu/src/storage/types/item.rs@@ -6,7 +6,7 @@ use std::{use download_manager::{FileDownloadManager, FileDownloadState, FileDownloadTask};use futures_util::future::join_all;-use shoji::types::File;+use shoji::types::model::File;use tokio::{runtime::Handle,sync::broadcast::{Sender, channel},diff --git a/crates/uzu/tests/common/test_helpers.rs b/crates/uzu/tests/common/test_helpers.rsindex c52ea5727..586179bbf 100644--- a/crates/uzu/tests/common/test_helpers.rs+++ b/crates/uzu/tests/common/test_helpers.rs@@ -2,7 +2,7 @@use std::path::PathBuf;-use shoji::types::{Accessibility, Entity, EntityType, File, Hash, HashMethod, Model, Reference};+use shoji::types::model::{Accessibility, Entity, EntityType, File, Hash, HashMethod, Model, Reference};use tokio::runtime::Handle;use uzu::{device::Device,diff --git a/crates/uzu/tests/test_engine.rs b/crates/uzu/tests/test_engine.rsindex 74b78a64b..e01fb250b 100644--- a/crates/uzu/tests/test_engine.rs+++ b/crates/uzu/tests/test_engine.rs@@ -1,7 +1,13 @@#![cfg(not(target_family = "wasm"))]+use futures::StreamExt;+use shoji::types::{+ encoding::Message,+ session::chat::{Config as ChatConfig, StreamConfig as ChatStreamConfig},+};use uzu::engine::{Config, Engine};+#[ignore]#[tokio::test]async fn test_engine() {dotenvy::dotenv().ok();@@ -14,6 +20,45 @@ async fn test_engine() {let identifier = model.identifier();let registry_identifier = model.registry_entity().map(|entity| entity.identifier);let download_phase = engine.downloader(&model).state().await.map_or(None, |state| Some(state.phase));- println!("{:?}, {}, {:?}", registry_identifier, identifier, download_phase);+ println!("{:?}, {}, {:?}, {:?}", registry_identifier, identifier, download_phase, model.specializations);+ }++ let model = engine.model("gpt-3.5-turbo").await.unwrap().unwrap();+ let session = engine.chat(model, ChatConfig::default()).await.unwrap();+ session.reset().await.unwrap();++ let messages = vec![+ Message::system().with_text("You are a helpful assistant.".to_string()),+ Message::user().with_text("My name is John Doe".to_string()),+ ];+ let (mut stream, _) = session.stream(messages, ChatStreamConfig::default());+ while let Some(progress) = stream.next().await {+ match progress {+ Ok(outputs) => {+ let state = session.state().await;+ let messages = session.messages().await;+ let roles = messages.iter().map(|message| message.role.clone()).collect::<Vec<_>>();+ println!("State: {state:?}");+ println!("Roles: {roles:?}");+ for output in outputs {+ let duration = output.stats.duration;+ let finish_reason = output.finish_reason;+ let text_length = output.message.text().len();+ println!(+ "\tDuration: {duration}\n\tFinish reason: {finish_reason:?}\n\tText length: {text_length}"+ );+ }+ },+ Err(error) => eprintln!("Stream error: {error}"),+ }+ }++ let messages = vec![Message::user().with_text("What is my name?".to_string())];+ let (mut stream, _) = session.stream(messages, ChatStreamConfig::default());+ while let Some(_) = stream.next().await {}++ let messages = session.messages().await;+ for message in messages {+ println!("Message: {message:?}");}}diff --git a/crates/uzu/tests/test_storage_comprehensive_state.rs b/crates/uzu/tests/test_storage_comprehensive_state.rsindex 7d7c2d2d3..4c5ade0c0 100644--- a/crates/uzu/tests/test_storage_comprehensive_state.rs+++ b/crates/uzu/tests/test_storage_comprehensive_state.rs@@ -96,6 +96,7 @@ async fn test_storage_comprehensive_state_pause_and_resume() -> Result<(), Box<dOk(())}+#[ignore]#[tokio::test(flavor = "multi_thread")]async fn test_storage_comprehensive_state_pause_quit_relaunch_resume() -> Result<(), Box<dyn std::error::Error>> {{@@ -169,6 +170,7 @@ async fn test_storage_comprehensive_state_pause_quit_relaunch_resume() -> ResultOk(())}+#[ignore]#[tokio::test(flavor = "multi_thread")]async fn test_storage_comprehensive_state_delete_from_various_states() -> Result<(), Box<dyn std::error::Error>> {let temp_dir = tempfile::tempdir()?;@@ -244,6 +246,7 @@ async fn test_storage_comprehensive_state_multiple_pause_resume_cycles() -> ResuOk(())}+#[ignore]#[tokio::test(flavor = "multi_thread")]async fn test_storage_comprehensive_state_completed_files_preserved_on_relaunch()-> Result<(), Box<dyn std::error::Error>> {@@ -331,6 +334,7 @@ async fn test_storage_comprehensive_state_completed_files_preserved_on_relaunch(Ok(())}+#[ignore]#[tokio::test(flavor = "multi_thread")]async fn test_storage_comprehensive_state_idempotent_pause() -> Result<(), Box<dyn std::error::Error>> {let temp_dir = tempfile::tempdir()?;@@ -416,6 +420,7 @@ async fn test_storage_comprehensive_state_crc_validation_on_init() -> Result<(),Ok(())}+#[ignore]#[tokio::test(flavor = "multi_thread")]async fn test_storage_comprehensive_state_progress_calculation_with_mixed_files()-> Result<(), Box<dyn std::error::Error>> {diff --git a/deny.toml b/deny.tomlindex 2c14d8148..0519c8b89 100644--- a/deny.toml+++ b/deny.toml@@ -4,6 +4,8 @@ ignore = ["RUSTSEC-2024-0436", # paste - no longer maintained"RUSTSEC-2024-0375", # `atty` is unmaintained"RUSTSEC-2025-0057", # fxhash - no longer maintained+ "RUSTSEC-2024-0384", # instant is unmaintained+ "RUSTSEC-2025-0012", # backoff is unmaintained][licenses]