Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Bindings migration 3/n (#355)

Aleksei Savin6139794b9fb2
diff --git a/.env.sample b/.env.sample
index 42ed54a35..06e02c764 100644
--- a/.env.sample
+++ b/.env.sample
@@ -1,6 +1,9 @@
MIRAI_API_KEY=
+LALAMO_PATH=
HF_TOKEN=
OPENAI_API_KEY=
-BASETEN_API_KEY=
ANTHROPIC_API_KEY=
+GEMINI_API_KEY=
+XAI_API_KEY=
+BASETEN_API_KEY=
OPENROUTER_API_KEY=
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index d45c984f8..22747da7f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -256,6 +256,46 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "async-openai"
+version = "0.35.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "583553a05a8cbf55bbd67d6a824a206c21030556fdc5f43de387959f888fb6be"
+dependencies = [
+ "async-openai-macros",
+ "backoff",
+ "base64 0.22.1",
+ "bytes",
+ "derive_builder",
+ "eventsource-stream",
+ "futures",
+ "getrandom 0.3.4",
+ "rand 0.9.4",
+ "reqwest 0.12.28",
+ "reqwest-eventsource",
+ "secrecy",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "thiserror 2.0.18",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "async-openai-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81872a8e595e8ceceab71c6ba1f9078e313b452a1e31934e6763ef5d308705e4"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
[[package]]
name = "async-shutdown"
version = "0.2.2"
@@ -502,6 +542,21 @@ dependencies = [
"fs_extra",
]
+[[package]]
+name = "backend-remote"
+version = "0.1.9"
+dependencies = [
+ "async-openai",
+ "futures",
+ "indexmap 2.14.0",
+ "reqwest 0.13.2",
+ "serde",
+ "serde_json",
+ "shoji",
+ "thiserror 2.0.18",
+ "tokio-util",
+]
+
[[package]]
name = "backend-uzu"
version = "0.1.9"
@@ -555,12 +610,27 @@ dependencies = [
"thiserror 2.0.18",
"tokenizers",
"tokio",
+ "tokio-util",
"walkdir",
"xgrammar-rs",
"xxhash-rust",
"zerocopy",
]
+[[package]]
+name = "backoff"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"
+dependencies = [
+ "futures-core",
+ "getrandom 0.2.17",
+ "instant",
+ "pin-project-lite",
+ "rand 0.8.5",
+ "tokio",
+]
+
[[package]]
name = "base64"
version = "0.13.1"
@@ -2026,6 +2096,17 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "eventsource-stream"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab"
+dependencies = [
+ "futures-core",
+ "nom 7.1.3",
+ "pin-project-lite",
+]
+
[[package]]
name = "exr"
version = "1.74.0"
@@ -2332,6 +2413,12 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
+[[package]]
+name = "futures-timer"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
+
[[package]]
name = "futures-util"
version = "0.3.32"
@@ -2753,6 +2840,7 @@ dependencies = [
"hyper 1.9.0",
"hyper-util",
"rustls",
+ "rustls-native-certs",
"tokio",
"tokio-rustls",
"tower-service",
@@ -3073,6 +3161,15 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "instant"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
+dependencies = [
+ "cfg-if 1.0.4",
+]
+
[[package]]
name = "interpolate_name"
version = "0.2.4"
@@ -3784,6 +3881,7 @@ dependencies = [
name = "nagare"
version = "0.1.9"
dependencies = [
+ "futures",
"hanashi",
"indexmap 2.14.0",
"reqwest 0.13.2",
@@ -3791,6 +3889,9 @@ dependencies = [
"serde_json",
"shoji",
"thiserror 2.0.18",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
]
[[package]]
@@ -4232,22 +4333,6 @@ version = "11.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
-[[package]]
-name = "openai-api-rs"
-version = "10.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba538da8c96b5f486ec8706a1b4663fa9c46be33cf12f8bf43e82e4c7da8acb5"
-dependencies = [
- "bytes",
- "futures-util",
- "reqwest 0.12.28",
- "serde",
- "serde_json",
- "tokio",
- "tokio-tungstenite",
- "url",
-]
-
[[package]]
name = "openai-harmony"
version = "0.0.8"
@@ -5331,6 +5416,7 @@ dependencies = [
"pin-project-lite",
"quinn",
"rustls",
+ "rustls-native-certs",
"rustls-pki-types",
"serde",
"serde_json",
@@ -5396,6 +5482,22 @@ dependencies = [
"web-sys",
]
+[[package]]
+name = "reqwest-eventsource"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "632c55746dbb44275691640e7b40c907c16a2dc1a5842aa98aaec90da6ec6bde"
+dependencies = [
+ "eventsource-stream",
+ "futures-core",
+ "futures-timer",
+ "mime",
+ "nom 7.1.3",
+ "pin-project-lite",
+ "reqwest 0.12.28",
+ "thiserror 1.0.69",
+]
+
[[package]]
name = "rgb"
version = "0.8.53"
@@ -5754,6 +5856,16 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "secrecy"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a"
+dependencies = [
+ "serde",
+ "zeroize",
+]
+
[[package]]
name = "security-framework"
version = "3.7.0"
@@ -5973,6 +6085,7 @@ name = "shoji"
version = "0.1.9"
dependencies = [
"bindings",
+ "futures",
"indexmap 2.14.0",
"napi",
"napi-build",
@@ -5982,6 +6095,7 @@ dependencies = [
"serde-wasm-bindgen",
"serde_json",
"thiserror 2.0.18",
+ "tokio-util",
"tsify",
"uniffi",
"wasm-bindgen",
@@ -6626,20 +6740,6 @@ dependencies = [
"tokio-util",
]
-[[package]]
-name = "tokio-tungstenite"
-version = "0.28.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857"
-dependencies = [
- "futures-util",
- "log",
- "native-tls",
- "tokio",
- "tokio-native-tls",
- "tungstenite",
-]
-
[[package]]
name = "tokio-util"
version = "0.7.18"
@@ -6649,6 +6749,7 @@ dependencies = [
"bytes",
"futures-core",
"futures-sink",
+ "futures-util",
"pin-project-lite",
"tokio",
]
@@ -6905,24 +7006,6 @@ dependencies = [
"syn 2.0.117",
]
-[[package]]
-name = "tungstenite"
-version = "0.28.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442"
-dependencies = [
- "bytes",
- "data-encoding",
- "http 1.4.0",
- "httparse",
- "log",
- "native-tls",
- "rand 0.9.4",
- "sha1",
- "thiserror 2.0.18",
- "utf-8",
-]
-
[[package]]
name = "typenum"
version = "1.19.0"
@@ -7207,12 +7290,6 @@ dependencies = [
"serde",
]
-[[package]]
-name = "utf-8"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
-
[[package]]
name = "utf8-zero"
version = "0.8.1"
@@ -7258,12 +7335,15 @@ dependencies = [
name = "uzu"
version = "0.1.9"
dependencies = [
+ "async-openai",
+ "backend-remote",
"backend-uzu",
"bindings",
"chrono",
"dirs",
"dotenvy",
"download-manager",
+ "fancy-regex 0.17.0",
"futures",
"futures-util",
"indexmap 2.14.0",
@@ -7275,9 +7355,7 @@ dependencies = [
"objc2",
"objc2-foundation",
"objc2-security",
- "openai-api-rs",
"pyo3",
- "regex",
"reqwest 0.13.2",
"serde",
"serde-wasm-bindgen",
diff --git a/Cargo.toml b/Cargo.toml
index 450193eef..3eb8f5043 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,12 +3,13 @@ members = [
"crates/dsl",
"crates/bindings",
"crates/shoji",
+ "crates/backend-uzu",
+ "crates/backend-remote",
"crates/json-transform",
"crates/token-stream-parser",
"crates/hanashi",
"crates/nagare",
"crates/download-manager",
- "crates/backend-uzu",
"crates/uzu",
"crates/benchmarks",
"crates/cli",
@@ -28,13 +29,14 @@ repository = "https://github.com/trymirai/uzu"
dsl = { path = "crates/dsl" }
bindings = { path = "crates/bindings", default-features = false }
shoji = { path = "crates/shoji", default-features = false }
+backend-uzu = { path = "crates/backend-uzu", default-features = false }
+backend-remote = { path = "crates/backend-remote", default-features = false }
json-transform = { path = "crates/json-transform", default-features = false }
token-stream-parser = { path = "crates/token-stream-parser", default-features = false }
hanashi = { path = "crates/hanashi", default-features = false }
nagare = { path = "crates/nagare", default-features = false }
download-manager = { path = "crates/download-manager", default-features = false }
benchmarks = { path = "crates/benchmarks", default-features = false }
-backend-uzu = { path = "crates/backend-uzu", default-features = false }
uzu = { path = "crates/uzu", default-features = false }
xgrammar = { package = "xgrammar-rs", version = "0.1", features = ["hf"] }
half = { version = "2.7", features = ["bytemuck", "num-traits"] }
@@ -118,7 +120,7 @@ reqwest = { version = "0.13.2", features = [
"blocking",
"query",
] }
-tokio-util = "0.7.17"
+tokio-util = { version = "0.7.17", default-features = false, features = ["rt"] }
async-shutdown = "0.2.2"
objc-sys = "0.3.5"
crc32c = "0.6.8"
@@ -139,7 +141,7 @@ dirs = "6.0.0"
dotenvy = "0.15.7"
ratatui = "0.30.0"
crossterm = "0.29.0"
-openai-api-rs = "10.0.0"
+async-openai = { version = "0.35.0", features = ["chat-completion", "model", "byot", "responses"] }
[profile.dev.build-override]
opt-level = 3
diff --git a/crates/backend-remote/Cargo.toml b/crates/backend-remote/Cargo.toml
new file mode 100644
index 000000000..c35ff571b
--- /dev/null
+++ b/crates/backend-remote/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "backend-remote"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+authors.workspace = true
+repository.workspace = true
+
+[dependencies]
+shoji.workspace = true
+async-openai.workspace = true
+futures.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tokio-util.workspace = true
+serde.workspace = true
+reqwest.workspace = true
+indexmap.workspace = true
diff --git a/crates/backend-remote/src/lib.rs b/crates/backend-remote/src/lib.rs
new file mode 100644
index 000000000..d8c308735
--- /dev/null
+++ b/crates/backend-remote/src/lib.rs
@@ -0,0 +1 @@
+pub mod openai;
diff --git a/crates/backend-remote/src/openai/bridging/chat/api_stream.rs b/crates/backend-remote/src/openai/bridging/chat/api_stream.rs
new file mode 100644
index 000000000..4c35a9aa2
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/chat/api_stream.rs
@@ -0,0 +1,66 @@
+use std::{pin::Pin, sync::Arc};
+
+use async_openai::{Client, config::OpenAIConfig};
+use futures::{FutureExt, Stream, StreamExt, stream};
+use shoji::{
+ traits::backend::{Error as BackendError, chat_message::Output},
+ types::{encoding::Message, session::chat::StreamConfig},
+};
+use tokio_util::sync::CancellationToken;
+
+use crate::openai::{
+ Error,
+ bridging::{
+ self,
+ chat::{request, stream_chunk},
+ },
+ stream_state::StreamState,
+};
+
+pub struct ApiStream;
+
+impl bridging::ApiStream for ApiStream {
+ fn stream(
+ &self,
+ client: Arc<Client<OpenAIConfig>>,
+ model: String,
+ config: StreamConfig,
+ messages: Vec<Message>,
+ cancel: CancellationToken,
+ ) -> Pin<Box<dyn Stream<Item = Result<Output, BackendError>> + Send>> {
+ let stream = async move {
+ let request = match request::build(&model, &config, messages) {
+ Ok(request) => request,
+ Err(error) => {
+ return stream::once(async move { Err(Box::new(error) as BackendError) }).boxed();
+ },
+ };
+ match client.chat().create_stream_byot::<_, stream_chunk::Response>(request).await {
+ Ok(upstream) => {
+ let mut state = StreamState::new();
+ upstream
+ .filter_map(move |response| {
+ let item = match response {
+ Ok(response) => {
+ stream_chunk::build(response).and_then(|chunk| state.process_chunk(chunk)).map(Ok)
+ },
+ Err(error) => Some(Err(Box::new(Error::Network {
+ message: error.to_string(),
+ }) as BackendError)),
+ };
+ futures::future::ready(item)
+ })
+ .boxed()
+ },
+ Err(error) => stream::once(async move {
+ Err(Box::new(Error::Network {
+ message: error.to_string(),
+ }) as BackendError)
+ })
+ .boxed(),
+ }
+ };
+
+ stream.flatten_stream().take_until(cancel.cancelled_owned()).boxed()
+ }
+}
diff --git a/crates/backend-remote/src/openai/bridging/chat/finish_reason.rs b/crates/backend-remote/src/openai/bridging/chat/finish_reason.rs
new file mode 100644
index 000000000..74e673661
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/chat/finish_reason.rs
@@ -0,0 +1,12 @@
+use async_openai::types::chat::FinishReason as ChatFinishReason;
+use shoji::types::session::chat::FinishReason;
+
+pub fn build(reason: ChatFinishReason) -> FinishReason {
+ match reason {
+ ChatFinishReason::Stop => FinishReason::Stop,
+ ChatFinishReason::Length => FinishReason::Length,
+ ChatFinishReason::ToolCalls => FinishReason::ToolCalls,
+ ChatFinishReason::ContentFilter => FinishReason::Rejected,
+ ChatFinishReason::FunctionCall => FinishReason::ToolCalls,
+ }
+}
diff --git a/crates/backend-remote/src/openai/bridging/chat/message.rs b/crates/backend-remote/src/openai/bridging/chat/message.rs
new file mode 100644
index 000000000..309cca8bf
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/chat/message.rs
@@ -0,0 +1,86 @@
+use async_openai::types::chat::{
+ ChatCompletionMessageToolCall, ChatCompletionMessageToolCalls, ChatCompletionRequestAssistantMessage,
+ ChatCompletionRequestAssistantMessageContent, ChatCompletionRequestMessage, ChatCompletionRequestSystemMessage,
+ ChatCompletionRequestSystemMessageContent, ChatCompletionRequestToolMessage,
+ ChatCompletionRequestToolMessageContent, ChatCompletionRequestUserMessage, ChatCompletionRequestUserMessageContent,
+ FunctionCall,
+};
+use shoji::types::encoding::{Message, Role};
+
+use crate::openai::Error;
+
+enum ResolvedRole {
+ System,
+ User,
+ Assistant,
+ Tool,
+}
+
+pub fn build(message: &Message) -> Result<ChatCompletionRequestMessage, Error> {
+ let (role, content, tool_calls, tool_call_id) = match message.role {
+ Role::System {} => (ResolvedRole::System, message.text(), None, None),
+ Role::User {} => (ResolvedRole::User, message.text(), None, None),
+ Role::Assistant {} => (ResolvedRole::Assistant, message.text(), Some(message.tool_calls()), None),
+ Role::Tool {} => {
+ let tool_call_results = message.tool_call_results();
+ if tool_call_results.len() != 1 {
+ return Err(Error::ToolCallResultRequired);
+ }
+ let (tool_call_id, _, value) = tool_call_results.first().ok_or(Error::ToolCallResultRequired)?;
+ let content = serde_json::to_string(&value).map_err(|error| Error::Serialization {
+ message: error.to_string(),
+ })?;
+ (ResolvedRole::Tool, content, None, tool_call_id.clone())
+ },
+ Role::Developer {}
+ | Role::Custom {
+ ..
+ } => return Err(Error::UnsupportedRole),
+ };
+
+ let tool_calls: Option<Vec<ChatCompletionMessageToolCalls>> = tool_calls.map(|calls| {
+ calls
+ .into_iter()
+ .map(|tool_call| {
+ ChatCompletionMessageToolCalls::Function(ChatCompletionMessageToolCall {
+ id: tool_call.identifier.unwrap_or_default(),
+ function: FunctionCall {
+ name: tool_call.name.clone(),
+ arguments: tool_call.arguments.json.clone(),
+ },
+ })
+ })
+ .collect()
+ });
+
+ Ok(match role {
+ ResolvedRole::System => ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage {
+ content: ChatCompletionRequestSystemMessageContent::Text(content),
+ name: None,
+ }),
+ ResolvedRole::User => ChatCompletionRequestMessage::User(ChatCompletionRequestUserMessage {
+ content: ChatCompletionRequestUserMessageContent::Text(content),
+ name: None,
+ }),
+ ResolvedRole::Assistant => {
+ let text_content = if content.is_empty() {
+ None
+ } else {
+ Some(ChatCompletionRequestAssistantMessageContent::Text(content))
+ };
+ #[allow(deprecated)]
+ ChatCompletionRequestMessage::Assistant(ChatCompletionRequestAssistantMessage {
+ content: text_content,
+ refusal: None,
+ name: None,
+ audio: None,
+ tool_calls,
+ function_call: None,
+ })
+ },
+ ResolvedRole::Tool => ChatCompletionRequestMessage::Tool(ChatCompletionRequestToolMessage {
+ content: ChatCompletionRequestToolMessageContent::Text(content),
+ tool_call_id: tool_call_id.unwrap_or_default(),
+ }),
+ })
+}
diff --git a/crates/backend-remote/src/openai/bridging/chat/mod.rs b/crates/backend-remote/src/openai/bridging/chat/mod.rs
new file mode 100644
index 000000000..0b53ca265
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/chat/mod.rs
@@ -0,0 +1,6 @@
+pub mod api_stream;
+pub mod finish_reason;
+pub mod message;
+pub mod request;
+pub mod stream_chunk;
+pub mod tool;
diff --git a/crates/backend-remote/src/openai/bridging/chat/request.rs b/crates/backend-remote/src/openai/bridging/chat/request.rs
new file mode 100644
index 000000000..dfdefdec0
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/chat/request.rs
@@ -0,0 +1,70 @@
+use async_openai::types::chat::{
+ ChatCompletionStreamOptions, ChatCompletionToolChoiceOption, CreateChatCompletionRequest, ToolChoiceOptions,
+};
+use shoji::types::{
+ basic::{SamplingMethod, SamplingPolicy},
+ encoding::{Message, MessageList},
+ session::chat::StreamConfig,
+};
+
+use crate::openai::{
+ Error,
+ bridging::{chat, reasoning_effort},
+};
+
+pub fn build(
+ model: &str,
+ config: &StreamConfig,
+ messages: Vec<Message>,
+) -> Result<CreateChatCompletionRequest, Error> {
+ let completion_messages = messages.iter().map(chat::message::build).collect::<Result<Vec<_>, _>>()?;
+
+ let tools = messages
+ .tool_namespaces()
+ .into_iter()
+ .flat_map(|namespace| namespace.tools.into_iter())
+ .map(chat::tool::build)
+ .collect::<Result<Vec<_>, _>>()?;
+
+ let reasoning_effort = messages.reasoning_effort().map(reasoning_effort::build);
+
+ let mut request = CreateChatCompletionRequest {
+ messages: completion_messages,
+ model: model.to_string(),
+ stream: Some(true),
+ stream_options: Some(ChatCompletionStreamOptions {
+ include_usage: Some(true),
+ include_obfuscation: None,
+ }),
+ ..Default::default()
+ };
+ if !tools.is_empty() {
+ request.tools = Some(tools);
+ request.tool_choice = Some(ChatCompletionToolChoiceOption::Mode(ToolChoiceOptions::Auto));
+ }
+ if let Some(effort) = reasoning_effort {
+ request.reasoning_effort = Some(effort);
+ }
+ if let Some(token_limit) = config.token_limit {
+ request.max_completion_tokens = Some(token_limit as u32);
+ }
+ match &config.sampling_policy {
+ SamplingPolicy::Default {} => {},
+ SamplingPolicy::Custom {
+ method,
+ } => match method {
+ SamplingMethod::Greedy {} => {},
+ SamplingMethod::Stochastic {
+ temperature,
+ top_k: _,
+ top_p,
+ min_p: _,
+ } => {
+ request.temperature = temperature.map(|v| v as f32);
+ request.top_p = top_p.map(|v| v as f32);
+ },
+ },
+ }
+
+ Ok(request)
+}
diff --git a/crates/backend-remote/src/openai/bridging/chat/stream_chunk.rs b/crates/backend-remote/src/openai/bridging/chat/stream_chunk.rs
new file mode 100644
index 000000000..21304056d
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/chat/stream_chunk.rs
@@ -0,0 +1,104 @@
+use async_openai::types::chat::FinishReason;
+use serde::Deserialize;
+
+use crate::openai::{
+ bridging::chat::finish_reason,
+ stream_state::{StreamChunk, ToolCallChunk},
+};
+
+#[derive(Debug, Deserialize)]
+pub struct Response {
+ #[serde(default)]
+ choices: Vec<Choice>,
+ #[serde(default)]
+ usage: Option<Usage>,
+}
+
+#[derive(Debug, Deserialize)]
+struct Choice {
+ delta: Delta,
+ #[serde(default)]
+ finish_reason: Option<FinishReason>,
+}
+
+#[derive(Debug, Deserialize)]
+struct Delta {
+ #[serde(default)]
+ content: Option<String>,
+ #[serde(default)]
+ tool_calls: Option<Vec<ToolCall>>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ToolCall {
+ #[serde(default)]
+ index: Option<u32>,
+ #[serde(default)]
+ id: Option<String>,
+ #[serde(default)]
+ function: Option<FunctionCallStream>,
+}
+
+#[derive(Debug, Deserialize)]
+struct FunctionCallStream {
+ #[serde(default)]
+ name: Option<String>,
+ #[serde(default)]
+ arguments: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+struct Usage {
+ #[serde(default)]
+ prompt_tokens: Option<u32>,
+ #[serde(default)]
+ completion_tokens: Option<u32>,
+}
+
+pub fn build(response: Response) -> Option<StreamChunk> {
+ let usage = response.usage;
+ let choice = response.choices.into_iter().next();
+ let tokens_input = usage.as_ref().and_then(|usage| usage.prompt_tokens);
+ let tokens_output = usage.as_ref().and_then(|usage| usage.completion_tokens);
+
+ let (content, tool_calls, finish_reason) = match choice {
+ Some(choice) => {
+ let delta = choice.delta;
+ let tool_calls = delta
+ .tool_calls
+ .unwrap_or_default()
+ .into_iter()
+ .enumerate()
+ .map(|(position, call)| {
+ let (name, arguments) = call.function.map_or((None, None), |f| (f.name, f.arguments));
+ ToolCallChunk {
+ index: call.index.unwrap_or(position as u32),
+ id: call.id,
+ name,
+ arguments,
+ }
+ })
+ .collect();
+ (delta.content, tool_calls, choice.finish_reason.map(finish_reason::build))
+ },
+ None => (None, Vec::new(), None),
+ };
+
+ if content.is_none()
+ && tool_calls.is_empty()
+ && finish_reason.is_none()
+ && tokens_input.is_none()
+ && tokens_output.is_none()
+ {
+ return None;
+ }
+
+ Some(StreamChunk {
+ content,
+ reasoning: None,
+ tool_calls,
+ finish_reason,
+ tokens_input,
+ tokens_output,
+ })
+}
diff --git a/crates/backend-remote/src/openai/bridging/chat/tool.rs b/crates/backend-remote/src/openai/bridging/chat/tool.rs
new file mode 100644
index 000000000..555b1bfdd
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/chat/tool.rs
@@ -0,0 +1,26 @@
+use async_openai::types::chat::{ChatCompletionTool, ChatCompletionTools, FunctionObject};
+use shoji::types::encoding::ToolDescription;
+
+use crate::openai::Error;
+
+pub fn build(description: ToolDescription) -> Result<ChatCompletionTools, Error> {
+ let ToolDescription::Function {
+ function,
+ } = description;
+ let parameters = match function.parameters {
+ Some(value) => {
+ Some(serde_json::from_str::<serde_json::Value>(&value.json).map_err(|error| Error::Serialization {
+ message: error.to_string(),
+ })?)
+ },
+ None => None,
+ };
+ Ok(ChatCompletionTools::Function(ChatCompletionTool {
+ function: FunctionObject {
+ name: function.name,
+ description: Some(function.description),
+ parameters,
+ strict: None,
+ },
+ }))
+}
diff --git a/crates/backend-remote/src/openai/bridging/mod.rs b/crates/backend-remote/src/openai/bridging/mod.rs
new file mode 100644
index 000000000..0af2cf9d7
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/mod.rs
@@ -0,0 +1,24 @@
+pub mod chat;
+pub mod reasoning_effort;
+pub mod responses;
+
+use std::{pin::Pin, sync::Arc};
+
+use async_openai::{Client, config::OpenAIConfig};
+use futures::Stream;
+use shoji::{
+ traits::backend::{Error as BackendError, chat_message::Output},
+ types::{encoding::Message, session::chat::StreamConfig},
+};
+use tokio_util::sync::CancellationToken;
+
+pub trait ApiStream: Send + Sync {
+ fn stream(
+ &self,
+ client: Arc<Client<OpenAIConfig>>,
+ model: String,
+ config: StreamConfig,
+ messages: Vec<Message>,
+ cancel: CancellationToken,
+ ) -> Pin<Box<dyn Stream<Item = Result<Output, BackendError>> + Send>>;
+}
diff --git a/crates/backend-remote/src/openai/bridging/reasoning_effort.rs b/crates/backend-remote/src/openai/bridging/reasoning_effort.rs
new file mode 100644
index 000000000..bbaa5d96d
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/reasoning_effort.rs
@@ -0,0 +1,12 @@
+use async_openai::types::chat::ReasoningEffort as OpenAIReasoningEffort;
+use shoji::types::encoding::ReasoningEffort;
+
+pub fn build(effort: ReasoningEffort) -> OpenAIReasoningEffort {
+ match effort {
+ ReasoningEffort::Disabled => OpenAIReasoningEffort::None,
+ ReasoningEffort::Default => OpenAIReasoningEffort::Medium,
+ ReasoningEffort::Low => OpenAIReasoningEffort::Low,
+ ReasoningEffort::Medium => OpenAIReasoningEffort::Medium,
+ ReasoningEffort::High => OpenAIReasoningEffort::High,
+ }
+}
diff --git a/crates/backend-remote/src/openai/bridging/responses/api_stream.rs b/crates/backend-remote/src/openai/bridging/responses/api_stream.rs
new file mode 100644
index 000000000..1b7ab79b5
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/responses/api_stream.rs
@@ -0,0 +1,66 @@
+use std::{pin::Pin, sync::Arc};
+
+use async_openai::{Client, config::OpenAIConfig};
+use futures::{FutureExt, Stream, StreamExt, stream};
+use shoji::{
+ traits::backend::{Error as BackendError, chat_message::Output},
+ types::{encoding::Message, session::chat::StreamConfig},
+};
+use tokio_util::sync::CancellationToken;
+
+use crate::openai::{
+ Error,
+ bridging::{
+ self,
+ responses::{request, stream_chunk},
+ },
+ stream_state::StreamState,
+};
+
+pub struct ApiStream;
+
+impl bridging::ApiStream for ApiStream {
+ fn stream(
+ &self,
+ client: Arc<Client<OpenAIConfig>>,
+ model: String,
+ config: StreamConfig,
+ messages: Vec<Message>,
+ cancel: CancellationToken,
+ ) -> Pin<Box<dyn Stream<Item = Result<Output, BackendError>> + Send>> {
+ let stream = async move {
+ let request = match request::build(&model, &config, messages) {
+ Ok(request) => request,
+ Err(error) => {
+ return stream::once(async move { Err(Box::new(error) as BackendError) }).boxed();
+ },
+ };
+ match client.responses().create_stream(request).await {
+ Ok(upstream) => {
+ let mut state = StreamState::new();
+ upstream
+ .filter_map(move |event| {
+ let item = match event {
+ Ok(event) => {
+ stream_chunk::build(event).and_then(|chunk| state.process_chunk(chunk)).map(Ok)
+ },
+ Err(error) => Some(Err(Box::new(Error::Network {
+ message: error.to_string(),
+ }) as BackendError)),
+ };
+ futures::future::ready(item)
+ })
+ .boxed()
+ },
+ Err(error) => stream::once(async move {
+ Err(Box::new(Error::Network {
+ message: error.to_string(),
+ }) as BackendError)
+ })
+ .boxed(),
+ }
+ };
+
+ stream.flatten_stream().take_until(cancel.cancelled_owned()).boxed()
+ }
+}
diff --git a/crates/backend-remote/src/openai/bridging/responses/finish_reason.rs b/crates/backend-remote/src/openai/bridging/responses/finish_reason.rs
new file mode 100644
index 000000000..564137639
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/responses/finish_reason.rs
@@ -0,0 +1,10 @@
+use async_openai::types::responses::Status;
+use shoji::types::session::chat::FinishReason;
+
+pub fn build(status: &Status) -> Option<FinishReason> {
+ match status {
+ Status::Completed | Status::Failed => Some(FinishReason::Stop),
+ Status::Cancelled => Some(FinishReason::Cancelled),
+ _ => None,
+ }
+}
diff --git a/crates/backend-remote/src/openai/bridging/responses/message.rs b/crates/backend-remote/src/openai/bridging/responses/message.rs
new file mode 100644
index 000000000..da95db218
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/responses/message.rs
@@ -0,0 +1,60 @@
+use async_openai::types::responses::{
+ EasyInputContent, EasyInputMessage, FunctionCallOutput, FunctionCallOutputItemParam, FunctionToolCall, InputItem,
+ Item, MessageType, Role as ResponseRole,
+};
+use shoji::types::encoding::{Message, Role};
+
+use crate::openai::Error;
+
+pub fn build(message: &Message) -> Result<Vec<InputItem>, Error> {
+ let mut items = Vec::new();
+ match message.role {
+ Role::System {} | Role::Developer {} | Role::User {} | Role::Assistant {} => {
+ let role = match message.role {
+ Role::System {} => ResponseRole::System,
+ Role::Developer {} => ResponseRole::Developer,
+ Role::User {} => ResponseRole::User,
+ Role::Assistant {} => ResponseRole::Assistant,
+ _ => return Err(Error::UnsupportedRole),
+ };
+ let text = message.text();
+ if !text.is_empty() {
+ items.push(InputItem::EasyMessage(EasyInputMessage {
+ r#type: MessageType::default(),
+ role,
+ content: EasyInputContent::Text(text),
+ phase: None,
+ }));
+ }
+ if matches!(message.role, Role::Assistant {}) {
+ for tool_call in message.tool_calls() {
+ items.push(InputItem::Item(Item::FunctionCall(FunctionToolCall {
+ arguments: tool_call.arguments.json.clone(),
+ call_id: tool_call.identifier.unwrap_or_default(),
+ namespace: None,
+ name: tool_call.name,
+ id: None,
+ status: None,
+ })));
+ }
+ }
+ },
+ Role::Tool {} => {
+ for (call_id, _name, value) in message.tool_call_results() {
+ let content = serde_json::to_string(&value).map_err(|error| Error::Serialization {
+ message: error.to_string(),
+ })?;
+ items.push(InputItem::Item(Item::FunctionCallOutput(FunctionCallOutputItemParam {
+ call_id: call_id.unwrap_or_default(),
+ output: FunctionCallOutput::Text(content),
+ id: None,
+ status: None,
+ })));
+ }
+ },
+ Role::Custom {
+ ..
+ } => return Err(Error::UnsupportedRole),
+ }
+ Ok(items)
+}
diff --git a/crates/backend-remote/src/openai/bridging/responses/mod.rs b/crates/backend-remote/src/openai/bridging/responses/mod.rs
new file mode 100644
index 000000000..0b53ca265
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/responses/mod.rs
@@ -0,0 +1,6 @@
+pub mod api_stream;
+pub mod finish_reason;
+pub mod message;
+pub mod request;
+pub mod stream_chunk;
+pub mod tool;
diff --git a/crates/backend-remote/src/openai/bridging/responses/request.rs b/crates/backend-remote/src/openai/bridging/responses/request.rs
new file mode 100644
index 000000000..44bf6533e
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/responses/request.rs
@@ -0,0 +1,71 @@
+use async_openai::types::responses::{
+ CreateResponse, InputParam, Reasoning, ReasoningSummary, Tool as ResponseTool, ToolChoiceOptions, ToolChoiceParam,
+};
+use shoji::types::{
+ basic::{SamplingMethod, SamplingPolicy},
+ encoding::{Message, MessageList},
+ session::chat::StreamConfig,
+};
+
+use crate::openai::{
+ Error,
+ bridging::{reasoning_effort, responses},
+};
+
+pub fn build(
+ model: &str,
+ config: &StreamConfig,
+ messages: Vec<Message>,
+) -> Result<CreateResponse, Error> {
+ let input_items = messages
+ .iter()
+ .map(responses::message::build)
+ .collect::<Result<Vec<_>, _>>()?
+ .into_iter()
+ .flatten()
+ .collect::<Vec<_>>();
+
+ let tools: Vec<ResponseTool> = messages
+ .tool_namespaces()
+ .into_iter()
+ .flat_map(|namespace| namespace.tools.into_iter())
+ .map(responses::tool::build)
+ .collect::<Result<Vec<_>, _>>()?;
+
+ let reasoning = messages.reasoning_effort().map(|effort| Reasoning {
+ effort: Some(reasoning_effort::build(effort)),
+ summary: Some(ReasoningSummary::Auto),
+ });
+
+ let (temperature, top_p) = match &config.sampling_policy {
+ SamplingPolicy::Default {} => (None, None),
+ SamplingPolicy::Custom {
+ method,
+ } => match method {
+ SamplingMethod::Greedy {} => (None, None),
+ SamplingMethod::Stochastic {
+ temperature,
+ top_k: _,
+ top_p,
+ min_p: _,
+ } => (temperature.map(|value| value as f32), top_p.map(|value| value as f32)),
+ },
+ };
+
+ let mut request = CreateResponse {
+ input: InputParam::Items(input_items),
+ model: Some(model.to_string()),
+ stream: Some(true),
+ max_output_tokens: config.token_limit.map(|value| value as u32),
+ temperature,
+ top_p,
+ reasoning,
+ ..Default::default()
+ };
+ if !tools.is_empty() {
+ request.tools = Some(tools);
+ request.tool_choice = Some(ToolChoiceParam::Mode(ToolChoiceOptions::Auto));
+ }
+
+ Ok(request)
+}
diff --git a/crates/backend-remote/src/openai/bridging/responses/stream_chunk.rs b/crates/backend-remote/src/openai/bridging/responses/stream_chunk.rs
new file mode 100644
index 000000000..d7357fbd7
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/responses/stream_chunk.rs
@@ -0,0 +1,63 @@
+use async_openai::types::responses::{OutputItem, ResponseStreamEvent};
+
+use crate::openai::{
+ bridging::responses::finish_reason,
+ stream_state::{StreamChunk, ToolCallChunk},
+};
+
+pub fn build(event: ResponseStreamEvent) -> Option<StreamChunk> {
+ match event {
+ ResponseStreamEvent::ResponseOutputTextDelta(event) => Some(StreamChunk {
+ content: Some(event.delta),
+ ..StreamChunk::default()
+ }),
+ ResponseStreamEvent::ResponseReasoningTextDelta(event) => Some(StreamChunk {
+ reasoning: Some(event.delta),
+ ..StreamChunk::default()
+ }),
+ ResponseStreamEvent::ResponseReasoningSummaryTextDelta(event) => Some(StreamChunk {
+ reasoning: Some(event.delta),
+ ..StreamChunk::default()
+ }),
+ ResponseStreamEvent::ResponseOutputItemAdded(event) => match event.item {
+ OutputItem::FunctionCall(call) => Some(StreamChunk {
+ tool_calls: vec![ToolCallChunk {
+ index: event.output_index,
+ id: Some(call.call_id),
+ name: Some(call.name),
+ arguments: None,
+ }],
+ ..StreamChunk::default()
+ }),
+ _ => None,
+ },
+ ResponseStreamEvent::ResponseFunctionCallArgumentsDelta(event) => Some(StreamChunk {
+ tool_calls: vec![ToolCallChunk {
+ index: event.output_index,
+ id: None,
+ name: None,
+ arguments: Some(event.delta),
+ }],
+ ..StreamChunk::default()
+ }),
+ ResponseStreamEvent::ResponseCompleted(event) => Some(StreamChunk {
+ finish_reason: finish_reason::build(&event.response.status),
+ tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),
+ tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),
+ ..StreamChunk::default()
+ }),
+ ResponseStreamEvent::ResponseFailed(event) => Some(StreamChunk {
+ finish_reason: finish_reason::build(&event.response.status),
+ tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),
+ tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),
+ ..StreamChunk::default()
+ }),
+ ResponseStreamEvent::ResponseIncomplete(event) => Some(StreamChunk {
+ finish_reason: finish_reason::build(&event.response.status),
+ tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),
+ tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),
+ ..StreamChunk::default()
+ }),
+ _ => None,
+ }
+}
diff --git a/crates/backend-remote/src/openai/bridging/responses/tool.rs b/crates/backend-remote/src/openai/bridging/responses/tool.rs
new file mode 100644
index 000000000..a068cd9e6
--- /dev/null
+++ b/crates/backend-remote/src/openai/bridging/responses/tool.rs
@@ -0,0 +1,25 @@
+use async_openai::types::responses::{FunctionTool, Tool};
+use shoji::types::encoding::ToolDescription;
+
+use crate::openai::Error;
+
+pub fn build(description: ToolDescription) -> Result<Tool, Error> {
+ let ToolDescription::Function {
+ function,
+ } = description;
+ let parameters = match function.parameters {
+ Some(value) => {
+ Some(serde_json::from_str::<serde_json::Value>(&value.json).map_err(|error| Error::Serialization {
+ message: error.to_string(),
+ })?)
+ },
+ None => None,
+ };
+ Ok(Tool::Function(FunctionTool {
+ name: function.name,
+ parameters,
+ strict: None,
+ description: Some(function.description),
+ defer_loading: None,
+ }))
+}
diff --git a/crates/backend-remote/src/openai/config.rs b/crates/backend-remote/src/openai/config.rs
new file mode 100644
index 000000000..69d856b4b
--- /dev/null
+++ b/crates/backend-remote/src/openai/config.rs
@@ -0,0 +1,20 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ApiType {
+ Completions,
+ Responses,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct Config {
+ pub identifier: String,
+ pub api_endpoint: String,
+ pub api_key: Option<String>,
+ pub api_type: ApiType,
+ pub headers: Option<HashMap<String, String>>,
+}
diff --git a/crates/backend-remote/src/openai/error.rs b/crates/backend-remote/src/openai/error.rs
new file mode 100644
index 000000000..16c62ba98
--- /dev/null
+++ b/crates/backend-remote/src/openai/error.rs
@@ -0,0 +1,19 @@
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error("Unable to create: {message}")]
+ UnableToCreate {
+ message: String,
+ },
+ #[error("Network error: {message}")]
+ Network {
+ message: String,
+ },
+ #[error("Serialization error: {message}")]
+ Serialization {
+ message: String,
+ },
+ #[error("Unsupported role")]
+ UnsupportedRole,
+ #[error("Tool call result is required")]
+ ToolCallResultRequired,
+}
diff --git a/crates/backend-remote/src/openai/instance.rs b/crates/backend-remote/src/openai/instance.rs
new file mode 100644
index 000000000..12be33fe7
--- /dev/null
+++ b/crates/backend-remote/src/openai/instance.rs
@@ -0,0 +1,73 @@
+use std::{pin::Pin, sync::Arc};
+
+use async_openai::{Client, config::OpenAIConfig};
+use futures::Stream;
+use shoji::{
+ traits::{
+ Instance as InstanceTrait, State as StateTrait,
+ backend::{
+ Error as BackendError,
+ chat_message::{StreamInput, StreamOutput},
+ },
+ },
+ types::session::chat::StreamConfig,
+};
+use tokio_util::sync::CancellationToken;
+
+use crate::openai::{
+ ApiType,
+ bridging::{self, ApiStream},
+};
+
+#[derive(Debug, Clone)]
+pub struct State;
+
+impl StateTrait for State {
+ fn clone_boxed(&self) -> Box<dyn StateTrait> {
+ Box::new(self.clone())
+ }
+}
+
+pub struct Instance {
+ client: Arc<Client<OpenAIConfig>>,
+ model_identifier: String,
+ api_stream: Box<dyn ApiStream>,
+}
+
+impl Instance {
+ pub fn new(
+ client: Arc<Client<OpenAIConfig>>,
+ model_identifier: String,
+ api_type: ApiType,
+ ) -> Self {
+ let api_stream: Box<dyn ApiStream> = match api_type {
+ ApiType::Completions => Box::new(bridging::chat::api_stream::ApiStream),
+ ApiType::Responses => Box::new(bridging::responses::api_stream::ApiStream),
+ };
+ Self {
+ client,
+ model_identifier,
+ api_stream,
+ }
+ }
+}
+
+impl InstanceTrait for Instance {
+ type StreamConfig = StreamConfig;
+ type StreamInput = StreamInput;
+ type StreamOutput = StreamOutput;
+
+ fn state(&self) -> Pin<Box<dyn Future<Output = Result<Box<dyn StateTrait>, BackendError>> + Send + '_>> {
+ Box::pin(async move { Ok(Box::new(State) as Box<dyn StateTrait>) })
+ }
+
+ fn stream<'a>(
+ &'a self,
+ input: &'a Self::StreamInput,
+ _state: &'a mut dyn StateTrait,
+ config: Self::StreamConfig,
+ cancel: CancellationToken,
+ ) -> Pin<Box<dyn Stream<Item = Result<Self::StreamOutput, BackendError>> + Send + 'a>> {
+ self.api_stream.stream(self.client.clone(), self.model_identifier.clone(), config, input.clone(), cancel)
+ }
+}
diff --git a/crates/backend-remote/src/openai/mod.rs b/crates/backend-remote/src/openai/mod.rs
new file mode 100644
index 000000000..da932284a
--- /dev/null
+++ b/crates/backend-remote/src/openai/mod.rs
@@ -0,0 +1,79 @@
+mod bridging;
+mod config;
+mod error;
+mod instance;
+mod stream_state;
+mod tool_call_state;
+
+use std::{pin::Pin, sync::Arc};
+
+use async_openai::{Client, config::OpenAIConfig};
+pub use config::{ApiType, Config};
+pub use error::Error;
+pub use instance::Instance;
+use shoji::{
+ traits::{
+ Backend as BackendTrait,
+ backend::{Error as BackendError, chat_message},
+ },
+ types::session::chat::Config as ChatConfig,
+};
+
+pub struct Backend {
+ config: Config,
+ client: Arc<Client<OpenAIConfig>>,
+}
+
+impl Backend {
+ pub fn new(config: Config) -> Result<Self, Error> {
+ let mut openai_config = OpenAIConfig::new().with_api_base(&config.api_endpoint);
+ if let Some(api_key) = config.api_key.as_ref() {
+ openai_config = openai_config.with_api_key(api_key);
+ }
+ if let Some(headers) = config.headers.as_ref() {
+ for (key, value) in headers {
+ let name =
+ reqwest::header::HeaderName::from_bytes(key.as_bytes()).map_err(|error| Error::UnableToCreate {
+ message: error.to_string(),
+ })?;
+ openai_config =
+ openai_config.with_header(name, value.as_str()).map_err(|error| Error::UnableToCreate {
+ message: error.to_string(),
+ })?;
+ }
+ }
+
+ Ok(Self {
+ config,
+ client: Arc::new(Client::with_config(openai_config)),
+ })
+ }
+}
+
+impl BackendTrait for Backend {
+ fn identifier(&self) -> String {
+ self.config.identifier.clone()
+ }
+
+ fn version(&self) -> String {
+ "default".to_string()
+ }
+
+ fn as_chat_via_message_capable(&self) -> Option<&dyn chat_message::Backend> {
+ Some(self)
+ }
+}
+
+impl chat_message::Backend for Backend {
+ fn instance(
+ &self,
+ reference: String,
+ _config: ChatConfig,
+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn chat_message::Instance>, BackendError>> + Send + '_>> {
+ let client = self.client.clone();
+ let api_type = self.config.api_type.clone();
+ Box::pin(
+ async move { Ok(Box::new(Instance::new(client, reference, api_type)) as Box<dyn chat_message::Instance>) },
+ )
+ }
+}
diff --git a/crates/backend-remote/src/openai/stream_state.rs b/crates/backend-remote/src/openai/stream_state.rs
new file mode 100644
index 000000000..b1cf8bc82
--- /dev/null
+++ b/crates/backend-remote/src/openai/stream_state.rs
@@ -0,0 +1,143 @@
+use std::time::Instant;
+
+use indexmap::IndexMap;
+use shoji::{
+ traits::backend::chat_message::Output,
+ types::session::chat::{FinishReason, Stats},
+};
+
+use crate::openai::tool_call_state::ToolCallState;
+
+#[derive(Debug)]
+pub struct ToolCallChunk {
+ pub index: u32,
+ pub id: Option<String>,
+ pub name: Option<String>,
+ pub arguments: Option<String>,
+}
+
+#[derive(Debug, Default)]
+pub struct StreamChunk {
+ pub content: Option<String>,
+ pub reasoning: Option<String>,
+ pub tool_calls: Vec<ToolCallChunk>,
+ pub finish_reason: Option<FinishReason>,
+ pub tokens_input: Option<u32>,
+ pub tokens_output: Option<u32>,
+}
+
+#[derive(Debug)]
+pub struct StreamState {
+ text: Option<String>,
+ reasoning: Option<String>,
+ tool_calls: IndexMap<u32, ToolCallState>,
+ finish_reason: Option<FinishReason>,
+ start_moment: Instant,
+ first_token_moment: Option<Instant>,
+ tokens_input: Option<u32>,
+ tokens_output: Option<u32>,
+}
+
+impl StreamState {
+ pub fn new() -> Self {
+ Self {
+ text: None,
+ reasoning: None,
+ tool_calls: IndexMap::new(),
+ finish_reason: None,
+ start_moment: Instant::now(),
+ first_token_moment: None,
+ tokens_input: None,
+ tokens_output: None,
+ }
+ }
+
+ pub fn process_chunk(
+ &mut self,
+ chunk: StreamChunk,
+ ) -> Option<Output> {
+ let mut processed = false;
+ let produced_output = chunk.content.is_some() || chunk.reasoning.is_some() || !chunk.tool_calls.is_empty();
+
+ if let Some(content) = chunk.content {
+ self.text.get_or_insert_with(String::new).push_str(&content);
+ processed = true;
+ }
+
+ if let Some(reasoning) = chunk.reasoning {
+ self.reasoning.get_or_insert_with(String::new).push_str(&reasoning);
+ processed = true;
+ }
+
+ if !chunk.tool_calls.is_empty() {
+ for call in chunk.tool_calls {
+ self.tool_calls.entry(call.index).or_default().merge(call);
+ }
+ processed = true;
+ }
+
+ if produced_output && self.first_token_moment.is_none() {
+ self.first_token_moment = Some(Instant::now());
+ }
+
+ if let Some(tokens) = chunk.tokens_input {
+ self.tokens_input = Some(tokens);
+ processed = true;
+ }
+
+ if let Some(tokens) = chunk.tokens_output {
+ self.tokens_output = Some(tokens);
+ processed = true;
+ }
+
+ if let Some(reason) = chunk.finish_reason {
+ self.finish_reason = Some(reason);
+ processed = true;
+ }
+
+ if processed {
+ Some(self.build())
+ } else {
+ None
+ }
+ }
+
+ fn build(&self) -> Output {
+ let finalize = self.finish_reason.is_some();
+ Output {
+ reasoning: self.reasoning.clone(),
+ text: self.text.clone(),
+ tool_calls: self.tool_calls.values().map(|state| state.build(finalize)).collect(),
+ finish_reason: self.finish_reason.clone(),
+ stats: self.stats(),
+ }
+ }
+
+ fn stats(&self) -> Stats {
+ let duration = self.start_moment.elapsed().as_secs_f64();
+ let time_to_first_token =
+ self.first_token_moment.map(|first| first.duration_since(self.start_moment).as_secs_f64());
+ let prefill_tokens_per_second = match (self.tokens_input, time_to_first_token) {
+ (Some(tokens_input), Some(time_to_first_token)) if time_to_first_token > 0.0 => {
+ Some(tokens_input as f64 / time_to_first_token)
+ },
+ _ => None,
+ };
+ let generate_tokens_per_second = match (self.tokens_output, time_to_first_token) {
+ (Some(tokens_output), Some(time_to_first_token))
+ if (tokens_output > 0) && (duration > time_to_first_token) =>
+ {
+ Some((tokens_output - 1) as f64 / (duration - time_to_first_token))
+ },
+ _ => None,
+ };
+ Stats {
+ duration,
+ time_to_first_token,
+ prefill_tokens_per_second,
+ generate_tokens_per_second,
+ tokens_count_input: self.tokens_input,
+ tokens_count_output: self.tokens_output,
+ }
+ }
+}
diff --git a/crates/backend-remote/src/openai/tool_call_state.rs b/crates/backend-remote/src/openai/tool_call_state.rs
new file mode 100644
index 000000000..ad3f557b2
--- /dev/null
+++ b/crates/backend-remote/src/openai/tool_call_state.rs
@@ -0,0 +1,48 @@
+use shoji::{
+ traits::backend::chat_message::ToolCallState as OutputToolCallState,
+ types::{basic::Value, encoding::ToolCall},
+};
+
+use crate::openai::stream_state::ToolCallChunk;
+
+#[derive(Debug, Default)]
+pub struct ToolCallState {
+ id: String,
+ name: String,
+ arguments: String,
+}
+
+impl ToolCallState {
+ pub fn merge(
+ &mut self,
+ chunk: ToolCallChunk,
+ ) {
+ if let Some(id) = chunk.id {
+ self.id.push_str(&id);
+ }
+ if let Some(name) = chunk.name {
+ self.name.push_str(&name);
+ }
+ if let Some(arguments) = chunk.arguments {
+ self.arguments.push_str(&arguments);
+ }
+ }
+
+ pub fn build(
+ &self,
+ finalize: bool,
+ ) -> OutputToolCallState {
+ if finalize {
+ if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(&self.arguments) {
+ return OutputToolCallState::Finished(ToolCall {
+ identifier: Some(self.id.clone()),
+ name: self.name.clone(),
+ arguments: Value {
+ json: parsed.to_string(),
+ },
+ });
+ }
+ }
+ OutputToolCallState::Candidate(self.arguments.clone())
+ }
+}
diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.toml
index 1874c0edc..9f61bf8f1 100644
--- a/crates/backend-uzu/Cargo.toml
+++ b/crates/backend-uzu/Cargo.toml
@@ -32,6 +32,8 @@ sysinfo.workspace = true
itertools.workspace = true
schemars.workspace = true
rustix.workspace = true
+futures.workspace = true
+tokio-util.workspace = true
# metal on macos requires objc2, bytesize, metal
[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]
diff --git a/crates/backend-uzu/src/inference_backend.rs b/crates/backend-uzu/src/inference_backend.rs
index ff12d9692..6d1bb5482 100644
--- a/crates/backend-uzu/src/inference_backend.rs
+++ b/crates/backend-uzu/src/inference_backend.rs
@@ -1,20 +1,46 @@
-use serde::{Deserialize, Serialize};
-use shoji::traits::BackendInstance as BackendInstanceTrait;
+use std::{
+ path::PathBuf,
+ pin::Pin,
+ sync::{Arc, Mutex},
+};
-use crate::TOOLCHAIN_VERSION;
+use futures::{Stream, stream};
+use shoji::{
+ traits::{
+ Backend as BackendTrait, Instance as InstanceTrait, State as StateTrait,
+ backend::{
+ Error as BackendError,
+ chat_token::{self, StreamInput, StreamOutput},
+ },
+ },
+ types::session::chat::{Config, StreamConfig},
+};
+use tokio_util::sync::CancellationToken;
-#[derive(Debug, thiserror::Error)]
-pub enum Error {}
+use crate::{
+ TOOLCHAIN_VERSION,
+ session::{ChatSession, config::DecodingConfig},
+};
-#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
-pub struct Config;
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error("Unable to load model")]
+ UnableToLoad,
+ #[error("Unable to create state")]
+ UnableToCreateState,
+ #[error("Stream failed")]
+ StreamFailed,
+}
pub struct Backend;
-impl BackendInstanceTrait for Backend {
- type Error = Error;
- type Config = Config;
+impl Backend {
+ pub fn new() -> Result<Self, Error> {
+ Ok(Self)
+ }
+}
+impl BackendTrait for Backend {
fn identifier(&self) -> String {
"uzu".to_string()
}
@@ -23,8 +49,67 @@ impl BackendInstanceTrait for Backend {
TOOLCHAIN_VERSION.to_string()
}
- #[allow(unused_variables)]
- fn new(config: Self::Config) -> Result<Self, Self::Error> {
- Ok(Self)
+ fn as_chat_via_token_capable(&self) -> Option<&dyn chat_token::Backend> {
+ Some(self)
+ }
+}
+
+impl chat_token::Backend for Backend {
+ fn instance(
+ &self,
+ reference: String,
+ _config: Config,
+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn chat_token::Instance>, BackendError>> + Send + '_>> {
+ Box::pin(async move {
+ let path = PathBuf::from(&reference);
+ let session = ChatSession::new(path, DecodingConfig::default())
+ .map_err(|_| Box::new(Error::UnableToLoad) as BackendError)?;
+ Ok(Box::new(Instance {
+ session: Arc::new(Mutex::new(session)),
+ }) as Box<dyn chat_token::Instance>)
+ })
+ }
+}
+
+pub struct Instance {
+ session: Arc<Mutex<ChatSession>>,
+}
+unsafe impl Send for Instance {}
+unsafe impl Sync for Instance {}
+
+#[derive(Debug, Clone)]
+pub struct State;
+
+impl StateTrait for State {
+ fn clone_boxed(&self) -> Box<dyn StateTrait> {
+ Box::new(self.clone())
+ }
+}
+
+impl InstanceTrait for Instance {
+ type StreamConfig = StreamConfig;
+ type StreamInput = StreamInput;
+ type StreamOutput = StreamOutput;
+
+ fn state(&self) -> Pin<Box<dyn Future<Output = Result<Box<dyn StateTrait>, BackendError>> + Send + '_>> {
+ let reset = self
+ .session
+ .lock()
+ .map_err(|_| Error::UnableToCreateState)
+ .and_then(|mut guard| guard.reset().map_err(|_| Error::UnableToCreateState));
+ Box::pin(async move {
+ reset.map_err(|error| Box::new(error) as BackendError)?;
+ Ok(Box::new(State) as Box<dyn StateTrait>)
+ })
+ }
+
+ fn stream<'a>(
+ &'a self,
+ _input: &'a Self::StreamInput,
+ _state: &'a mut dyn StateTrait,
+ _config: Self::StreamConfig,
+ _cancel_token: CancellationToken,
+ ) -> Pin<Box<dyn Stream<Item = Result<Self::StreamOutput, BackendError>> + Send + 'a>> {
+ Box::pin(stream::once(async { Err(Box::new(Error::StreamFailed) as BackendError) }))
}
}
diff --git a/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs b/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs
index f200c7271..c278e8b7d 100644
--- a/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs
+++ b/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs
@@ -11,7 +11,7 @@ use backend_uzu::{
types::{Input, Message, Output},
},
};
-use shoji::types::ClassificationFeature;
+use shoji::types::session::chat::Feature;
use test_tag::tag;
use tokenizers::Tokenizer;
@@ -53,7 +53,7 @@ fn test_text_session_with_prompt_lookup_speculator() {
fn test_text_session_with_fixed_speculator() {
let tokenizer = Tokenizer::from_file(get_test_model_path().join("tokenizer.json")).unwrap();
- let feature = ClassificationFeature {
+ let feature = Feature {
name: String::from("sentiment"),
values: vec!["Happy", "Sad", "Angry", "Fearful", "Surprised", "Disgusted"]
.into_iter()
diff --git a/crates/cli-storage/src/app.rs b/crates/cli-storage/src/app.rs
index 6d98832db..7ca987a93 100644
--- a/crates/cli-storage/src/app.rs
+++ b/crates/cli-storage/src/app.rs
@@ -3,7 +3,7 @@ use std::{collections::HashMap, sync::Arc};
use crossterm::event::KeyCode;
use futures_util::StreamExt;
use ratatui::widgets::ListState;
-use shoji::types::Model;
+use shoji::types::model::Model;
use tokio::{sync::Mutex as TokioMutex, task::JoinHandle};
use uzu::{engine::Engine, storage::types::DownloadState};
@@ -31,7 +31,8 @@ impl App {
engine: Arc<Engine>,
tokio_handle: tokio::runtime::Handle,
) -> Self {
- let models = engine.models().await.unwrap().into_iter().filter(|model| model.is_local()).collect::<Vec<_>>();
+ let models =
+ engine.models().await.unwrap().into_iter().filter(|model| model.is_downloadable()).collect::<Vec<_>>();
// Fetch initial state for all models
let mut models_with_state = HashMap::new();
diff --git a/crates/hanashi/src/chat/config.rs b/crates/hanashi/src/chat/config.rs
index a2fcbd402..a59c1deca 100644
--- a/crates/hanashi/src/chat/config.rs
+++ b/crates/hanashi/src/chat/config.rs
@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
-use shoji::types::Capabilities;
+use shoji::types::encoding::Capabilities;
use crate::chat::{Error, hanashi::Config as HanashiConfig, harmony::Config as HarmonyConfig};
diff --git a/crates/hanashi/src/chat/hanashi/config/mod.rs b/crates/hanashi/src/chat/hanashi/config/mod.rs
index 7b5ab509e..641270ce0 100644
--- a/crates/hanashi/src/chat/hanashi/config/mod.rs
+++ b/crates/hanashi/src/chat/hanashi/config/mod.rs
@@ -2,7 +2,7 @@ mod bundled;
mod tokens;
use serde::{Deserialize, Serialize};
-use shoji::types::{Capabilities, ContentBlockType, ReasoningEffort};
+use shoji::types::encoding::{Capabilities, ContentBlockType, ReasoningEffort};
use token_stream_parser::token_stream::TokenStreamParserConfig;
pub use tokens::TokensConfig;
diff --git a/crates/hanashi/src/chat/hanashi/config/tokens.rs b/crates/hanashi/src/chat/hanashi/config/tokens.rs
index 6ad3d76bf..bc7ad2075 100644
--- a/crates/hanashi/src/chat/hanashi/config/tokens.rs
+++ b/crates/hanashi/src/chat/hanashi/config/tokens.rs
@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
-use shoji::types::TokenId;
+use shoji::types::basic::TokenId;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TokensConfig {
diff --git a/crates/hanashi/src/chat/hanashi/messages/canonical/content_block.rs b/crates/hanashi/src/chat/hanashi/messages/canonical/content_block.rs
index 8225db393..d42a21de7 100644
--- a/crates/hanashi/src/chat/hanashi/messages/canonical/content_block.rs
+++ b/crates/hanashi/src/chat/hanashi/messages/canonical/content_block.rs
@@ -1,5 +1,5 @@
use serde_json::Value;
-use shoji::types::ContentBlockType;
+use shoji::types::encoding::ContentBlockType;
pub struct ContentBlock {
pub r#type: ContentBlockType,
diff --git a/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs b/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs
index 5a6ba48df..ea8c7a213 100644
--- a/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs
+++ b/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs
@@ -2,7 +2,7 @@ use std::collections::HashMap;
use indexmap::IndexMap;
use serde_json::Value;
-use shoji::types::{
+use shoji::types::encoding::{
ContentBlock as OriginalContentBlock, ContentBlockType, Message as OriginalMessage, Role, TranslationInput,
};
@@ -127,6 +127,7 @@ impl Message {
value,
} => serde_json::to_value(value.clone()),
OriginalContentBlock::ToolCallResult {
+ identifier: _,
name: _,
value,
} => serde_json::to_value(value.clone()),
diff --git a/crates/hanashi/src/chat/hanashi/messages/error.rs b/crates/hanashi/src/chat/hanashi/messages/error.rs
index c18f69f40..e8adfe05f 100644
--- a/crates/hanashi/src/chat/hanashi/messages/error.rs
+++ b/crates/hanashi/src/chat/hanashi/messages/error.rs
@@ -1,4 +1,4 @@
-use shoji::types::{ContentBlockType, Role};
+use shoji::types::encoding::{ContentBlockType, Role};
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
diff --git a/crates/hanashi/src/chat/hanashi/messages/rendered/config.rs b/crates/hanashi/src/chat/hanashi/messages/rendered/config.rs
index 5e7b1b148..c68c00f10 100644
--- a/crates/hanashi/src/chat/hanashi/messages/rendered/config.rs
+++ b/crates/hanashi/src/chat/hanashi/messages/rendered/config.rs
@@ -1,7 +1,7 @@
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use serde_json::Value;
-use shoji::types::ContentBlockType;
+use shoji::types::encoding::ContentBlockType;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Field {
diff --git a/crates/hanashi/src/chat/hanashi/messages/rendered/message.rs b/crates/hanashi/src/chat/hanashi/messages/rendered/message.rs
index bb9312144..9ae81f1b8 100644
--- a/crates/hanashi/src/chat/hanashi/messages/rendered/message.rs
+++ b/crates/hanashi/src/chat/hanashi/messages/rendered/message.rs
@@ -2,7 +2,7 @@ use std::collections::HashMap;
use indexmap::IndexMap;
use serde_json::{Map, Value};
-use shoji::types::{ContentBlockType, Message as OriginalMessage, Role};
+use shoji::types::encoding::{ContentBlockType, Message as OriginalMessage, Role};
use crate::chat::hanashi::messages::{
Error,
diff --git a/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs b/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs
index 153da9c78..13308a949 100644
--- a/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs
+++ b/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
-use shoji::types::{ContentBlock, Role, ToolCall};
+use shoji::types::encoding::{ContentBlock, Role, ToolCall};
use crate::chat::hanashi::messages::streamed::Section;
@@ -23,6 +23,7 @@ impl Content {
}],
Content::Value(value) => match role {
Role::Tool {} => vec![ContentBlock::ToolCallResult {
+ identifier: None,
name: None,
value: value.into(),
}],
diff --git a/crates/hanashi/src/chat/hanashi/messages/streamed/mod.rs b/crates/hanashi/src/chat/hanashi/messages/streamed/mod.rs
index 28ae48533..c3840a598 100644
--- a/crates/hanashi/src/chat/hanashi/messages/streamed/mod.rs
+++ b/crates/hanashi/src/chat/hanashi/messages/streamed/mod.rs
@@ -6,7 +6,7 @@ use std::{collections::HashMap, str::FromStr};
pub use content::Content;
pub use section::Section;
use serde::{Deserialize, Deserializer};
-use shoji::types::{Message as CommonMessage, Role};
+use shoji::types::encoding::{Message as CommonMessage, Role};
#[derive(Deserialize)]
pub struct Message {
diff --git a/crates/hanashi/src/chat/hanashi/mod.rs b/crates/hanashi/src/chat/hanashi/mod.rs
index 488ab0eff..4af218f0b 100644
--- a/crates/hanashi/src/chat/hanashi/mod.rs
+++ b/crates/hanashi/src/chat/hanashi/mod.rs
@@ -9,7 +9,10 @@ use std::path::PathBuf;
pub use config::Config;
pub use error::Error;
-use shoji::types::{ContentBlock, Message, Token, TokenId};
+use shoji::types::{
+ basic::{Token, TokenId},
+ encoding::{ContentBlock, Message},
+};
use token_stream_parser::{Parser as _, token_stream::TokenStreamParser};
use tokenizers::{Tokenizer, step_decode_stream};
diff --git a/crates/hanashi/src/chat/hanashi/ordering/config.rs b/crates/hanashi/src/chat/hanashi/ordering/config.rs
index 379658e5c..418cac9c0 100644
--- a/crates/hanashi/src/chat/hanashi/ordering/config.rs
+++ b/crates/hanashi/src/chat/hanashi/ordering/config.rs
@@ -1,6 +1,6 @@
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
-use shoji::types::Role;
+use shoji::types::encoding::Role;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Config {
diff --git a/crates/hanashi/src/chat/hanashi/ordering/error.rs b/crates/hanashi/src/chat/hanashi/ordering/error.rs
index 3cf8e2ffa..d79bf96fe 100644
--- a/crates/hanashi/src/chat/hanashi/ordering/error.rs
+++ b/crates/hanashi/src/chat/hanashi/ordering/error.rs
@@ -1,4 +1,4 @@
-use shoji::types::Role;
+use shoji::types::encoding::Role;
#[derive(Debug, thiserror::Error)]
pub enum Error {
diff --git a/crates/hanashi/src/chat/hanashi/ordering/validator.rs b/crates/hanashi/src/chat/hanashi/ordering/validator.rs
index 729012385..1058abe98 100644
--- a/crates/hanashi/src/chat/hanashi/ordering/validator.rs
+++ b/crates/hanashi/src/chat/hanashi/ordering/validator.rs
@@ -1,4 +1,4 @@
-use shoji::types::Role;
+use shoji::types::encoding::Role;
use crate::chat::hanashi::ordering::{Config, Error};
diff --git a/crates/hanashi/src/chat/hanashi/renderer/config/mod.rs b/crates/hanashi/src/chat/hanashi/renderer/config/mod.rs
index 32396622d..dc44f9bfd 100644
--- a/crates/hanashi/src/chat/hanashi/renderer/config/mod.rs
+++ b/crates/hanashi/src/chat/hanashi/renderer/config/mod.rs
@@ -5,7 +5,7 @@ use indexmap::IndexMap;
pub use jinja::JinjaConfig;
pub use jinja_function::JinjaFunction;
use serde::{Deserialize, Serialize};
-use shoji::types::{ContentBlockType, Role};
+use shoji::types::encoding::{ContentBlockType, Role};
use crate::chat::hanashi::messages::{
canonical::Config as CanonicalConfig,
diff --git a/crates/hanashi/src/chat/hanashi/renderer/mod.rs b/crates/hanashi/src/chat/hanashi/renderer/mod.rs
index 1830392fb..c7518396d 100644
--- a/crates/hanashi/src/chat/hanashi/renderer/mod.rs
+++ b/crates/hanashi/src/chat/hanashi/renderer/mod.rs
@@ -8,7 +8,7 @@ use functions::{raise_exception, strftime_now, to_json};
use indexmap::IndexMap;
use minijinja::Environment;
use minijinja_contrib::pycompat::unknown_method_callback;
-use shoji::types::{Message, Token};
+use shoji::types::{basic::Token, encoding::Message};
use crate::chat::hanashi::messages::rendered::Message as RenderedMessage;
diff --git a/crates/hanashi/src/chat/hanashi/token.rs b/crates/hanashi/src/chat/hanashi/token.rs
index 2eeb93014..4b98a905b 100644
--- a/crates/hanashi/src/chat/hanashi/token.rs
+++ b/crates/hanashi/src/chat/hanashi/token.rs
@@ -1,4 +1,4 @@
-use shoji::types::Token;
+use shoji::types::basic::Token;
use token_stream_parser::types::Token as ParserToken;
pub trait ToParserToken {
diff --git a/crates/hanashi/src/chat/harmony/bridging/message.rs b/crates/hanashi/src/chat/harmony/bridging/message.rs
index b25deafc9..3fb892529 100644
--- a/crates/hanashi/src/chat/harmony/bridging/message.rs
+++ b/crates/hanashi/src/chat/harmony/bridging/message.rs
@@ -4,7 +4,10 @@ use openai_harmony::chat::{
Author as ExternalAuthor, Content as ExternalContent, DeveloperContent as ExternalDeveloperContent,
Message as ExternalMessage, Role as ExternalRole, SystemContent as ExternalSystemContent, TextContent,
};
-use shoji::types::{ContentBlock, Message, ReasoningEffort, Role, ToolCall, ToolNamespace, Value};
+use shoji::types::{
+ basic::Value,
+ encoding::{ContentBlock, Message, ReasoningEffort, Role, ToolCall, ToolNamespace},
+};
use crate::chat::harmony::bridging::{Error, FromHarmony, ToHarmony};
@@ -248,6 +251,7 @@ pub fn bridge_messages_to_harmony(messages: &[Message]) -> Result<Vec<ExternalMe
let block = &message.content[0];
let ContentBlock::ToolCallResult {
+ identifier: _,
name,
value,
} = block
@@ -429,6 +433,7 @@ pub fn bridge_messages_from_harmony(messages: &[ExternalMessage]) -> Result<Vec<
Ok(json_value) => {
content.push(ContentBlock::ToolCall {
value: ToolCall {
+ identifier: None,
name: name.to_string(),
arguments: Value::from(json_value),
},
@@ -471,6 +476,7 @@ pub fn bridge_messages_from_harmony(messages: &[ExternalMessage]) -> Result<Vec<
result.push(Message {
role: Role::Tool {},
content: vec![ContentBlock::ToolCallResult {
+ identifier: None,
name: Some(name.to_string()),
value,
}],
diff --git a/crates/hanashi/src/chat/harmony/bridging/mod.rs b/crates/hanashi/src/chat/harmony/bridging/mod.rs
index 8b22da73f..c264fbb4d 100644
--- a/crates/hanashi/src/chat/harmony/bridging/mod.rs
+++ b/crates/hanashi/src/chat/harmony/bridging/mod.rs
@@ -5,7 +5,7 @@ mod role;
mod tool_namespace;
pub use message::{bridge_messages_from_harmony, bridge_messages_to_harmony};
-use shoji::types::{ContentBlockType, ReasoningEffort, Role};
+use shoji::types::encoding::{ContentBlockType, ReasoningEffort, Role};
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
diff --git a/crates/hanashi/src/chat/harmony/bridging/reasoning_effort.rs b/crates/hanashi/src/chat/harmony/bridging/reasoning_effort.rs
index 930f7403f..ef8187978 100644
--- a/crates/hanashi/src/chat/harmony/bridging/reasoning_effort.rs
+++ b/crates/hanashi/src/chat/harmony/bridging/reasoning_effort.rs
@@ -1,5 +1,5 @@
use openai_harmony::chat::ReasoningEffort as ExternalReasoningEffort;
-use shoji::types::ReasoningEffort;
+use shoji::types::encoding::ReasoningEffort;
use crate::chat::harmony::bridging::{Error, FromHarmony, ToHarmony};
diff --git a/crates/hanashi/src/chat/harmony/bridging/role.rs b/crates/hanashi/src/chat/harmony/bridging/role.rs
index 58485e84b..64b4fa908 100644
--- a/crates/hanashi/src/chat/harmony/bridging/role.rs
+++ b/crates/hanashi/src/chat/harmony/bridging/role.rs
@@ -1,5 +1,5 @@
use openai_harmony::chat::Role as ExternalRole;
-use shoji::types::Role;
+use shoji::types::encoding::Role;
use crate::chat::harmony::bridging::{Error, FromHarmony, ToHarmony};
diff --git a/crates/hanashi/src/chat/harmony/bridging/tool_namespace.rs b/crates/hanashi/src/chat/harmony/bridging/tool_namespace.rs
index b983ddefa..992fbdd70 100644
--- a/crates/hanashi/src/chat/harmony/bridging/tool_namespace.rs
+++ b/crates/hanashi/src/chat/harmony/bridging/tool_namespace.rs
@@ -1,7 +1,10 @@
use openai_harmony::chat::{
ToolDescription as ExternalToolDescription, ToolNamespaceConfig as ExternalToolNamespaceConfig,
};
-use shoji::types::{ToolDescription, ToolFunction, ToolNamespace, Value};
+use shoji::types::{
+ basic::Value,
+ encoding::{ToolDescription, ToolFunction, ToolNamespace},
+};
use crate::chat::harmony::bridging::{Error, FromHarmony, ToHarmony};
diff --git a/crates/hanashi/src/chat/harmony/config.rs b/crates/hanashi/src/chat/harmony/config.rs
index 314dba09d..3af85990f 100644
--- a/crates/hanashi/src/chat/harmony/config.rs
+++ b/crates/hanashi/src/chat/harmony/config.rs
@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
-use shoji::types::Capabilities;
+use shoji::types::encoding::Capabilities;
use crate::chat::harmony::EncodingName;
diff --git a/crates/hanashi/src/chat/harmony/mod.rs b/crates/hanashi/src/chat/harmony/mod.rs
index 1661a4080..760b56eeb 100644
--- a/crates/hanashi/src/chat/harmony/mod.rs
+++ b/crates/hanashi/src/chat/harmony/mod.rs
@@ -15,7 +15,10 @@ use openai_harmony::{
},
load_harmony_encoding,
};
-use shoji::types::{Message, Token, TokenId};
+use shoji::types::{
+ basic::{Token, TokenId},
+ encoding::Message,
+};
use crate::{
Encoding as EncodingTrait,
diff --git a/crates/hanashi/src/chat/mod.rs b/crates/hanashi/src/chat/mod.rs
index 39dad86df..9bbb9e398 100644
--- a/crates/hanashi/src/chat/mod.rs
+++ b/crates/hanashi/src/chat/mod.rs
@@ -8,7 +8,7 @@ mod state;
pub use config::Config;
pub use context::{Context, TokenizerLocation};
pub use error::Error;
-use shoji::types::{Message, TokenId};
+use shoji::types::{basic::TokenId, encoding::Message};
pub use state::{State, SynchronizationError, SynchronizationResult};
use crate::{
diff --git a/crates/hanashi/src/chat/state.rs b/crates/hanashi/src/chat/state.rs
index 8d3e1f7e9..837c88c00 100644
--- a/crates/hanashi/src/chat/state.rs
+++ b/crates/hanashi/src/chat/state.rs
@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
-use shoji::types::{Message, Token};
+use shoji::types::{basic::Token, encoding::Message};
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
diff --git a/crates/hanashi/tests/helpers/mod.rs b/crates/hanashi/tests/helpers/mod.rs
index f374d7870..4c90fca0e 100644
--- a/crates/hanashi/tests/helpers/mod.rs
+++ b/crates/hanashi/tests/helpers/mod.rs
@@ -3,7 +3,9 @@
use std::{collections::HashMap, path::PathBuf};
use serde::Deserialize;
-use shoji::types::{ContentBlock, Message, ReasoningEffort, Role, ToolDescription, ToolNamespace, TranslationInput};
+use shoji::types::encoding::{
+ ContentBlock, Message, ReasoningEffort, Role, ToolDescription, ToolNamespace, TranslationInput,
+};
use tokenizers::Tokenizer;
fn test_data_path() -> PathBuf {
diff --git a/crates/hanashi/tests/test_encoding.rs b/crates/hanashi/tests/test_encoding.rs
index f32d04702..dd63fb25d 100644
--- a/crates/hanashi/tests/test_encoding.rs
+++ b/crates/hanashi/tests/test_encoding.rs
@@ -14,7 +14,7 @@ use helpers::{
build_messages, load_registry, load_response_test_data, load_tokenizer, normalize_pattern, response_path,
tokenizer_directory,
};
-use shoji::types::{ContentBlock, Message, ReasoningEffort, Role};
+use shoji::types::encoding::{ContentBlock, Message, ReasoningEffort, Role};
fn print_warning(message: &str) {
println!("\x1b[33m{}\x1b[0m", message);
diff --git a/crates/nagare/Cargo.toml b/crates/nagare/Cargo.toml
index 3b8e049a1..42ed30bb2 100644
--- a/crates/nagare/Cargo.toml
+++ b/crates/nagare/Cargo.toml
@@ -15,3 +15,7 @@ serde_json.workspace = true
thiserror.workspace = true
indexmap.workspace = true
reqwest.workspace = true
+futures.workspace = true
+tokio-util.workspace = true
+tokio.workspace = true
+tokio-stream.workspace = true
diff --git a/crates/nagare/src/chat/error.rs b/crates/nagare/src/chat/error.rs
new file mode 100644
index 000000000..9894084b4
--- /dev/null
+++ b/crates/nagare/src/chat/error.rs
@@ -0,0 +1,12 @@
+#[derive(Debug, thiserror::Error)]
+#[non_exhaustive]
+pub enum Error {
+ #[error("Backend error: {message}")]
+ Backend {
+ message: String,
+ },
+ #[error("Unsupported model")]
+ UnsupportedModel,
+ #[error("Unable to perform operation in current state")]
+ UnableToPerformOperationInCurrentState,
+}
diff --git a/crates/nagare/src/chat/message.rs b/crates/nagare/src/chat/message.rs
new file mode 100644
index 000000000..42fb2bdc1
--- /dev/null
+++ b/crates/nagare/src/chat/message.rs
@@ -0,0 +1,63 @@
+use std::pin::Pin;
+
+use futures::{Stream, StreamExt};
+use shoji::{
+ traits::{
+ State,
+ backend::chat_message::{Backend, Instance, Output},
+ },
+ types::{
+ encoding::Message,
+ session::chat::{Config, StreamConfig},
+ },
+};
+use tokio_util::sync::CancellationToken;
+
+use super::Error;
+
+pub struct Session {
+ instance: Box<dyn Instance>,
+ state: Box<dyn State>,
+}
+
+impl Session {
+ pub async fn new(
+ backend: &dyn Backend,
+ config: Config,
+ reference: String,
+ ) -> Result<Self, Error> {
+ let instance = backend.instance(reference, config).await.map_err(|error| Error::Backend {
+ message: error.to_string(),
+ })?;
+ let state = instance.state().await.map_err(|error| Error::Backend {
+ message: error.to_string(),
+ })?;
+ Ok(Self {
+ instance,
+ state,
+ })
+ }
+
+ pub async fn reset(&mut self) -> Result<(), Error> {
+ self.state = self.instance.state().await.map_err(|error| Error::Backend {
+ message: error.to_string(),
+ })?;
+ Ok(())
+ }
+
+ pub fn stream<'a>(
+ &'a mut self,
+ input: &'a Vec<Message>,
+ config: StreamConfig,
+ cancel_token: CancellationToken,
+ ) -> Pin<Box<dyn Stream<Item = Result<Output, Error>> + Send + 'a>> {
+ self.instance
+ .stream(input, self.state.as_mut(), config, cancel_token)
+ .map(|event| {
+ event.map_err(|error| Error::Backend {
+ message: error.to_string(),
+ })
+ })
+ .boxed()
+ }
+}
diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rs
new file mode 100644
index 000000000..6d7d3accd
--- /dev/null
+++ b/crates/nagare/src/chat/mod.rs
@@ -0,0 +1,214 @@
+mod error;
+pub mod message;
+pub mod token;
+
+use std::{pin::Pin, sync::Arc};
+
+pub use error::Error;
+use futures::{Stream, StreamExt};
+use indexmap::IndexMap;
+use shoji::{
+ traits::{
+ Backend,
+ backend::chat_message::{Output as BackendOutput, ToolCallState},
+ },
+ types::{
+ basic::{CancellationToken, Value},
+ encoding::{Message, ToolCall},
+ model::{Model, Specialization},
+ session::chat::{Config, Output, StreamConfig},
+ },
+};
+use tokio::sync::{Mutex, mpsc};
+use tokio_stream::wrappers::UnboundedReceiverStream;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub enum State {
+ Idle,
+ Generation,
+ ToolCalling,
+ Resetting,
+}
+
+enum Instance {
+ Token(token::Session),
+ Message(message::Session),
+}
+
+pub struct Session {
+ instance: Arc<Mutex<Instance>>,
+ state: Arc<Mutex<State>>,
+ messages: Arc<Mutex<Vec<Message>>>,
+}
+
+impl Session {
+ pub async fn new(
+ backend: &dyn Backend,
+ config: Config,
+ model: Model,
+ path: Option<String>,
+ ) -> Result<Self, Error> {
+ if !model.specializations.contains(&Specialization::Chat) {
+ return Err(Error::UnsupportedModel);
+ }
+ let reference = path.unwrap_or_else(|| model.identifier.clone());
+
+ let instance = if let Some(token_backend) = backend.as_chat_via_token_capable() {
+ Instance::Token(token::Session::new(token_backend, config, reference).await?)
+ } else if let Some(message_backend) = backend.as_chat_via_message_capable() {
+ Instance::Message(message::Session::new(message_backend, config, reference).await?)
+ } else {
+ return Err(Error::UnsupportedModel);
+ };
+
+ Ok(Self {
+ instance: Arc::new(Mutex::new(instance)),
+ state: Arc::new(Mutex::new(State::Idle)),
+ messages: Arc::new(Mutex::new(Vec::new())),
+ })
+ }
+
+ pub async fn state(&self) -> State {
+ *self.state.lock().await
+ }
+
+ pub async fn messages(&self) -> Vec<Message> {
+ self.messages.lock().await.clone()
+ }
+
+ pub async fn reset(&self) -> Result<(), Error> {
+ {
+ let mut state = self.state.lock().await;
+ match *state {
+ State::Idle | State::ToolCalling => {
+ *state = State::Resetting;
+ },
+ State::Generation | State::Resetting => {
+ return Err(Error::UnableToPerformOperationInCurrentState);
+ },
+ }
+ }
+
+ let result = {
+ let mut guard = self.instance.lock().await;
+ match &mut *guard {
+ Instance::Token(session) => session.reset().await,
+ Instance::Message(session) => session.reset().await,
+ }
+ };
+
+ self.messages.lock().await.clear();
+ *self.state.lock().await = State::Idle;
+
+ result
+ }
+
+ pub fn stream(
+ &self,
+ input: Vec<Message>,
+ config: StreamConfig,
+ ) -> (Pin<Box<dyn Stream<Item = Result<Vec<Output>, Error>> + Send>>, CancellationToken) {
+ let cancel_token_to_return = CancellationToken::new();
+ let (sender, receiver) = mpsc::unbounded_channel::<Result<Vec<Output>, Error>>();
+
+ let instance = self.instance.clone();
+ let state = self.state.clone();
+ let messages = self.messages.clone();
+ let cancel_token = cancel_token_to_return.inner().clone();
+
+ tokio::spawn(async move {
+ {
+ let mut state = state.lock().await;
+ match *state {
+ State::Idle => {
+ *state = State::Generation;
+ },
+ State::Generation | State::Resetting | State::ToolCalling => {
+ let _ = sender.send(Err(Error::UnableToPerformOperationInCurrentState));
+ return;
+ },
+ }
+ drop(state);
+ }
+
+ let all_messages = {
+ let mut messages = messages.lock().await;
+ messages.extend(input);
+ messages.clone()
+ };
+
+ let mut outputs: IndexMap<u32, Output> = IndexMap::new();
+
+ let mut instance = instance.lock().await;
+ let mut stream = match &mut *instance {
+ Instance::Token(session) => session.stream(&all_messages, config, cancel_token),
+ Instance::Message(session) => session.stream(&all_messages, config, cancel_token),
+ };
+
+ let turn_index: u32 = 0;
+ while let Some(partial_output) = stream.next().await {
+ match partial_output {
+ Ok(backend_output) => {
+ let message = build_message(&backend_output);
+ let output = Output {
+ message: message.clone(),
+ stats: backend_output.stats.clone(),
+ finish_reason: backend_output.finish_reason.clone(),
+ };
+ let is_new = outputs.insert(turn_index, output).is_none();
+
+ {
+ let mut messages = messages.lock().await;
+ if is_new {
+ messages.push(message.clone());
+ } else if let Some(last) = messages.last_mut() {
+ *last = message.clone();
+ }
+ }
+
+ if sender.send(Ok(outputs.values().cloned().collect())).is_err() {
+ break;
+ }
+ },
+ Err(error) => {
+ let _ = sender.send(Err(error));
+ break;
+ },
+ }
+ }
+
+ drop(stream);
+ drop(instance);
+
+ {
+ let mut state = state.lock().await;
+ *state = State::Idle;
+ }
+ });
+
+ (Box::pin(UnboundedReceiverStream::new(receiver)), cancel_token_to_return)
+ }
+}
+
+fn build_message(output: &BackendOutput) -> Message {
+ let mut message = Message::assistant();
+ if let Some(reasoning) = &output.reasoning {
+ message = message.with_reasoning(reasoning.clone());
+ }
+ if let Some(text) = &output.text {
+ message = message.with_text(text.clone());
+ }
+ for tool_call in &output.tool_calls {
+ message = match tool_call {
+ ToolCallState::Candidate(candidate) => {
+ message.with_tool_call_candidate(Value::from(serde_json::Value::String(candidate.clone())))
+ },
+ ToolCallState::Finished(tool_call) => message.with_tool_call(ToolCall {
+ identifier: tool_call.identifier.clone(),
+ name: tool_call.name.clone(),
+ arguments: tool_call.arguments.clone(),
+ }),
+ };
+ }
+ message
+}
diff --git a/crates/nagare/src/chat/token.rs b/crates/nagare/src/chat/token.rs
new file mode 100644
index 000000000..a5e26be15
--- /dev/null
+++ b/crates/nagare/src/chat/token.rs
@@ -0,0 +1,62 @@
+use std::pin::Pin;
+
+use futures::{Stream, stream};
+use shoji::{
+ traits::{
+ State,
+ backend::{
+ chat_message::Output,
+ chat_token::{Backend, Instance},
+ },
+ },
+ types::{
+ encoding::Message,
+ session::chat::{Config, StreamConfig},
+ },
+};
+use tokio_util::sync::CancellationToken;
+
+use super::Error;
+
+pub struct Session {
+ _instance: Box<dyn Instance>,
+ _state: Box<dyn State>,
+}
+
+impl Session {
+ pub async fn new(
+ backend: &dyn Backend,
+ config: Config,
+ reference: String,
+ ) -> Result<Self, Error> {
+ let instance = backend.instance(reference, config).await.map_err(|error| Error::Backend {
+ message: error.to_string(),
+ })?;
+ let state = instance.state().await.map_err(|error| Error::Backend {
+ message: error.to_string(),
+ })?;
+ Ok(Self {
+ _instance: instance,
+ _state: state,
+ })
+ }
+
+ pub async fn reset(&mut self) -> Result<(), Error> {
+ Err(Error::Backend {
+ message: "token chat session reset not implemented".to_string(),
+ })
+ }
+
+ pub fn stream<'a>(
+ &'a mut self,
+ _input: &'a Vec<Message>,
+ _config: StreamConfig,
+ _cancel_token: CancellationToken,
+ ) -> Pin<Box<dyn Stream<Item = Result<Output, Error>> + Send + 'a>> {
+ Box::pin(stream::once(async {
+ Err(Error::Backend {
+ message: "Token chat session stream not implemented".to_string(),
+ })
+ }))
+ }
+}
diff --git a/crates/nagare/src/lib.rs b/crates/nagare/src/lib.rs
index e5fdf85ee..38dfdaeba 100644
--- a/crates/nagare/src/lib.rs
+++ b/crates/nagare/src/lib.rs
@@ -1 +1,2 @@
pub mod api;
+pub mod chat;
diff --git a/crates/shoji/Cargo.toml b/crates/shoji/Cargo.toml
index 7be0cd8e6..85b9b5a8f 100644
--- a/crates/shoji/Cargo.toml
+++ b/crates/shoji/Cargo.toml
@@ -34,6 +34,8 @@ serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
indexmap.workspace = true
+futures.workspace = true
+tokio-util.workspace = true
uniffi = { workspace = true, optional = true }
napi = { workspace = true, optional = true }
napi-derive = { workspace = true, optional = true }
diff --git a/crates/shoji/src/traits/backend.rs b/crates/shoji/src/traits/backend.rs
deleted file mode 100644
index c864f8a46..000000000
--- a/crates/shoji/src/traits/backend.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-use std::error::Error;
-
-use serde::{Serialize, de::DeserializeOwned};
-
-pub trait BackendInstance: Send + Sync {
- type Error: Error;
- type Config: Serialize + DeserializeOwned;
-
- fn identifier(&self) -> String;
- fn version(&self) -> String;
-
- fn new(config: Self::Config) -> Result<Self, Self::Error>
- where
- Self: Sized;
-}
-
-pub trait Backend: Send + Sync {
- fn identifier(&self) -> String;
- fn version(&self) -> String;
-}
-
-impl<B: BackendInstance> Backend for B {
- fn identifier(&self) -> String {
- self.identifier()
- }
-
- fn version(&self) -> String {
- self.version()
- }
-}
diff --git a/crates/shoji/src/traits/backend/chat_message.rs b/crates/shoji/src/traits/backend/chat_message.rs
new file mode 100644
index 000000000..a0e599d5e
--- /dev/null
+++ b/crates/shoji/src/traits/backend/chat_message.rs
@@ -0,0 +1,48 @@
+use std::pin::Pin;
+
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ traits::backend::{Error, Instance as InstanceTrait},
+ types::{
+ encoding::{Message, ToolCall},
+ session::chat::{Config, FinishReason, Stats, StreamConfig},
+ },
+};
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(tag = "state", rename_all = "snake_case")]
+pub enum ToolCallState {
+ Candidate(String),
+ Finished(ToolCall),
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
+pub struct Output {
+ pub reasoning: Option<String>,
+ pub text: Option<String>,
+ pub tool_calls: Vec<ToolCallState>,
+ pub finish_reason: Option<FinishReason>,
+ pub stats: Stats,
+}
+
+pub type StreamInput = Vec<Message>;
+pub type StreamOutput = Output;
+
+pub trait Backend: Send + Sync {
+ fn instance(
+ &self,
+ reference: String,
+ config: Config,
+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + '_>>;
+}
+
+pub trait Instance:
+ InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>
+{
+}
+
+impl<T> Instance for T where
+ T: InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>
+{
+}
diff --git a/crates/shoji/src/traits/backend/chat_token.rs b/crates/shoji/src/traits/backend/chat_token.rs
new file mode 100644
index 000000000..8b09279d3
--- /dev/null
+++ b/crates/shoji/src/traits/backend/chat_token.rs
@@ -0,0 +1,27 @@
+use std::pin::Pin;
+
+use crate::{
+ traits::backend::{Error, Instance as InstanceTrait},
+ types::session::chat::{Config, StreamConfig},
+};
+
+pub type StreamInput = Vec<u64>;
+pub type StreamOutput = u64;
+
+pub trait Backend: Send + Sync {
+ fn instance(
+ &self,
+ reference: String,
+ config: Config,
+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + '_>>;
+}
+
+pub trait Instance:
+ InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>
+{
+}
+
+impl<T> Instance for T where
+ T: InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>
+{
+}
diff --git a/crates/shoji/src/traits/backend/classification.rs b/crates/shoji/src/traits/backend/classification.rs
new file mode 100644
index 000000000..7762f3e15
--- /dev/null
+++ b/crates/shoji/src/traits/backend/classification.rs
@@ -0,0 +1,29 @@
+use std::pin::Pin;
+
+use crate::{
+ traits::backend::{Error, Instance as InstanceTrait},
+ types::{encoding::Message, session::classification::Output},
+};
+
+pub type Config = ();
+pub type StreamConfig = ();
+pub type StreamInput = Vec<Message>;
+pub type StreamOutput = Output;
+
+pub trait Backend: Send + Sync {
+ fn instance(
+ &self,
+ reference: String,
+ config: Config,
+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + '_>>;
+}
+
+pub trait Instance:
+ InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>
+{
+}
+
+impl<T> Instance for T where
+ T: InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>
+{
+}
diff --git a/crates/shoji/src/traits/backend/mod.rs b/crates/shoji/src/traits/backend/mod.rs
new file mode 100644
index 000000000..db41a35a3
--- /dev/null
+++ b/crates/shoji/src/traits/backend/mod.rs
@@ -0,0 +1,52 @@
+pub mod chat_message;
+pub mod chat_token;
+pub mod classification;
+pub mod text_to_speech;
+
+use std::pin::Pin;
+
+use futures::Stream;
+use tokio_util::sync::CancellationToken;
+
+pub type Error = Box<dyn std::error::Error + Send + Sync>;
+
+pub trait Backend: Send + Sync {
+ fn identifier(&self) -> String;
+ fn version(&self) -> String;
+
+ fn as_chat_via_token_capable(&self) -> Option<&dyn chat_token::Backend> {
+ None
+ }
+
+ fn as_chat_via_message_capable(&self) -> Option<&dyn chat_message::Backend> {
+ None
+ }
+
+ fn as_classification_capable(&self) -> Option<&dyn classification::Backend> {
+ None
+ }
+
+ fn as_text_to_speech_capable(&self) -> Option<&dyn text_to_speech::Backend> {
+ None
+ }
+}
+
+pub trait Instance: Send + Sync {
+ type StreamConfig;
+ type StreamInput;
+ type StreamOutput;
+
+ fn state(&self) -> Pin<Box<dyn Future<Output = Result<Box<dyn State>, Error>> + Send + '_>>;
+
+ fn stream<'a>(
+ &'a self,
+ input: &'a Self::StreamInput,
+ state: &'a mut dyn State,
+ config: Self::StreamConfig,
+ cancel_token: CancellationToken,
+ ) -> Pin<Box<dyn Stream<Item = Result<Self::StreamOutput, Error>> + Send + 'a>>;
+}
+
+pub trait State: Send + Sync + 'static {
+ fn clone_boxed(&self) -> Box<dyn State>;
+}
diff --git a/crates/shoji/src/traits/backend/text_to_speech.rs b/crates/shoji/src/traits/backend/text_to_speech.rs
new file mode 100644
index 000000000..d79213df4
--- /dev/null
+++ b/crates/shoji/src/traits/backend/text_to_speech.rs
@@ -0,0 +1,29 @@
+use std::pin::Pin;
+
+use crate::{
+ traits::backend::{Error, Instance as InstanceTrait},
+ types::{encoding::Message, session::text_to_speech::PcmBatch},
+};
+
+pub type Config = ();
+pub type StreamConfig = ();
+pub type StreamInput = Vec<Message>;
+pub type StreamOutput = PcmBatch;
+
+pub trait Backend: Send + Sync {
+ fn instance(
+ &self,
+ reference: String,
+ config: Config,
+ ) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + '_>>;
+}
+
+pub trait Instance:
+ InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>
+{
+}
+
+impl<T> Instance for T where
+ T: InstanceTrait<StreamConfig = StreamConfig, StreamInput = StreamInput, StreamOutput = StreamOutput>
+{
+}
diff --git a/crates/shoji/src/traits/mod.rs b/crates/shoji/src/traits/mod.rs
index bfc9cfba3..80dd54bd3 100644
--- a/crates/shoji/src/traits/mod.rs
+++ b/crates/shoji/src/traits/mod.rs
@@ -1,5 +1,5 @@
-mod backend;
+pub mod backend;
mod registry;
-pub use backend::{Backend, BackendInstance};
+pub use backend::{Backend, Instance, State};
pub use registry::Registry;
diff --git a/crates/shoji/src/traits/registry.rs b/crates/shoji/src/traits/registry.rs
index 9ba5a3192..57a3bbe44 100644
--- a/crates/shoji/src/traits/registry.rs
+++ b/crates/shoji/src/traits/registry.rs
@@ -1,6 +1,6 @@
use std::{error::Error, future::Future, pin::Pin};
-use crate::types::Model;
+use crate::types::model::Model;
pub trait Registry: Send + Sync {
type Error: Error;
diff --git a/crates/shoji/src/types/basic/cancellation_token.rs b/crates/shoji/src/types/basic/cancellation_token.rs
new file mode 100644
index 000000000..1f2b24688
--- /dev/null
+++ b/crates/shoji/src/types/basic/cancellation_token.rs
@@ -0,0 +1,52 @@
+use tokio_util::sync::CancellationToken as TokioCancellationToken;
+
+#[derive(Debug, Clone, Default)]
+pub struct CancellationToken {
+ inner: TokioCancellationToken,
+}
+
+impl CancellationToken {
+ pub fn new() -> Self {
+ Self {
+ inner: TokioCancellationToken::new(),
+ }
+ }
+
+ pub fn cancel(&self) {
+ self.inner.cancel();
+ }
+
+ pub fn is_cancelled(&self) -> bool {
+ self.inner.is_cancelled()
+ }
+
+ pub fn child(&self) -> Self {
+ Self {
+ inner: self.inner.child_token(),
+ }
+ }
+}
+
+impl CancellationToken {
+ pub fn inner(&self) -> &TokioCancellationToken {
+ &self.inner
+ }
+
+ pub fn into_inner(self) -> TokioCancellationToken {
+ self.inner
+ }
+}
+
+impl From<TokioCancellationToken> for CancellationToken {
+ fn from(inner: TokioCancellationToken) -> Self {
+ Self {
+ inner,
+ }
+ }
+}
+
+impl From<CancellationToken> for TokioCancellationToken {
+ fn from(token: CancellationToken) -> Self {
+ token.inner
+ }
+}
diff --git a/crates/shoji/src/types/basic/mod.rs b/crates/shoji/src/types/basic/mod.rs
index 9bfa2d446..738faef4e 100644
--- a/crates/shoji/src/types/basic/mod.rs
+++ b/crates/shoji/src/types/basic/mod.rs
@@ -1,5 +1,13 @@
+mod cancellation_token;
+mod sampling_method;
+mod sampling_policy;
+mod sampling_seed;
mod token;
mod value;
+pub use cancellation_token::CancellationToken;
+pub use sampling_method::SamplingMethod;
+pub use sampling_policy::SamplingPolicy;
+pub use sampling_seed::SamplingSeed;
pub use token::{Token, TokenId, TokenValue};
pub use value::Value;
diff --git a/crates/shoji/src/types/basic/sampling_method.rs b/crates/shoji/src/types/basic/sampling_method.rs
new file mode 100644
index 000000000..3ea9f933c
--- /dev/null
+++ b/crates/shoji/src/types/basic/sampling_method.rs
@@ -0,0 +1,13 @@
+use serde::{Deserialize, Serialize};
+
+#[bindings::export(Enum, name = "SamplingMethod")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum SamplingMethod {
+ Greedy {},
+ Stochastic {
+ temperature: Option<f64>,
+ top_k: Option<i64>,
+ top_p: Option<f64>,
+ min_p: Option<f64>,
+ },
+}
diff --git a/crates/shoji/src/types/basic/sampling_policy.rs b/crates/shoji/src/types/basic/sampling_policy.rs
new file mode 100644
index 000000000..d82a28a1a
--- /dev/null
+++ b/crates/shoji/src/types/basic/sampling_policy.rs
@@ -0,0 +1,18 @@
+use serde::{Deserialize, Serialize};
+
+use crate::types::basic::SamplingMethod;
+
+#[bindings::export(Enum, name = "SamplingPolicy")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum SamplingPolicy {
+ Default {},
+ Custom {
+ method: SamplingMethod,
+ },
+}
+
+impl Default for SamplingPolicy {
+ fn default() -> Self {
+ SamplingPolicy::Default {}
+ }
+}
diff --git a/crates/shoji/src/types/basic/sampling_seed.rs b/crates/shoji/src/types/basic/sampling_seed.rs
new file mode 100644
index 000000000..de03c47c0
--- /dev/null
+++ b/crates/shoji/src/types/basic/sampling_seed.rs
@@ -0,0 +1,16 @@
+use serde::{Deserialize, Serialize};
+
+#[bindings::export(Enum, name = "SamplingSeed")]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum SamplingSeed {
+ Default {},
+ Custom {
+ seed: i64,
+ },
+}
+
+impl Default for SamplingSeed {
+ fn default() -> Self {
+ SamplingSeed::Default {}
+ }
+}
diff --git a/crates/shoji/src/types/basic/token.rs b/crates/shoji/src/types/basic/token.rs
index 5311ebb8b..0ac0ea47b 100644
--- a/crates/shoji/src/types/basic/token.rs
+++ b/crates/shoji/src/types/basic/token.rs
@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
pub type TokenId = u32;
pub type TokenValue = String;
-#[bindings::export(Struct)]
+#[bindings::export(Struct, name = "Token")]
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct Token {
diff --git a/crates/shoji/src/types/basic/value.rs b/crates/shoji/src/types/basic/value.rs
index 2a42887cd..6bee593d2 100644
--- a/crates/shoji/src/types/basic/value.rs
+++ b/crates/shoji/src/types/basic/value.rs
@@ -2,7 +2,7 @@ use std::fmt;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
-#[bindings::export(Struct)]
+#[bindings::export(Struct, name = "Value")]
#[derive(Clone, PartialEq)]
pub struct Value {
pub json: String,
diff --git a/crates/shoji/src/types/chat/message.rs b/crates/shoji/src/types/chat/message.rs
deleted file mode 100644
index 743842b3b..000000000
--- a/crates/shoji/src/types/chat/message.rs
+++ /dev/null
@@ -1,14 +0,0 @@
-use std::collections::HashMap;
-
-use serde::{Deserialize, Serialize};
-
-use crate::types::{ContentBlock, Role, Value};
-
-#[bindings::export(Struct)]
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub struct Message {
- pub role: Role,
- pub content: Vec<ContentBlock>,
- pub metadata: HashMap<String, Value>,
-}
diff --git a/crates/shoji/src/types/chat/capabilities.rs b/crates/shoji/src/types/encoding/capabilities.rs
similarity index 100%
rename from crates/shoji/src/types/chat/capabilities.rs
rename to crates/shoji/src/types/encoding/capabilities.rs
diff --git a/crates/shoji/src/types/chat/content_block.rs b/crates/shoji/src/types/encoding/content_block.rs
similarity index 97%
rename from crates/shoji/src/types/chat/content_block.rs
rename to crates/shoji/src/types/encoding/content_block.rs
index b70bebc66..2085f0849 100644
--- a/crates/shoji/src/types/chat/content_block.rs
+++ b/crates/shoji/src/types/encoding/content_block.rs
@@ -5,7 +5,10 @@ use std::{
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
-use crate::types::{ReasoningEffort, ToolCall, ToolNamespace, TranslationInput, Value};
+use crate::types::{
+ basic::Value,
+ encoding::{ReasoningEffort, ToolCall, ToolNamespace, TranslationInput},
+};
#[bindings::export(Enum)]
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -147,6 +150,8 @@ pub enum ContentBlock {
value: Value,
},
ToolCallResult {
+ #[serde(rename = "id")]
+ identifier: Option<String>,
name: Option<String>,
value: Value,
},
diff --git a/crates/shoji/src/types/encoding/message.rs b/crates/shoji/src/types/encoding/message.rs
new file mode 100644
index 000000000..34c9d99a8
--- /dev/null
+++ b/crates/shoji/src/types/encoding/message.rs
@@ -0,0 +1,177 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use crate::types::{
+ basic::Value,
+ encoding::{ContentBlock, ReasoningEffort, Role, ToolCall, ToolNamespace},
+};
+
+#[bindings::export(Struct)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct Message {
+ pub role: Role,
+ pub content: Vec<ContentBlock>,
+ pub metadata: HashMap<String, Value>,
+}
+
+impl Message {
+ pub fn for_role(role: Role) -> Self {
+ Self {
+ role,
+ content: vec![],
+ metadata: HashMap::new(),
+ }
+ }
+
+ pub fn system() -> Self {
+ Self::for_role(Role::System {})
+ }
+
+ pub fn developer() -> Self {
+ Self::for_role(Role::Developer {})
+ }
+
+ pub fn user() -> Self {
+ Self::for_role(Role::User {})
+ }
+
+ pub fn assistant() -> Self {
+ Self::for_role(Role::Assistant {})
+ }
+
+ pub fn tool() -> Self {
+ Self::for_role(Role::Tool {})
+ }
+}
+
+impl Message {
+ fn with_block(
+ self,
+ block: ContentBlock,
+ ) -> Self {
+ let mut content = self.content;
+ content.push(block);
+ Self {
+ content,
+ ..self
+ }
+ }
+
+ pub fn with_reasoning_effort(
+ self,
+ reasoning_effort: ReasoningEffort,
+ ) -> Self {
+ self.with_block(ContentBlock::ReasoningEffort {
+ value: reasoning_effort,
+ })
+ }
+
+ pub fn with_tool_namespaces(
+ self,
+ tool_namespaces: Vec<ToolNamespace>,
+ ) -> Self {
+ self.with_block(ContentBlock::Tools {
+ namespaces: tool_namespaces,
+ })
+ }
+
+ pub fn with_text(
+ self,
+ text: String,
+ ) -> Self {
+ self.with_block(ContentBlock::Text {
+ value: text,
+ })
+ }
+
+ pub fn with_reasoning(
+ self,
+ reasoning: String,
+ ) -> Self {
+ self.with_block(ContentBlock::Reasoning {
+ value: reasoning,
+ })
+ }
+
+ pub fn with_tool_call(
+ self,
+ tool_call: ToolCall,
+ ) -> Self {
+ self.with_block(ContentBlock::ToolCall {
+ value: tool_call,
+ })
+ }
+
+ pub fn with_tool_call_candidate(
+ self,
+ value: Value,
+ ) -> Self {
+ self.with_block(ContentBlock::ToolCallCandidate {
+ value,
+ })
+ }
+}
+
+macro_rules! blocks_by_type {
+ ($self:expr, $variant:ident, $field:ident) => {
+ $self.content.iter().filter_map(|block| match block {
+ ContentBlock::$variant {
+ $field,
+ } => Some($field.clone()),
+ _ => None,
+ })
+ };
+}
+
+impl Message {
+ pub fn reasoning_effort(&self) -> Option<ReasoningEffort> {
+ blocks_by_type!(self, ReasoningEffort, value).next()
+ }
+
+ pub fn text(&self) -> String {
+ blocks_by_type!(self, Text, value).collect::<String>()
+ }
+
+ pub fn reasoning(&self) -> String {
+ blocks_by_type!(self, Reasoning, value).collect::<String>()
+ }
+
+ pub fn tool_namespaces(&self) -> Vec<ToolNamespace> {
+ blocks_by_type!(self, Tools, namespaces).flatten().collect()
+ }
+
+ pub fn tool_calls(&self) -> Vec<ToolCall> {
+ blocks_by_type!(self, ToolCall, value).collect()
+ }
+
+ pub fn tool_call_results(&self) -> Vec<(Option<String>, Option<String>, Value)> {
+ self.content
+ .iter()
+ .filter_map(|block| match block {
+ ContentBlock::ToolCallResult {
+ identifier,
+ name,
+ value,
+ } => Some((identifier.clone(), name.clone(), value.clone())),
+ _ => None,
+ })
+ .collect()
+ }
+}
+
+pub trait MessageList {
+ fn reasoning_effort(&self) -> Option<ReasoningEffort>;
+ fn tool_namespaces(&self) -> Vec<ToolNamespace>;
+}
+
+impl MessageList for Vec<Message> {
+ fn reasoning_effort(&self) -> Option<ReasoningEffort> {
+ self.iter().find_map(|message| message.reasoning_effort())
+ }
+
+ fn tool_namespaces(&self) -> Vec<ToolNamespace> {
+ self.iter().flat_map(|message| message.tool_namespaces()).collect()
+ }
+}
diff --git a/crates/shoji/src/types/chat/mod.rs b/crates/shoji/src/types/encoding/mod.rs
similarity index 92%
rename from crates/shoji/src/types/chat/mod.rs
rename to crates/shoji/src/types/encoding/mod.rs
index 32b790ae2..3814d6d42 100644
--- a/crates/shoji/src/types/chat/mod.rs
+++ b/crates/shoji/src/types/encoding/mod.rs
@@ -11,7 +11,7 @@ mod translation_input;
pub use capabilities::Capabilities;
pub use content_block::{ContentBlock, ContentBlockType};
-pub use message::Message;
+pub use message::{Message, MessageList};
pub use reasoning_effort::ReasoningEffort;
pub use role::Role;
pub use tool_call::ToolCall;
diff --git a/crates/shoji/src/types/chat/reasoning_effort.rs b/crates/shoji/src/types/encoding/reasoning_effort.rs
similarity index 100%
rename from crates/shoji/src/types/chat/reasoning_effort.rs
rename to crates/shoji/src/types/encoding/reasoning_effort.rs
diff --git a/crates/shoji/src/types/chat/role.rs b/crates/shoji/src/types/encoding/role.rs
similarity index 100%
rename from crates/shoji/src/types/chat/role.rs
rename to crates/shoji/src/types/encoding/role.rs
diff --git a/crates/shoji/src/types/chat/tool_call.rs b/crates/shoji/src/types/encoding/tool_call.rs
similarity index 70%
rename from crates/shoji/src/types/chat/tool_call.rs
rename to crates/shoji/src/types/encoding/tool_call.rs
index d0dd4519d..360355b27 100644
--- a/crates/shoji/src/types/chat/tool_call.rs
+++ b/crates/shoji/src/types/encoding/tool_call.rs
@@ -1,11 +1,13 @@
use serde::{Deserialize, Serialize};
-use crate::types::Value;
+use crate::types::basic::Value;
#[bindings::export(Struct)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct ToolCall {
+ #[serde(rename = "id")]
+ pub identifier: Option<String>,
pub name: String,
pub arguments: Value,
}
diff --git a/crates/shoji/src/types/chat/tool_description.rs b/crates/shoji/src/types/encoding/tool_description.rs
similarity index 85%
rename from crates/shoji/src/types/chat/tool_description.rs
rename to crates/shoji/src/types/encoding/tool_description.rs
index a2777cc20..71cb8c485 100644
--- a/crates/shoji/src/types/chat/tool_description.rs
+++ b/crates/shoji/src/types/encoding/tool_description.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::types::ToolFunction;
+use crate::types::encoding::ToolFunction;
#[bindings::export(Enum)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
diff --git a/crates/shoji/src/types/chat/tool_function.rs b/crates/shoji/src/types/encoding/tool_function.rs
similarity index 90%
rename from crates/shoji/src/types/chat/tool_function.rs
rename to crates/shoji/src/types/encoding/tool_function.rs
index 9f86aff28..bbddcfd20 100644
--- a/crates/shoji/src/types/chat/tool_function.rs
+++ b/crates/shoji/src/types/encoding/tool_function.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::types::Value;
+use crate::types::basic::Value;
#[bindings::export(Struct)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
diff --git a/crates/shoji/src/types/chat/tool_namespace.rs b/crates/shoji/src/types/encoding/tool_namespace.rs
similarity index 86%
rename from crates/shoji/src/types/chat/tool_namespace.rs
rename to crates/shoji/src/types/encoding/tool_namespace.rs
index 45868863f..1143571dc 100644
--- a/crates/shoji/src/types/chat/tool_namespace.rs
+++ b/crates/shoji/src/types/encoding/tool_namespace.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::types::ToolDescription;
+use crate::types::encoding::ToolDescription;
#[bindings::export(Struct)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
diff --git a/crates/shoji/src/types/chat/translation_input.rs b/crates/shoji/src/types/encoding/translation_input.rs
similarity index 100%
rename from crates/shoji/src/types/chat/translation_input.rs
rename to crates/shoji/src/types/encoding/translation_input.rs
diff --git a/crates/shoji/src/types/mod.rs b/crates/shoji/src/types/mod.rs
index 800f82e3c..34d1e0f50 100644
--- a/crates/shoji/src/types/mod.rs
+++ b/crates/shoji/src/types/mod.rs
@@ -1,9 +1,4 @@
-mod basic;
-mod chat;
-mod model;
-mod session;
-
-pub use basic::*;
-pub use chat::*;
-pub use model::*;
-pub use session::*;
+pub mod basic;
+pub mod encoding;
+pub mod model;
+pub mod session;
diff --git a/crates/shoji/src/types/model/accessibility.rs b/crates/shoji/src/types/model/accessibility.rs
index 4cd5372aa..b620faa0a 100644
--- a/crates/shoji/src/types/model/accessibility.rs
+++ b/crates/shoji/src/types/model/accessibility.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::types::{Reference, Repository};
+use crate::types::model::{Reference, Repository};
#[bindings::export(Enum)]
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
diff --git a/crates/shoji/src/types/model/entity.rs b/crates/shoji/src/types/model/entity.rs
index e4a9807c3..4a66341f3 100644
--- a/crates/shoji/src/types/model/entity.rs
+++ b/crates/shoji/src/types/model/entity.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::types::Image;
+use crate::types::model::Image;
#[bindings::export(Enum)]
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
diff --git a/crates/shoji/src/types/model/file.rs b/crates/shoji/src/types/model/file.rs
index 8ac7457d8..fd71a3c3a 100644
--- a/crates/shoji/src/types/model/file.rs
+++ b/crates/shoji/src/types/model/file.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::types::{Hash, HashMethod};
+use crate::types::model::{Hash, HashMethod};
#[bindings::export(Struct)]
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
diff --git a/crates/shoji/src/types/model/model.rs b/crates/shoji/src/types/model/model.rs
index 0e7ecd18a..a16d397f4 100644
--- a/crates/shoji/src/types/model/model.rs
+++ b/crates/shoji/src/types/model/model.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::types::{Accessibility, Entity, EntityType, Quantization, Reference, Specialization};
+use crate::types::model::{Accessibility, Entity, EntityType, Quantization, Reference, Specialization};
#[bindings::export(Struct)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -20,7 +20,7 @@ impl Model {
}
pub fn cache_identifier(&self) -> String {
- self.identifier.replace(":", "-")
+ self.identifier.replace(":", "-").replace("/", "-")
}
pub fn is_local(&self) -> bool {
@@ -31,6 +31,15 @@ impl Model {
matches!(self.accessibility, Accessibility::Remote { .. })
}
+ pub fn is_downloadable(&self) -> bool {
+ matches!(
+ self.accessibility,
+ Accessibility::Local {
+ reference: Reference::Mirai { .. } | Reference::HuggingFace { .. }
+ }
+ )
+ }
+
pub fn repo_ids(&self) -> Vec<String> {
match &self.accessibility {
Accessibility::Local {
@@ -54,6 +63,9 @@ impl Model {
Reference::HuggingFace {
repository,
} => vec![repository.identifier.clone()],
+ Reference::Local {
+ ..
+ } => vec![],
},
Accessibility::Remote {
repository,
@@ -68,12 +80,34 @@ impl Model {
}
}
- pub fn reference_type(&self) -> Option<String> {
+ pub fn local_external_path(&self) -> Option<String> {
match &self.accessibility {
Accessibility::Local {
reference,
..
- } => Some(reference.r#type()),
+ } => match reference {
+ Reference::Mirai {
+ ..
+ } => None,
+ Reference::HuggingFace {
+ ..
+ } => None,
+ Reference::Local {
+ path,
+ } => Some(path.clone()),
+ },
+ Accessibility::Remote {
+ ..
+ } => None,
+ }
+ }
+
+ pub fn reference_name(&self) -> Option<String> {
+ match &self.accessibility {
+ Accessibility::Local {
+ reference,
+ ..
+ } => Some(reference.name()),
Accessibility::Remote {
..
} => None,
@@ -93,6 +127,9 @@ impl Model {
Reference::HuggingFace {
repository,
} => Some(repository.commit_hash.clone()),
+ Reference::Local {
+ ..
+ } => None,
},
Accessibility::Remote {
..
diff --git a/crates/shoji/src/types/model/reference.rs b/crates/shoji/src/types/model/reference.rs
index eb3f99d5d..25daa58e9 100644
--- a/crates/shoji/src/types/model/reference.rs
+++ b/crates/shoji/src/types/model/reference.rs
@@ -1,31 +1,37 @@
use serde::{Deserialize, Serialize};
-use crate::types::{File, Repository};
+use crate::types::model::{File, Repository};
#[bindings::export(Enum)]
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum Reference {
- HuggingFace {
- repository: Repository,
- },
Mirai {
toolchain_version: String,
repository: Option<Repository>,
source_repository: Option<Repository>,
files: Vec<File>,
},
+ HuggingFace {
+ repository: Repository,
+ },
+ Local {
+ path: String,
+ },
}
impl Reference {
- pub fn r#type(&self) -> String {
+ pub fn name(&self) -> String {
match self {
+ Reference::Mirai {
+ ..
+ } => "mirai".to_string(),
Reference::HuggingFace {
..
} => "huggingface".to_string(),
- Reference::Mirai {
+ Reference::Local {
..
- } => "mirai".to_string(),
+ } => "local".to_string(),
}
}
}
diff --git a/crates/shoji/src/types/session/chat/config.rs b/crates/shoji/src/types/session/chat/config.rs
new file mode 100644
index 000000000..b8ea8d0ef
--- /dev/null
+++ b/crates/shoji/src/types/session/chat/config.rs
@@ -0,0 +1,60 @@
+use serde::{Deserialize, Serialize};
+
+use crate::types::{
+ basic::SamplingSeed,
+ session::chat::{ContextLength, SpeculationPreset},
+};
+
+#[bindings::export(Struct, name = "ChatConfig")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Config {
+ pub context_length: ContextLength,
+ pub sampling_seed: SamplingSeed,
+ pub speculation_preset: Option<SpeculationPreset>,
+}
+
+impl Default for Config {
+ fn default() -> Self {
+ Self {
+ context_length: ContextLength::default(),
+ sampling_seed: SamplingSeed::default(),
+ speculation_preset: None,
+ }
+ }
+}
+
+impl Config {
+ pub fn with_context_length(
+ self,
+ context_length: ContextLength,
+ ) -> Self {
+ Self {
+ context_length,
+ ..self
+ }
+ }
+}
+
+impl Config {
+ pub fn with_sampling_seed(
+ self,
+ sampling_seed: SamplingSeed,
+ ) -> Self {
+ Self {
+ sampling_seed,
+ ..self
+ }
+ }
+}
+
+impl Config {
+ pub fn with_speculation_preset(
+ self,
+ speculation_preset: Option<SpeculationPreset>,
+ ) -> Self {
+ Self {
+ speculation_preset,
+ ..self
+ }
+ }
+}
diff --git a/crates/shoji/src/types/session/chat/context_length.rs b/crates/shoji/src/types/session/chat/context_length.rs
new file mode 100644
index 000000000..b3b75e617
--- /dev/null
+++ b/crates/shoji/src/types/session/chat/context_length.rs
@@ -0,0 +1,17 @@
+use serde::{Deserialize, Serialize};
+
+#[bindings::export(Enum, name = "ChatContextLength")]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum ContextLength {
+ Default {},
+ Maximal {},
+ Custom {
+ length: i64,
+ },
+}
+
+impl Default for ContextLength {
+ fn default() -> Self {
+ ContextLength::Default {}
+ }
+}
diff --git a/crates/shoji/src/types/session/classification_feature.rs b/crates/shoji/src/types/session/chat/feature.rs
similarity index 69%
rename from crates/shoji/src/types/session/classification_feature.rs
rename to crates/shoji/src/types/session/chat/feature.rs
index f60698022..bd330d6fa 100644
--- a/crates/shoji/src/types/session/classification_feature.rs
+++ b/crates/shoji/src/types/session/chat/feature.rs
@@ -1,8 +1,8 @@
use serde::{Deserialize, Serialize};
-#[bindings::export(Struct)]
+#[bindings::export(Struct, name = "ChatFeature")]
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
-pub struct ClassificationFeature {
+pub struct Feature {
pub name: String,
pub values: Vec<String>,
}
diff --git a/crates/shoji/src/types/session/chat/finish_reason.rs b/crates/shoji/src/types/session/chat/finish_reason.rs
new file mode 100644
index 000000000..694cb1ed8
--- /dev/null
+++ b/crates/shoji/src/types/session/chat/finish_reason.rs
@@ -0,0 +1,12 @@
+use serde::{Deserialize, Serialize};
+
+#[bindings::export(Enum, name = "ChatFinishReason")]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum FinishReason {
+ Stop,
+ Length,
+ Cancelled,
+ ContextLimitReached,
+ ToolCalls,
+ Rejected,
+}
diff --git a/crates/shoji/src/types/session/chat/grammar.rs b/crates/shoji/src/types/session/chat/grammar.rs
new file mode 100644
index 000000000..7f7b060b3
--- /dev/null
+++ b/crates/shoji/src/types/session/chat/grammar.rs
@@ -0,0 +1,13 @@
+use serde::{Deserialize, Serialize};
+
+#[bindings::export(Enum, name = "ChatGrammar")]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum Grammar {
+ JsonAny {},
+ JsonSchema {
+ schema: String,
+ },
+ Regex {
+ pattern: String,
+ },
+}
diff --git a/crates/shoji/src/types/session/chat/mod.rs b/crates/shoji/src/types/session/chat/mod.rs
new file mode 100644
index 000000000..cfe4d3bea
--- /dev/null
+++ b/crates/shoji/src/types/session/chat/mod.rs
@@ -0,0 +1,19 @@
+mod config;
+mod context_length;
+mod feature;
+mod finish_reason;
+mod grammar;
+mod output;
+mod speculation_preset;
+mod stats;
+mod stream_config;
+
+pub use config::Config;
+pub use context_length::ContextLength;
+pub use feature::Feature;
+pub use finish_reason::FinishReason;
+pub use grammar::Grammar;
+pub use output::Output;
+pub use speculation_preset::SpeculationPreset;
+pub use stats::Stats;
+pub use stream_config::StreamConfig;
diff --git a/crates/shoji/src/types/session/chat/output.rs b/crates/shoji/src/types/session/chat/output.rs
new file mode 100644
index 000000000..ba6fecb32
--- /dev/null
+++ b/crates/shoji/src/types/session/chat/output.rs
@@ -0,0 +1,14 @@
+use serde::{Deserialize, Serialize};
+
+use crate::types::{
+ encoding::Message,
+ session::chat::{FinishReason, Stats},
+};
+
+#[bindings::export(Struct, name = "ChatOutput")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Output {
+ pub message: Message,
+ pub stats: Stats,
+ pub finish_reason: Option<FinishReason>,
+}
diff --git a/crates/shoji/src/types/session/chat/speculation_preset.rs b/crates/shoji/src/types/session/chat/speculation_preset.rs
new file mode 100644
index 000000000..9ec68696c
--- /dev/null
+++ b/crates/shoji/src/types/session/chat/speculation_preset.rs
@@ -0,0 +1,13 @@
+use serde::{Deserialize, Serialize};
+
+use crate::types::session::chat::Feature;
+
+#[bindings::export(Enum, name = "ChatSpeculationPreset")]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum SpeculationPreset {
+ GeneralChat {},
+ Summarization {},
+ Classification {
+ feature: Feature,
+ },
+}
diff --git a/crates/shoji/src/types/session/chat/stats.rs b/crates/shoji/src/types/session/chat/stats.rs
new file mode 100644
index 000000000..cb58ad58b
--- /dev/null
+++ b/crates/shoji/src/types/session/chat/stats.rs
@@ -0,0 +1,18 @@
+use serde::{Deserialize, Serialize};
+
+#[bindings::export(Struct, name = "ChatStats")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
+pub struct Stats {
+ pub duration: f64,
+ pub time_to_first_token: Option<f64>,
+ pub prefill_tokens_per_second: Option<f64>,
+ pub generate_tokens_per_second: Option<f64>,
+ pub tokens_count_input: Option<u32>,
+ pub tokens_count_output: Option<u32>,
+}
+
+impl Stats {
+ pub fn tokens_count(&self) -> Option<u32> {
+ self.tokens_count_input.and_then(|input| self.tokens_count_output.map(|output| input + output))
+ }
+}
diff --git a/crates/shoji/src/types/session/chat/stream_config.rs b/crates/shoji/src/types/session/chat/stream_config.rs
new file mode 100644
index 000000000..1fffed2b1
--- /dev/null
+++ b/crates/shoji/src/types/session/chat/stream_config.rs
@@ -0,0 +1,68 @@
+use serde::{Deserialize, Serialize};
+
+use crate::types::{
+ basic::{SamplingMethod, SamplingPolicy},
+ session::chat::Grammar,
+};
+
+#[bindings::export(Struct, name = "ChatStreamConfig")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct StreamConfig {
+ pub token_limit: Option<u32>,
+ pub sampling_policy: SamplingPolicy,
+ pub grammar: Option<Grammar>,
+}
+
+impl Default for StreamConfig {
+ fn default() -> Self {
+ Self {
+ token_limit: None,
+ sampling_policy: SamplingPolicy::default(),
+ grammar: None,
+ }
+ }
+}
+
+impl StreamConfig {
+ pub fn with_token_limit(
+ self,
+ token_limit: Option<u32>,
+ ) -> Self {
+ Self {
+ token_limit,
+ ..self
+ }
+ }
+
+ pub fn with_sampling_policy(
+ self,
+ sampling_policy: SamplingPolicy,
+ ) -> Self {
+ Self {
+ sampling_policy,
+ ..self
+ }
+ }
+
+ pub fn with_sampling_method(
+ self,
+ sampling_method: SamplingMethod,
+ ) -> Self {
+ Self {
+ sampling_policy: SamplingPolicy::Custom {
+ method: sampling_method,
+ },
+ ..self
+ }
+ }
+
+ pub fn with_grammar(
+ self,
+ grammar: Option<Grammar>,
+ ) -> Self {
+ Self {
+ grammar,
+ ..self
+ }
+ }
+}
diff --git a/crates/shoji/src/types/session/classification/label.rs b/crates/shoji/src/types/session/classification/label.rs
new file mode 100644
index 000000000..ed0c90437
--- /dev/null
+++ b/crates/shoji/src/types/session/classification/label.rs
@@ -0,0 +1,11 @@
+use serde::{Deserialize, Serialize};
+
+#[bindings::export(Struct)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Label {
+ pub index: i64,
+ pub label: String,
+ pub title: String,
+ pub description: String,
+ pub recommended_threshold: f64,
+}
diff --git a/crates/shoji/src/types/session/classification/mod.rs b/crates/shoji/src/types/session/classification/mod.rs
new file mode 100644
index 000000000..c0ea53c95
--- /dev/null
+++ b/crates/shoji/src/types/session/classification/mod.rs
@@ -0,0 +1,7 @@
+mod label;
+mod output;
+mod stats;
+
+pub use label::Label;
+pub use output::Output;
+pub use stats::Stats;
diff --git a/crates/shoji/src/types/session/classification/output.rs b/crates/shoji/src/types/session/classification/output.rs
new file mode 100644
index 000000000..712d53dde
--- /dev/null
+++ b/crates/shoji/src/types/session/classification/output.rs
@@ -0,0 +1,13 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use crate::types::session::classification::Stats;
+
+#[bindings::export(Struct, name = "ClassificationOutput")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Output {
+ pub logits: Vec<f64>,
+ pub probabilities: HashMap<String, f64>,
+ pub stats: Stats,
+}
diff --git a/crates/shoji/src/types/session/classification/stats.rs b/crates/shoji/src/types/session/classification/stats.rs
new file mode 100644
index 000000000..f5a7401cb
--- /dev/null
+++ b/crates/shoji/src/types/session/classification/stats.rs
@@ -0,0 +1,14 @@
+use serde::{Deserialize, Serialize};
+
+#[bindings::export(Struct, name = "ClassificationStats")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Stats {
+ pub preprocessing_duration: f64,
+ pub forward_pass_duration: f64,
+ pub postprocessing_duration: f64,
+ pub total_duration: f64,
+ pub tokens_count: i64,
+ pub tokens_per_second: f64,
+ pub predicted_label: String,
+ pub confidence: f64,
+}
diff --git a/crates/shoji/src/types/session/mod.rs b/crates/shoji/src/types/session/mod.rs
index 83eefede0..6527a7365 100644
--- a/crates/shoji/src/types/session/mod.rs
+++ b/crates/shoji/src/types/session/mod.rs
@@ -1,3 +1,3 @@
-mod classification_feature;
-
-pub use classification_feature::ClassificationFeature;
+pub mod chat;
+pub mod classification;
+pub mod text_to_speech;
diff --git a/crates/shoji/src/types/session/text_to_speech/mod.rs b/crates/shoji/src/types/session/text_to_speech/mod.rs
new file mode 100644
index 000000000..bfcd648a0
--- /dev/null
+++ b/crates/shoji/src/types/session/text_to_speech/mod.rs
@@ -0,0 +1,5 @@
+mod pcm_batch;
+mod stats;
+
+pub use pcm_batch::PcmBatch;
+pub use stats::Stats;
diff --git a/crates/shoji/src/types/session/text_to_speech/pcm_batch.rs b/crates/shoji/src/types/session/text_to_speech/pcm_batch.rs
new file mode 100644
index 000000000..c42564d96
--- /dev/null
+++ b/crates/shoji/src/types/session/text_to_speech/pcm_batch.rs
@@ -0,0 +1,10 @@
+use serde::{Deserialize, Serialize};
+
+#[bindings::export(Struct)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct PcmBatch {
+ pub samples: Vec<f64>,
+ pub sample_rate: u32,
+ pub channels: u32,
+ pub lengths: Vec<u32>,
+}
diff --git a/crates/shoji/src/types/session/text_to_speech/stats.rs b/crates/shoji/src/types/session/text_to_speech/stats.rs
new file mode 100644
index 000000000..3bb8efa76
--- /dev/null
+++ b/crates/shoji/src/types/session/text_to_speech/stats.rs
@@ -0,0 +1,19 @@
+use serde::{Deserialize, Serialize};
+
+#[bindings::export(Struct, name = "TextToSpeechStats")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Stats {
+ pub text_length: u32,
+ pub first_chunk_seconds: f64,
+ pub generation_duration: f64,
+ pub audio_duration: f64,
+}
+
+impl Stats {
+ pub fn real_time_factor(&self) -> f64 {
+ if self.generation_duration <= 0.0 {
+ return 0.0;
+ }
+ self.audio_duration / self.generation_duration
+ }
+}
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 121f1147b..967f8cedb 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -51,6 +51,7 @@ wasm-bindgen-futures = { workspace = true, optional = true }
tsify = { workspace = true, optional = true }
[target.'cfg(not(target_family = "wasm"))'.dependencies]
+backend-remote.workspace = true
nagare.workspace = true
download-manager.workspace = true
serde.workspace = true
@@ -68,8 +69,8 @@ tracing-subscriber.workspace = true
tracing-appender.workspace = true
uuid.workspace = true
futures-util.workspace = true
-openai-api-rs.workspace = true
-regex.workspace = true
+async-openai.workspace = true
+fancy-regex.workspace = true
[target.'cfg(target_vendor = "apple")'.dependencies]
objc2.workspace = true
diff --git a/crates/uzu/src/engine/config.rs b/crates/uzu/src/engine/config.rs
index 4eeaf207c..ac9927576 100644
--- a/crates/uzu/src/engine/config.rs
+++ b/crates/uzu/src/engine/config.rs
@@ -7,10 +7,13 @@ use serde::{Deserialize, Serialize};
#[serde(rename_all = "snake_case")]
pub struct Config {
pub mirai_api_key: Option<String>,
+ pub lalamo_path: Option<String>,
pub huggingface_api_key: Option<String>,
pub openai_api_key: Option<String>,
- pub baseten_api_key: Option<String>,
pub anthropic_api_key: Option<String>,
+ pub gemini_api_key: Option<String>,
+ pub xai_api_key: Option<String>,
+ pub baseten_api_key: Option<String>,
pub openrouter_api_key: Option<String>,
}
@@ -18,10 +21,13 @@ impl Default for Config {
fn default() -> Self {
Self {
mirai_api_key: env::var("MIRAI_API_KEY").ok(),
+ lalamo_path: env::var("LALAMO_PATH").ok(),
huggingface_api_key: env::var("HF_TOKEN").ok(),
openai_api_key: env::var("OPENAI_API_KEY").ok(),
- baseten_api_key: env::var("BASETEN_API_KEY").ok(),
anthropic_api_key: env::var("ANTHROPIC_API_KEY").ok(),
+ gemini_api_key: env::var("GEMINI_API_KEY").ok(),
+ xai_api_key: env::var("XAI_API_KEY").ok(),
+ baseten_api_key: env::var("BASETEN_API_KEY").ok(),
openrouter_api_key: env::var("OPENROUTER_API_KEY").ok(),
}
}
@@ -36,6 +42,14 @@ impl Config {
self
}
+ pub fn with_lalamo_path(
+ mut self,
+ lalamo_path: String,
+ ) -> Self {
+ self.lalamo_path = Some(lalamo_path);
+ self
+ }
+
pub fn with_huggingface_api_key(
mut self,
huggingface_api_key: String,
@@ -52,19 +66,35 @@ impl Config {
self
}
- pub fn with_baseten_api_key(
+ pub fn with_anthropic_api_key(
mut self,
- baseten_api_key: String,
+ anthropic_api_key: String,
) -> Self {
- self.baseten_api_key = Some(baseten_api_key);
+ self.anthropic_api_key = Some(anthropic_api_key);
self
}
- pub fn with_anthropic_api_key(
+ pub fn with_gemini_api_key(
mut self,
- anthropic_api_key: String,
+ gemini_api_key: String,
) -> Self {
- self.anthropic_api_key = Some(anthropic_api_key);
+ self.gemini_api_key = Some(gemini_api_key);
+ self
+ }
+
+ pub fn with_xai_api_key(
+ mut self,
+ xai_api_key: String,
+ ) -> Self {
+ self.xai_api_key = Some(xai_api_key);
+ self
+ }
+
+ pub fn with_baseten_api_key(
+ mut self,
+ baseten_api_key: String,
+ ) -> Self {
+ self.baseten_api_key = Some(baseten_api_key);
self
}
diff --git a/crates/uzu/src/engine/error.rs b/crates/uzu/src/engine/error.rs
index 2f283f9ac..466beea22 100644
--- a/crates/uzu/src/engine/error.rs
+++ b/crates/uzu/src/engine/error.rs
@@ -1,3 +1,5 @@
+use nagare::chat::Error as ChatError;
+
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
@@ -11,8 +13,10 @@ pub enum Error {
Storage(#[from] crate::storage::Error),
#[error(transparent)]
Registry(#[from] crate::registry::Error),
- #[error("Backend error: {message}")]
- Backend {
- message: String,
- },
+ #[error("Unable to create backend")]
+ UnableToCreateBackend,
+ #[error("Backend not found")]
+ BackendNotFound,
+ #[error(transparent)]
+ Chat(#[from] ChatError),
}
diff --git a/crates/uzu/src/engine/mod.rs b/crates/uzu/src/engine/mod.rs
index aea1d399c..ebb32d55f 100644
--- a/crates/uzu/src/engine/mod.rs
+++ b/crates/uzu/src/engine/mod.rs
@@ -2,15 +2,17 @@ mod config;
mod downloader;
mod error;
-use std::collections::HashMap;
+use std::{collections::HashMap, sync::Arc};
-use backend_uzu::inference_backend::{Backend as UzuBackend, Config as UzuBackendConfig};
+use backend_remote::openai::Backend as OpenAIBackend;
+use backend_uzu::inference_backend::Backend as UzuBackend;
pub use config::Config;
pub use downloader::Downloader;
pub use error::Error;
+use nagare::chat::Session as ChatSession;
use shoji::{
- traits::{Backend, BackendInstance, Registry},
- types::Model,
+ traits::{Backend, Registry},
+ types::{model::Model, session::chat::Config as ChatConfig},
};
use tokio::runtime::Handle;
use tokio_stream::wrappers::BroadcastStream;
@@ -21,16 +23,20 @@ use crate::{
logs,
registry::{
CachedRegistry, Error as RegistryError, MergedRegistry,
+ local::{Config as LocalRegistryConfig, Registry as LocalRegistry},
mirai::{Backend as MiraiBackend, Config as MiraiRegistryConfig, Registry as MiraiRegistry},
openai::{Config as OpenAIConfig, Registry as OpenAIRegistry},
},
- storage::{Config as StorageConfig, Storage, types::DownloadState},
+ storage::{
+ Config as StorageConfig, Storage,
+ types::{DownloadPhase, DownloadState},
+ },
};
pub struct Engine {
registry: SharedAccess<MergedRegistry>,
storage: SharedAccess<Storage>,
- backends: HashMap<String, Box<dyn Backend>>,
+ backends: HashMap<String, Arc<dyn Backend>>,
}
impl Engine {
@@ -46,55 +52,74 @@ impl Engine {
let storage = SharedAccess::new(Storage::new(tokio_handle.clone(), storage_config).await?);
- let uzu_backend_config = UzuBackendConfig {};
- let uzu_backend: Box<dyn Backend> =
- Box::new(UzuBackend::new(uzu_backend_config).map_err(|error| Error::Backend {
- message: error.to_string(),
- })?);
-
- let mirai_registry_config = MiraiRegistryConfig {
- api_key: config.mirai_api_key,
- device: device.clone(),
- backends: vec![MiraiBackend {
- identifier: uzu_backend.identifier(),
- version: uzu_backend.version(),
- }],
- include_traces: false,
- };
- let mirai_registry = Box::new(MiraiRegistry::new(mirai_registry_config)?);
-
let mut engine = Self {
storage,
registry,
backends: HashMap::new(),
};
- engine.add_backend(uzu_backend);
- engine.add_registry(mirai_registry).await?;
- if let Some(openai_api_key) = config.openai_api_key {
- let openai_registry = OpenAIRegistry::new(OpenAIConfig::openai(openai_api_key))?;
- engine.add_registry(Box::new(openai_registry)).await?;
- }
+ {
+ let uzu_backend = UzuBackend::new().map_err(|_| Error::UnableToCreateBackend)?;
+ let uzu_backend_identifier = uzu_backend.identifier();
+ let uzu_backend_version = uzu_backend.version();
- let ollama_config = OpenAIConfig::ollama();
- if is_endpoint_reachable(&ollama_config.api_endpoint).await {
- let ollama_registry = OpenAIRegistry::new(ollama_config)?;
- engine.add_registry(Box::new(ollama_registry)).await?;
- }
+ let mirai_registry_config = MiraiRegistryConfig {
+ api_key: config.mirai_api_key,
+ device: device.clone(),
+ backends: vec![MiraiBackend {
+ identifier: uzu_backend_identifier.clone(),
+ version: uzu_backend_version,
+ }],
+ include_traces: false,
+ };
+ let mirai_registry = Box::new(MiraiRegistry::new(mirai_registry_config)?);
- if let Some(baseten_api_key) = config.baseten_api_key {
- let baseten_registry = OpenAIRegistry::new(OpenAIConfig::baseten(baseten_api_key))?;
- engine.add_registry(Box::new(baseten_registry)).await?;
+ engine.add_backend(Arc::new(uzu_backend) as Arc<dyn Backend>);
+ engine.add_registry(mirai_registry).await?;
+
+ if let Some(lalamo_path) = config.lalamo_path {
+ let lalamo_registry =
+ LocalRegistry::new(LocalRegistryConfig::lalamo(uzu_backend_identifier, lalamo_path))?;
+ engine.add_registry(Box::new(lalamo_registry)).await?;
+ }
}
+ let mut openai_configs: Vec<OpenAIConfig> = vec![];
+ {
+ let ollama_config = OpenAIConfig::ollama();
+ if is_endpoint_reachable(&ollama_config.api_endpoint).await {
+ openai_configs.push(ollama_config);
+ }
+ }
+ {
+ let lmstudio_config = OpenAIConfig::lmstudio();
+ if is_endpoint_reachable(&lmstudio_config.api_endpoint).await {
+ openai_configs.push(lmstudio_config);
+ }
+ }
+ if let Some(openai_api_key) = config.openai_api_key {
+ openai_configs.push(OpenAIConfig::openai(openai_api_key));
+ }
if let Some(anthropic_api_key) = config.anthropic_api_key {
- let anthropic_registry = OpenAIRegistry::new(OpenAIConfig::anthropic(anthropic_api_key))?;
- engine.add_registry(Box::new(anthropic_registry)).await?;
+ openai_configs.push(OpenAIConfig::anthropic(anthropic_api_key));
+ }
+ if let Some(gemini_api_key) = config.gemini_api_key {
+ openai_configs.push(OpenAIConfig::gemini(gemini_api_key));
+ }
+ if let Some(xai_api_key) = config.xai_api_key {
+ openai_configs.push(OpenAIConfig::xai(xai_api_key));
+ }
+ if let Some(baseten_api_key) = config.baseten_api_key {
+ openai_configs.push(OpenAIConfig::baseten(baseten_api_key));
}
-
if let Some(openrouter_api_key) = config.openrouter_api_key {
- let openrouter_registry = OpenAIRegistry::new(OpenAIConfig::openrouter(openrouter_api_key))?;
- engine.add_registry(Box::new(openrouter_registry)).await?;
+ openai_configs.push(OpenAIConfig::openrouter(openrouter_api_key));
+ }
+ for config in openai_configs {
+ let registry = OpenAIRegistry::new(config.clone())?;
+ let backend = OpenAIBackend::new(config.into()).map_err(|_| Error::UnableToCreateBackend)?;
+ engine.add_registry(Box::new(registry)).await?;
+ engine.add_backend(Arc::new(backend) as Arc<dyn Backend>);
}
Ok(engine)
@@ -124,7 +149,7 @@ impl Engine {
impl Engine {
pub fn add_backend(
&mut self,
- backend: Box<dyn Backend>,
+ backend: Arc<dyn Backend>,
) {
self.backends.insert(backend.identifier(), backend);
}
@@ -142,6 +167,19 @@ impl Engine {
self.registry.lock().await.models().await.map_err(Error::from)
}
+ pub async fn model(
+ &self,
+ identifier: &str,
+ ) -> Result<Option<Model>, Error> {
+ if let Some(model) = self.model_by_identifier(identifier).await? {
+ return Ok(Some(model));
+ }
+ if let Some(model) = self.model_by_repo_id(identifier).await? {
+ return Ok(Some(model));
+ }
+ self.model_by_path(identifier).await
+ }
+
pub async fn model_by_identifier(
&self,
identifier: &str,
@@ -155,6 +193,21 @@ impl Engine {
) -> Result<Option<Model>, Error> {
self.registry.lock().await.model_by_repo_id(repo_id).await.map_err(Error::from)
}
+
+ pub async fn model_by_path(
+ &self,
+ path: &str,
+ ) -> Result<Option<Model>, Error> {
+ let models = self.models().await?;
+ for model in models {
+ if let Some(model_path) = self.model_path(&model).await {
+ if model_path == path {
+ return Ok(Some(model));
+ }
+ }
+ }
+ Ok(None)
+ }
}
impl Engine {
@@ -169,9 +222,53 @@ impl Engine {
self.storage.lock().await.subscribe()
}
+ pub async fn model_path(
+ &self,
+ model: &Model,
+ ) -> Option<String> {
+ let path = if model.is_local() {
+ if let Some(local_external_path) = model.local_external_path() {
+ Some(local_external_path.clone())
+ } else {
+ let storage = self.storage.lock().await;
+ let state = storage.state(&model.identifier()).await?;
+ match state.phase {
+ DownloadPhase::Downloaded => {
+ storage.config.cache_model_path(model).map(|path| path.to_string_lossy().to_string())
+ },
+ DownloadPhase::NotDownloaded
+ | DownloadPhase::Downloading
+ | DownloadPhase::Paused
+ | DownloadPhase::Locked
+ | DownloadPhase::Error(_) => None,
+ }
+ }
+ } else {
+ None
+ };
+ path
+ }
+
async fn handle_registry_resfresh(&self) -> Result<(), Error> {
let models = self.registry.lock().await.models().await?;
self.storage.lock().await.refresh(models).await?;
Ok(())
}
}
+
+impl Engine {
+ pub async fn chat(
+ &self,
+ model: Model,
+ config: ChatConfig,
+ ) -> Result<ChatSession, Error> {
+ let path = self.model_path(&model).await;
+ if let Some(backend_entity) = model.backend_entity() {
+ let backend = self.backends.get(&backend_entity.identifier).ok_or(Error::BackendNotFound)?;
+ let session = ChatSession::new(backend.as_ref(), config, model, path).await?;
+ Ok(session)
+ } else {
+ return Err(Error::BackendNotFound);
+ }
+ }
+}
diff --git a/crates/uzu/src/registry/cached.rs b/crates/uzu/src/registry/cached.rs
index ee8c1e4f8..08b2055c7 100644
--- a/crates/uzu/src/registry/cached.rs
+++ b/crates/uzu/src/registry/cached.rs
@@ -1,6 +1,6 @@
use std::{future::Future, pin::Pin};
-use shoji::{traits::Registry, types::Model};
+use shoji::{traits::Registry, types::model::Model};
use tokio::sync::Mutex;
use crate::registry::Error;
diff --git a/crates/uzu/src/registry/fixed.rs b/crates/uzu/src/registry/fixed.rs
index cf08faa23..a7df1003a 100644
--- a/crates/uzu/src/registry/fixed.rs
+++ b/crates/uzu/src/registry/fixed.rs
@@ -1,6 +1,6 @@
use std::{future::Future, pin::Pin};
-use shoji::{traits::Registry, types::Model};
+use shoji::{traits::Registry, types::model::Model};
use tokio::sync::Mutex;
use crate::registry::Error;
diff --git a/crates/uzu/src/registry/local/config.rs b/crates/uzu/src/registry/local/config.rs
new file mode 100644
index 000000000..dcf3f590b
--- /dev/null
+++ b/crates/uzu/src/registry/local/config.rs
@@ -0,0 +1,41 @@
+use std::path::PathBuf;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct Config {
+ pub identifier: String,
+ pub backend_identifier: String,
+ pub name: String,
+ pub path: String,
+}
+
+impl Config {
+ pub fn new(
+ identifier: String,
+ backend_identifier: String,
+ name: String,
+ path: String,
+ ) -> Self {
+ Self {
+ identifier,
+ backend_identifier,
+ name,
+ path,
+ }
+ }
+
+ pub fn lalamo(
+ backend_identifier: String,
+ path: String,
+ ) -> Self {
+ let models_path = PathBuf::from(&path).join("models");
+ Self::new(
+ "lalamo".to_string(),
+ backend_identifier,
+ "Lalamo".to_string(),
+ models_path.to_string_lossy().to_string(),
+ )
+ }
+}
diff --git a/crates/uzu/src/registry/local/mod.rs b/crates/uzu/src/registry/local/mod.rs
new file mode 100644
index 000000000..11d522878
--- /dev/null
+++ b/crates/uzu/src/registry/local/mod.rs
@@ -0,0 +1,108 @@
+mod config;
+
+use std::{fs, future::Future, path::Path, pin::Pin};
+
+pub use config::Config;
+use shoji::{
+ traits::Registry as RegistryTrait,
+ types::model::{Accessibility, Entity, EntityType, Model, Reference, Specialization},
+};
+
+use crate::registry::Error;
+
+pub struct Registry {
+ config: Config,
+}
+
+impl Registry {
+ pub fn new(config: Config) -> Result<Self, Error> {
+ Ok(Self {
+ config,
+ })
+ }
+}
+
+impl RegistryTrait for Registry {
+ type Error = Error;
+
+ fn indentifier(&self) -> String {
+ self.config.identifier.clone()
+ }
+
+ fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, Error>> + Send + '_>> {
+ Box::pin(async {
+ let path = Path::new(&self.config.path);
+ if !path.exists() {
+ Err(Error::UnableToGetModels {
+ message: format!("Path not found: {}", path.display()),
+ })?;
+ }
+
+ let entries = fs::read_dir(path).map_err(|error| Error::UnableToGetModels {
+ message: error.to_string(),
+ })?;
+
+ let mut models = Vec::new();
+ for entry in entries.flatten() {
+ let path = entry.path();
+ let Ok(file_type) = entry.file_type() else {
+ continue;
+ };
+ if !file_type.is_dir() {
+ continue;
+ }
+ let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
+ continue;
+ };
+ models.push(self.model(name));
+ }
+
+ Ok(models)
+ })
+ }
+}
+
+impl Registry {
+ fn model(
+ &self,
+ name: &str,
+ ) -> Model {
+ let path = Path::new(&self.config.path).join(name);
+ let identifier = name.to_string();
+ Model {
+ identifier: identifier.clone(),
+ entities: vec![
+ self.entity(EntityType::Registry, &self.config.identifier, &self.config.name),
+ self.entity(EntityType::Backend, &self.config.backend_identifier, &self.config.backend_identifier),
+ self.entity(EntityType::Vendor, &self.config.identifier, &self.config.name),
+ self.entity(EntityType::Family, &self.config.identifier, &self.config.name),
+ self.entity(EntityType::Variant, &identifier, &identifier),
+ ],
+ specializations: vec![Specialization::Chat],
+ number_of_parameters: None,
+ quantization: None,
+ accessibility: Accessibility::Local {
+ reference: Reference::Local {
+ path: path.to_string_lossy().to_string(),
+ },
+ },
+ }
+ }
+
+ fn entity(
+ &self,
+ r#type: EntityType,
+ identifier: &str,
+ name: &str,
+ ) -> Entity {
+ Entity {
+ r#type,
+ identifier: identifier.to_string(),
+ parent_identifier: None,
+ name: name.to_string(),
+ description: None,
+ version: None,
+ icons: vec![],
+ }
+ }
+}
diff --git a/crates/uzu/src/registry/merged.rs b/crates/uzu/src/registry/merged.rs
index 4cbe8d694..b5feb8447 100644
--- a/crates/uzu/src/registry/merged.rs
+++ b/crates/uzu/src/registry/merged.rs
@@ -1,6 +1,6 @@
use std::{future::Future, pin::Pin};
-use shoji::{traits::Registry, types::Model};
+use shoji::{traits::Registry, types::model::Model};
use crate::registry::Error;
diff --git a/crates/uzu/src/registry/mirai/mod.rs b/crates/uzu/src/registry/mirai/mod.rs
index a022790b1..162fa1cb0 100644
--- a/crates/uzu/src/registry/mirai/mod.rs
+++ b/crates/uzu/src/registry/mirai/mod.rs
@@ -9,7 +9,7 @@ pub use config::{Backend, Config};
use indexmap::IndexMap;
use nagare::api::{Client, Config as ClientConfig};
use reqwest::header::AUTHORIZATION;
-use shoji::{traits::Registry as RegistryTrait, types::Model};
+use shoji::{traits::Registry as RegistryTrait, types::model::Model};
pub use types::Response;
use crate::registry::Error;
diff --git a/crates/uzu/src/registry/mirai/types.rs b/crates/uzu/src/registry/mirai/types.rs
index 10f40e8e4..f721b3f9a 100644
--- a/crates/uzu/src/registry/mirai/types.rs
+++ b/crates/uzu/src/registry/mirai/types.rs
@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
-use shoji::types::{Accessibility, Entity, Model, Quantization, Specialization};
+use shoji::types::model::{Accessibility, Entity, Model, Quantization, Specialization};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
diff --git a/crates/uzu/src/registry/mod.rs b/crates/uzu/src/registry/mod.rs
index 8f11c8419..62a0a781e 100644
--- a/crates/uzu/src/registry/mod.rs
+++ b/crates/uzu/src/registry/mod.rs
@@ -1,6 +1,7 @@
mod cached;
mod error;
mod fixed;
+pub mod local;
mod merged;
pub mod mirai;
pub mod openai;
diff --git a/crates/uzu/src/registry/openai/config.rs b/crates/uzu/src/registry/openai/config.rs
index d2c690ff4..f1ae0627f 100644
--- a/crates/uzu/src/registry/openai/config.rs
+++ b/crates/uzu/src/registry/openai/config.rs
@@ -1,7 +1,15 @@
use std::collections::HashMap;
+use backend_remote::openai::{ApiType as BackendApiType, Config as BackendConfig};
use serde::{Deserialize, Serialize};
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ApiType {
+ Completions,
+ Responses,
+}
+
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct Config {
@@ -9,6 +17,7 @@ pub struct Config {
pub name: String,
pub api_endpoint: String,
pub api_key: Option<String>,
+ pub api_type: ApiType,
pub model_filter_pattern: Option<String>,
pub headers: Option<HashMap<String, String>>,
}
@@ -19,6 +28,7 @@ impl Config {
name: String,
api_endpoint: String,
api_key: Option<String>,
+ api_type: ApiType,
model_filter_pattern: Option<String>,
headers: Option<HashMap<String, String>>,
) -> Self {
@@ -27,33 +37,44 @@ impl Config {
name,
api_endpoint,
api_key,
+ api_type,
model_filter_pattern,
headers,
}
}
- pub fn openai(api_key: String) -> Self {
+ pub fn ollama() -> Self {
Self::new(
- "openai".to_string(),
- "OpenAI".to_string(),
- "https://api.openai.com/v1".to_string(),
- Some(api_key),
- Some(r"^(?:gpt-\d[\d.]*[a-z]?|o\d+)(?:-(?:mini|nano|pro|turbo|codex(?:-mini|-max)?))*$".to_string()),
+ "ollama".to_string(),
+ "Ollama".to_string(),
+ "http://localhost:11434/v1".to_string(),
+ None,
+ ApiType::Responses,
+ None,
None,
)
}
- pub fn ollama() -> Self {
- Self::new("ollama".to_string(), "Ollama".to_string(), "http://localhost:11434/v1".to_string(), None, None, None)
+ pub fn lmstudio() -> Self {
+ Self::new(
+ "lmstudio".to_string(),
+ "LM Studio".to_string(),
+ "http://localhost:1234/v1".to_string(),
+ None,
+ ApiType::Responses,
+ Some(r"^(?!.*embedding).+$".to_string()),
+ None,
+ )
}
- pub fn baseten(api_key: String) -> Self {
+ pub fn openai(api_key: String) -> Self {
Self::new(
- "baseten".to_string(),
- "Baseten".to_string(),
- "https://inference.baseten.co/v1".to_string(),
+ "openai".to_string(),
+ "OpenAI".to_string(),
+ "https://api.openai.com/v1".to_string(),
Some(api_key),
- None,
+ ApiType::Responses,
+ Some(r"^(?:gpt-\d[\d.]*[a-z]?|o\d+)(?:-(?:mini|nano|pro|turbo|codex(?:-mini|-max)?))*$".to_string()),
None,
)
}
@@ -62,8 +83,9 @@ impl Config {
Self::new(
"anthropic".to_string(),
"Anthropic".to_string(),
- "https://api.anthropic.com/v1/".to_string(),
+ "https://api.anthropic.com/v1".to_string(),
Some(api_key.clone()),
+ ApiType::Completions,
Some(r"^claude-(?:opus|sonnet|haiku)-\d{1,2}(?:-\d{1,2})*$".to_string()),
Some(HashMap::from([
("X-Api-Key".to_string(), api_key.clone()),
@@ -72,14 +94,72 @@ impl Config {
)
}
+ pub fn gemini(api_key: String) -> Self {
+ Self::new(
+ "gemini".to_string(),
+ "Gemini".to_string(),
+ "https://generativelanguage.googleapis.com/v1beta/openai".to_string(),
+ Some(api_key),
+ ApiType::Completions,
+ Some(r"^models/(?:gemini|gemma)-(?!.*(?:image|audio|tts|live|embedding|computer-use|robotics|preview|latest|-\d{3}$|-\d{2}-\d{4}$)).+$".to_string()),
+ None,
+ )
+ }
+
+ pub fn xai(api_key: String) -> Self {
+ Self::new(
+ "xai".to_string(),
+ "xAI".to_string(),
+ "https://api.x.ai/v1".to_string(),
+ Some(api_key),
+ ApiType::Responses,
+ Some(r"^grok-(?!.*(?:imagine|-\d{4})).+$".to_string()),
+ None,
+ )
+ }
+
+ pub fn baseten(api_key: String) -> Self {
+ Self::new(
+ "baseten".to_string(),
+ "Baseten".to_string(),
+ "https://inference.baseten.co/v1".to_string(),
+ Some(api_key),
+ ApiType::Completions,
+ None,
+ None,
+ )
+ }
+
pub fn openrouter(api_key: String) -> Self {
Self::new(
"openrouter".to_string(),
"OpenRouter".to_string(),
"https://openrouter.ai/api/v1".to_string(),
Some(api_key.clone()),
+ ApiType::Responses,
None,
None,
)
}
}
+
+impl From<Config> for BackendConfig {
+ fn from(config: Config) -> Self {
+ Self {
+ identifier: config.identifier,
+ api_endpoint: config.api_endpoint,
+ api_key: config.api_key,
+ api_type: config.api_type.into(),
+ headers: config.headers,
+ }
+ }
+}
+
+impl From<ApiType> for BackendApiType {
+ fn from(value: ApiType) -> Self {
+ match value {
+ ApiType::Completions => BackendApiType::Completions,
+ ApiType::Responses => BackendApiType::Responses,
+ }
+ }
+}
diff --git a/crates/uzu/src/registry/openai/mod.rs b/crates/uzu/src/registry/openai/mod.rs
index fb357bb11..dae4442fa 100644
--- a/crates/uzu/src/registry/openai/mod.rs
+++ b/crates/uzu/src/registry/openai/mod.rs
@@ -2,37 +2,53 @@ mod config;
use std::{future::Future, pin::Pin};
+use async_openai::{Client, config::OpenAIConfig};
pub use config::Config;
-use openai_api_rs::v1::api::OpenAIClient;
-use regex::Regex;
+use fancy_regex::Regex;
+use serde::Deserialize;
use shoji::{
traits::Registry as RegistryTrait,
- types::{Accessibility, Entity, EntityType, Model, Specialization},
+ types::model::{Accessibility, Entity, EntityType, Model as ShojiModel, Specialization},
};
+#[derive(Debug, Deserialize)]
+struct Model {
+ id: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct ListModelsResponse {
+ data: Vec<Model>,
+}
+
use crate::registry::Error;
pub struct Registry {
config: Config,
- client: OpenAIClient,
+ client: Client<OpenAIConfig>,
model_filter: Option<Regex>,
}
impl Registry {
pub fn new(config: Config) -> Result<Self, Error> {
- let mut client_builder = OpenAIClient::builder().with_endpoint(config.api_endpoint.clone());
- if let Some(api_key) = config.api_key.clone() {
- client_builder = client_builder.with_api_key(api_key);
+ let mut openai_config = OpenAIConfig::new().with_api_base(&config.api_endpoint);
+ if let Some(api_key) = config.api_key.as_ref() {
+ openai_config = openai_config.with_api_key(api_key);
}
- if let Some(headers) = config.headers.clone() {
- for (key, value) in headers.iter() {
- client_builder = client_builder.with_header(key, value);
+ if let Some(headers) = config.headers.as_ref() {
+ for (key, value) in headers {
+ let name =
+ reqwest::header::HeaderName::from_bytes(key.as_bytes()).map_err(|error| Error::UnableToCreate {
+ message: error.to_string(),
+ })?;
+ openai_config =
+ openai_config.with_header(name, value.as_str()).map_err(|error| Error::UnableToCreate {
+ message: error.to_string(),
+ })?;
}
}
- let client = client_builder.build().map_err(|error| Error::UnableToCreate {
- message: error.to_string(),
- })?;
+ let client = Client::with_config(openai_config);
let model_filter = config.model_filter_pattern.as_deref().map(Regex::new).transpose().map_err(|error| {
Error::UnableToCreate {
@@ -55,13 +71,13 @@ impl RegistryTrait for Registry {
self.config.identifier.clone()
}
- fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, Error>> + Send + '_>> {
+ fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<ShojiModel>, Error>> + Send + '_>> {
Box::pin(async {
- let response = self.client.list_models().await.map_err(|error| Error::UnableToGetModels {
- message: error.to_string(),
- })?;
- let mut identifiers =
- response.inner.data.into_iter().filter_map(|model| model.id.clone()).collect::<Vec<_>>();
+ let response: ListModelsResponse =
+ self.client.models().list_byot().await.map_err(|error| Error::UnableToGetModels {
+ message: error.to_string(),
+ })?;
+ let mut identifiers = response.data.into_iter().map(|model| model.id).collect::<Vec<_>>();
identifiers.sort();
let registry_entity = self.create_entity(EntityType::Registry);
@@ -71,7 +87,9 @@ impl RegistryTrait for Registry {
let models = identifiers
.into_iter()
- .filter(|identifier| self.model_filter.as_ref().is_none_or(|regex| regex.is_match(identifier)))
+ .filter(|identifier| {
+ self.model_filter.as_ref().is_none_or(|regex| regex.is_match(identifier).unwrap_or(false))
+ })
.map(|identifier| {
let variant_entity = Entity {
r#type: EntityType::Variant,
@@ -82,7 +100,7 @@ impl RegistryTrait for Registry {
version: None,
icons: vec![],
};
- let model = Model {
+ let model = ShojiModel {
identifier: identifier.clone(),
entities: vec![
registry_entity.clone(),
diff --git a/crates/uzu/src/storage/config.rs b/crates/uzu/src/storage/config.rs
index 84b307da2..d13867d80 100644
--- a/crates/uzu/src/storage/config.rs
+++ b/crates/uzu/src/storage/config.rs
@@ -1,7 +1,7 @@
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
-use shoji::types::Model;
+use shoji::types::model::Model;
use crate::device::Device;
@@ -38,9 +38,9 @@ impl Config {
&self,
model: &Model,
) -> Option<PathBuf> {
- let reference_type = model.reference_type()?;
+ let reference_name = model.reference_name()?;
let checkpoint_version = model.checkpoint_version()?;
- Some(self.cache_models_path().join(reference_type).join(model.cache_identifier()).join(checkpoint_version))
+ Some(self.cache_models_path().join(reference_name).join(model.cache_identifier()).join(checkpoint_version))
}
pub fn log_name(&self) -> String {
diff --git a/crates/uzu/src/storage/mod.rs b/crates/uzu/src/storage/mod.rs
index aa3c87818..2f9ad7f55 100644
--- a/crates/uzu/src/storage/mod.rs
+++ b/crates/uzu/src/storage/mod.rs
@@ -13,7 +13,7 @@ use download_manager::{
FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase, create_download_manager,
};
pub use error::Error;
-use shoji::types::{Accessibility, File, Model, Reference};
+use shoji::types::model::{Accessibility, File, Model, Reference};
use tokio::{
runtime::Handle,
sync::broadcast::{Sender, channel},
@@ -26,7 +26,8 @@ use crate::{
};
pub struct Storage {
- config: Config,
+ pub config: Config,
+
download_manager: SharedAccess<Arc<dyn FileDownloadManager>>,
items: SharedAccess<HashMap<String, Item>>,
items_broadcast_sender: Sender<(String, DownloadState)>,
@@ -74,7 +75,7 @@ impl Storage {
&self,
models: Vec<Model>,
) -> Result<(), Error> {
- let models = models.into_iter().filter(|model| model.is_local()).collect::<Vec<_>>();
+ let models = models.into_iter().filter(|model| model.is_downloadable()).collect::<Vec<_>>();
let actual_model_identifiers: HashSet<String> = models.iter().map(|model| model.identifier()).collect();
let download_manager = { self.download_manager.lock().await.clone() };
@@ -264,6 +265,11 @@ impl Storage {
} => Err(Error::UnsupportedItem {
identifier: model.identifier(),
}),
+ Reference::Local {
+ ..
+ } => Err(Error::UnsupportedItem {
+ identifier: model.identifier(),
+ }),
},
Accessibility::Remote {
..
diff --git a/crates/uzu/src/storage/types/item.rs b/crates/uzu/src/storage/types/item.rs
index 86abf1252..dfa73bf9f 100644
--- a/crates/uzu/src/storage/types/item.rs
+++ b/crates/uzu/src/storage/types/item.rs
@@ -6,7 +6,7 @@ use std::{
use download_manager::{FileDownloadManager, FileDownloadState, FileDownloadTask};
use futures_util::future::join_all;
-use shoji::types::File;
+use shoji::types::model::File;
use tokio::{
runtime::Handle,
sync::broadcast::{Sender, channel},
diff --git a/crates/uzu/tests/common/test_helpers.rs b/crates/uzu/tests/common/test_helpers.rs
index c52ea5727..586179bbf 100644
--- a/crates/uzu/tests/common/test_helpers.rs
+++ b/crates/uzu/tests/common/test_helpers.rs
@@ -2,7 +2,7 @@
use std::path::PathBuf;
-use shoji::types::{Accessibility, Entity, EntityType, File, Hash, HashMethod, Model, Reference};
+use shoji::types::model::{Accessibility, Entity, EntityType, File, Hash, HashMethod, Model, Reference};
use tokio::runtime::Handle;
use uzu::{
device::Device,
diff --git a/crates/uzu/tests/test_engine.rs b/crates/uzu/tests/test_engine.rs
index 74b78a64b..e01fb250b 100644
--- a/crates/uzu/tests/test_engine.rs
+++ b/crates/uzu/tests/test_engine.rs
@@ -1,7 +1,13 @@
#![cfg(not(target_family = "wasm"))]
+use futures::StreamExt;
+use shoji::types::{
+ encoding::Message,
+ session::chat::{Config as ChatConfig, StreamConfig as ChatStreamConfig},
+};
use uzu::engine::{Config, Engine};
+#[ignore]
#[tokio::test]
async fn test_engine() {
dotenvy::dotenv().ok();
@@ -14,6 +20,45 @@ async fn test_engine() {
let identifier = model.identifier();
let registry_identifier = model.registry_entity().map(|entity| entity.identifier);
let download_phase = engine.downloader(&model).state().await.map_or(None, |state| Some(state.phase));
- println!("{:?}, {}, {:?}", registry_identifier, identifier, download_phase);
+ println!("{:?}, {}, {:?}, {:?}", registry_identifier, identifier, download_phase, model.specializations);
+ }
+
+ let model = engine.model("gpt-3.5-turbo").await.unwrap().unwrap();
+ let session = engine.chat(model, ChatConfig::default()).await.unwrap();
+ session.reset().await.unwrap();
+
+ let messages = vec![
+ Message::system().with_text("You are a helpful assistant.".to_string()),
+ Message::user().with_text("My name is John Doe".to_string()),
+ ];
+ let (mut stream, _) = session.stream(messages, ChatStreamConfig::default());
+ while let Some(progress) = stream.next().await {
+ match progress {
+ Ok(outputs) => {
+ let state = session.state().await;
+ let messages = session.messages().await;
+ let roles = messages.iter().map(|message| message.role.clone()).collect::<Vec<_>>();
+ println!("State: {state:?}");
+ println!("Roles: {roles:?}");
+ for output in outputs {
+ let duration = output.stats.duration;
+ let finish_reason = output.finish_reason;
+ let text_length = output.message.text().len();
+ println!(
+ "\tDuration: {duration}\n\tFinish reason: {finish_reason:?}\n\tText length: {text_length}"
+ );
+ }
+ },
+ Err(error) => eprintln!("Stream error: {error}"),
+ }
+ }
+
+ let messages = vec![Message::user().with_text("What is my name?".to_string())];
+ let (mut stream, _) = session.stream(messages, ChatStreamConfig::default());
+ while let Some(_) = stream.next().await {}
+
+ let messages = session.messages().await;
+ for message in messages {
+ println!("Message: {message:?}");
}
}
diff --git a/crates/uzu/tests/test_storage_comprehensive_state.rs b/crates/uzu/tests/test_storage_comprehensive_state.rs
index 7d7c2d2d3..4c5ade0c0 100644
--- a/crates/uzu/tests/test_storage_comprehensive_state.rs
+++ b/crates/uzu/tests/test_storage_comprehensive_state.rs
@@ -96,6 +96,7 @@ async fn test_storage_comprehensive_state_pause_and_resume() -> Result<(), Box<d
Ok(())
}
+#[ignore]
#[tokio::test(flavor = "multi_thread")]
async fn test_storage_comprehensive_state_pause_quit_relaunch_resume() -> Result<(), Box<dyn std::error::Error>> {
{
@@ -169,6 +170,7 @@ async fn test_storage_comprehensive_state_pause_quit_relaunch_resume() -> Result
Ok(())
}
+#[ignore]
#[tokio::test(flavor = "multi_thread")]
async fn test_storage_comprehensive_state_delete_from_various_states() -> Result<(), Box<dyn std::error::Error>> {
let temp_dir = tempfile::tempdir()?;
@@ -244,6 +246,7 @@ async fn test_storage_comprehensive_state_multiple_pause_resume_cycles() -> Resu
Ok(())
}
+#[ignore]
#[tokio::test(flavor = "multi_thread")]
async fn test_storage_comprehensive_state_completed_files_preserved_on_relaunch()
-> Result<(), Box<dyn std::error::Error>> {
@@ -331,6 +334,7 @@ async fn test_storage_comprehensive_state_completed_files_preserved_on_relaunch(
Ok(())
}
+#[ignore]
#[tokio::test(flavor = "multi_thread")]
async fn test_storage_comprehensive_state_idempotent_pause() -> Result<(), Box<dyn std::error::Error>> {
let temp_dir = tempfile::tempdir()?;
@@ -416,6 +420,7 @@ async fn test_storage_comprehensive_state_crc_validation_on_init() -> Result<(),
Ok(())
}
+#[ignore]
#[tokio::test(flavor = "multi_thread")]
async fn test_storage_comprehensive_state_progress_calculation_with_mixed_files()
-> Result<(), Box<dyn std::error::Error>> {
diff --git a/deny.toml b/deny.toml
index 2c14d8148..0519c8b89 100644
--- a/deny.toml
+++ b/deny.toml
@@ -4,6 +4,8 @@ ignore = [
"RUSTSEC-2024-0436", # paste - no longer maintained
"RUSTSEC-2024-0375", # `atty` is unmaintained
"RUSTSEC-2025-0057", # fxhash - no longer maintained
+ "RUSTSEC-2024-0384", # instant is unmaintained
+ "RUSTSEC-2025-0012", # backoff is unmaintained
]
[licenses]