A high-performance inference engine for AI models
Shareable weights (#733)
Aleksandr Golokozebfb07085dcb
diff --git a/README.md b/README.mdindex 5c1c4edde..480954a92 100644--- a/README.md+++ b/README.md@@ -241,7 +241,7 @@ Everything from model downloading to inference configuration is handled automati## Examples-You can run any example via `cargo tools example` \<**rust** | **python** | **swift** | **typescript**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:+You can run any example via `cargo tools example` \<**rust** | **python** | **swift** | **typescript**\> \<**chat** | **chat-cloud** | **chat-shared-instance** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:### Chat@@ -606,6 +606,222 @@ main().catch((error) => {</details>+### Chat with shared instance++This example shows how to reuse chat instance without reloading model into memory:++<details>+<summary>Rust</summary>++```rust+use std::io::{self, Write};++use uzu::{+ engine::{Engine, EngineConfig},+ types::session::chat::{ChatConfig, ChatMessage, ChatReplyConfig},+};++#[tokio::main]+async fn main() -> Result<(), Box<dyn std::error::Error>> {+ let engine_config = EngineConfig::default();+ let engine = Engine::new(engine_config).await?;++ let model = engine.model("alibaba:qwen3.5:0.8b:mirai:mirai-m:4".to_string()).await?.ok_or("Model not found")?;+ let downloader = engine.download(&model).await?;+ while let Some(update) = downloader.next().await {+ print!("\r\u{001B}[2KDownload progress: {:.2}%", update.progress() * 100.0);+ io::stdout().flush()?;+ }+ println!();++ // The chat_instance owns the loaded model and can be shared between sessions+ let chat_instance = engine.chat_instance(model, ChatConfig::default()).await?;++ let first_session = engine.chat_with_instance(&chat_instance).await?;+ let replies = first_session+ .reply(+ vec![ChatMessage::user().with_text("Tell me a short, funny story about a robot".to_string())],+ ChatReplyConfig::default(),+ )+ .await?;+ if let Some(reply) = replies.last() {+ println!("First session reasoning: {}", reply.message.reasoning().unwrap_or_default());+ println!("First session text: {}", reply.message.text().unwrap_or_default());+ }++ // The second session reuses the already-loaded weights instead of loading the model again+ let second_session = engine.chat_with_instance(&chat_instance).await?;+ let replies = second_session+ .reply(+ vec![ChatMessage::user().with_text("What is the capital of France?".to_string())],+ ChatReplyConfig::default(),+ )+ .await?;+ if let Some(reply) = replies.last() {+ println!("\nSecond session reasoning: {}", reply.message.reasoning().unwrap_or_default());+ println!("Second session text: {}", reply.message.text().unwrap_or_default());+ }++ Ok(())+}+```++</details>++<details>+<summary>Python</summary>++```python+import asyncio++from uzu import ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig+++async def main() -> None:+ engine_config = EngineConfig.create()+ engine = await Engine.create(engine_config)++ model = await engine.model("alibaba:qwen3.5:0.8b:mirai:mirai-m:4")+ if model is None:+ raise RuntimeError("Model not found")++ async for update in (await engine.download(model)).iterator():+ print(f"\rDownload progress: {update.progress:.2%}", end="", flush=True)+ print()++ # The chat_instance owns the loaded model and can be shared between sessions.+ chat_instance = await engine.chat_instance(model, ChatConfig.create())++ first_session = await engine.chat_with_instance(chat_instance)+ replies = await first_session.reply(+ [ChatMessage.user().with_text("Tell me a short, funny story about a robot")],+ ChatReplyConfig.create(),+ )+ if replies:+ message = replies[-1].message+ print(f"First session reasoning: {message.reasoning}")+ print(f"First session text: {message.text}")++ # The second session reuses the already-loaded weights instead of loading the model again.+ second_session = await engine.chat_with_instance(chat_instance)+ replies = await second_session.reply(+ [ChatMessage.user().with_text("What is the capital of France?")],+ ChatReplyConfig.create(),+ )+ if replies:+ message = replies[-1].message+ print(f"\nSecond session reasoning: {message.reasoning}")+ print(f"Second session text: {message.text}")+++if __name__ == "__main__":+ asyncio.run(main())+```++</details>++<details>+<summary>Swift</summary>++```swift+import Foundation+import Uzu++public func runChatSharedInstance() async throws {+ let engineConfig = EngineConfig.create()+ let engine = try await Engine.create(config: engineConfig)++ guard let model = try await engine.model(identifier: "alibaba:qwen3.5:0.8b:mirai:mirai-m:4") else {+ return+ }+ for try await update in try await engine.download(model: model).iterator() {+ print(String(format: "\r\u{001B}[2KDownload progress: %.2f%%", update.progress() * 100), terminator: "")+ fflush(stdout)+ }+ print()++ // The chatInstance owns the loaded model and can be shared between sessions.+ let chatInstance = try await engine.chatInstance(model: model, config: .create())++ let firstSession = try await engine.chatWithInstance(instance: chatInstance)+ let replies = try await firstSession.reply(+ input: [ChatMessage.user().withText(text: "Tell me a short, funny story about a robot")],+ config: .create()+ )+ if let message = replies.last?.message {+ print("First session reasoning: \(message.reasoning() ?? "")")+ print("First session text: \(message.text() ?? "")")+ }++ // The second session reuses the already-loaded weights instead of loading the model again.+ let secondSession = try await engine.chatWithInstance(instance: chatInstance)+ let secondReplies = try await secondSession.reply(+ input: [ChatMessage.user().withText(text: "What is the capital of France?")],+ config: .create()+ )+ if let message = secondReplies.last?.message {+ print("\nSecond session reasoning: \(message.reasoning() ?? "")")+ print("Second session text: \(message.text() ?? "")")+ }+}+```++</details>++<details>+<summary>TypeScript</summary>++```ts+import { ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig } from '@trymirai/uzu';++async function main() {+ let engineConfig = EngineConfig.create();+ let engine = await Engine.create(engineConfig);++ let model = await engine.model('alibaba:qwen3.5:0.8b:mirai:mirai-m:4');+ if (!model) {+ throw new Error('Model not found');+ }+ for await (const update of await engine.download(model)) {+ process.stdout.write(`\rDownload progress: ${(update.progress * 100).toFixed(2)}%`);+ }+ console.log();++ // The chat instance owns the loaded model and can be shared between sessions.+ let chatInstance = await engine.chatInstance(model, ChatConfig.create());++ let firstSession = await engine.chatWithInstance(chatInstance);+ let replies = await firstSession.reply(+ [ChatMessage.user().withText('Tell me a short, funny story about a robot')],+ ChatReplyConfig.create(),+ );+ let reply = replies[replies.length - 1];+ if (reply) {+ console.log('First session reasoning: ', reply.message.reasoning);+ console.log('First session text: ', reply.message.text);+ }++ // The second session reuses the already-loaded weights instead of loading the model again.+ let secondSession = await engine.chatWithInstance(chatInstance);+ replies = await secondSession.reply(+ [ChatMessage.user().withText('What is the capital of France?')],+ ChatReplyConfig.create(),+ );+ reply = replies[replies.length - 1];+ if (reply) {+ console.log('\nSecond session reasoning: ', reply.message.reasoning);+ console.log('Second session text: ', reply.message.text);+ }+}++main().catch((error) => {+ console.error(error);+});+```++</details>++### Chat with structured outputSometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:diff --git a/bindings/python/README.md b/bindings/python/README.mdindex 482867940..90550e0cb 100644--- a/bindings/python/README.md+++ b/bindings/python/README.md@@ -73,7 +73,7 @@ Everything from model downloading to inference configuration is handled automati## Examples-You can run any example via `cargo tools example` \<**python**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:+You can run any example via `cargo tools example` \<**python**\> \<**chat** | **chat-cloud** | **chat-shared-instance** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:### Chat@@ -160,6 +160,57 @@ async def main() -> None:print(f"Text: {message.text}")+if __name__ == "__main__":+ asyncio.run(main())+```++### Chat with shared instance++This example shows how to reuse chat instance without reloading model into memory:++```python+import asyncio++from uzu import ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig+++async def main() -> None:+ engine_config = EngineConfig.create()+ engine = await Engine.create(engine_config)++ model = await engine.model("alibaba:qwen3.5:0.8b:mirai:mirai-m:4")+ if model is None:+ raise RuntimeError("Model not found")++ async for update in (await engine.download(model)).iterator():+ print(f"\rDownload progress: {update.progress:.2%}", end="", flush=True)+ print()++ # The chat_instance owns the loaded model and can be shared between sessions.+ chat_instance = await engine.chat_instance(model, ChatConfig.create())++ first_session = await engine.chat_with_instance(chat_instance)+ replies = await first_session.reply(+ [ChatMessage.user().with_text("Tell me a short, funny story about a robot")],+ ChatReplyConfig.create(),+ )+ if replies:+ message = replies[-1].message+ print(f"First session reasoning: {message.reasoning}")+ print(f"First session text: {message.text}")++ # The second session reuses the already-loaded weights instead of loading the model again.+ second_session = await engine.chat_with_instance(chat_instance)+ replies = await second_session.reply(+ [ChatMessage.user().with_text("What is the capital of France?")],+ ChatReplyConfig.create(),+ )+ if replies:+ message = replies[-1].message+ print(f"\nSecond session reasoning: {message.reasoning}")+ print(f"Second session text: {message.text}")++if __name__ == "__main__":asyncio.run(main())```diff --git a/bindings/python/examples/chat_shared_instance.py b/bindings/python/examples/chat_shared_instance.pynew file mode 100644index 000000000..4ec2fc870--- /dev/null+++ b/bindings/python/examples/chat_shared_instance.py@@ -0,0 +1,44 @@+import asyncio++from uzu import ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig+++async def main() -> None:+ engine_config = EngineConfig.create()+ engine = await Engine.create(engine_config)++ model = await engine.model("alibaba:qwen3.5:0.8b:mirai:mirai-m:4")+ if model is None:+ raise RuntimeError("Model not found")++ async for update in (await engine.download(model)).iterator():+ print(f"\rDownload progress: {update.progress:.2%}", end="", flush=True)+ print()++ # The chat_instance owns the loaded model and can be shared between sessions.+ chat_instance = await engine.chat_instance(model, ChatConfig.create())++ first_session = await engine.chat_with_instance(chat_instance)+ replies = await first_session.reply(+ [ChatMessage.user().with_text("Tell me a short, funny story about a robot")],+ ChatReplyConfig.create(),+ )+ if replies:+ message = replies[-1].message+ print(f"First session reasoning: {message.reasoning}")+ print(f"First session text: {message.text}")++ # The second session reuses the already-loaded weights instead of loading the model again.+ second_session = await engine.chat_with_instance(chat_instance)+ replies = await second_session.reply(+ [ChatMessage.user().with_text("What is the capital of France?")],+ ChatReplyConfig.create(),+ )+ if replies:+ message = replies[-1].message+ print(f"\nSecond session reasoning: {message.reasoning}")+ print(f"Second session text: {message.text}")+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/bindings/python/uzu/__init__.pyi b/bindings/python/uzu/__init__.pyiindex dd63e625e..08609d816 100644--- a/bindings/python/uzu/__init__.pyi+++ b/bindings/python/uzu/__init__.pyi@@ -11,6 +11,7 @@ __all__ = ["ChatConfig","ChatContentBlock","ChatContentBlockType",+ "ChatInstance","ChatMessage","ChatMessageMetadata","ChatModelCapabilities",@@ -239,6 +240,16 @@ class ChatContentBlock:...+class ChatInstance:+ r"""+ A loaded chat backend instance (model weights, tokenizer, configuration).+ Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s+ can be created from one [`ChatInstance`] without loading the model again.+ """+ @property+ def model(self) -> Model: ...+@typing.finalclass ChatMessage:@property@@ -667,6 +678,8 @@ class Engine:def download_state(self, model: Model) -> collections.abc.Awaitable[DownloadState | None]: ...def download_states(self) -> collections.abc.Awaitable[builtins.dict[builtins.str, DownloadState]]: ...def chat(self, model: Model, config: ChatConfig) -> collections.abc.Awaitable[ChatSession]: ...+ def chat_instance(self, model: Model, config: ChatConfig) -> collections.abc.Awaitable[ChatInstance]: ...+ def chat_with_instance(self, instance: ChatInstance) -> collections.abc.Awaitable[ChatSession]: ...def classification(self, model: Model) -> collections.abc.Awaitable[ClassificationSession]: ...def text_to_speech(self, model: Model) -> collections.abc.Awaitable[TextToSpeechSession]: ...def settings(self) -> collections.abc.Awaitable[Settings]: ...diff --git a/bindings/swift/README.md b/bindings/swift/README.mdindex d003cbabc..cd87a5342 100644--- a/bindings/swift/README.md+++ b/bindings/swift/README.md@@ -72,7 +72,7 @@ Everything from model downloading to inference configuration is handled automati## Examples-You can run any example via `cargo tools example` \<**swift**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:+You can run any example via `cargo tools example` \<**swift**\> \<**chat** | **chat-cloud** | **chat-shared-instance** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:### Chat@@ -150,6 +150,53 @@ public func runChatCloud() async throws {}```+### Chat with shared instance++This example shows how to reuse chat instance without reloading model into memory:++```swift+import Foundation+import Uzu++public func runChatSharedInstance() async throws {+ let engineConfig = EngineConfig.create()+ let engine = try await Engine.create(config: engineConfig)++ guard let model = try await engine.model(identifier: "alibaba:qwen3.5:0.8b:mirai:mirai-m:4") else {+ return+ }+ for try await update in try await engine.download(model: model).iterator() {+ print(String(format: "\r\u{001B}[2KDownload progress: %.2f%%", update.progress() * 100), terminator: "")+ fflush(stdout)+ }+ print()++ // The chatInstance owns the loaded model and can be shared between sessions.+ let chatInstance = try await engine.chatInstance(model: model, config: .create())++ let firstSession = try await engine.chatWithInstance(instance: chatInstance)+ let replies = try await firstSession.reply(+ input: [ChatMessage.user().withText(text: "Tell me a short, funny story about a robot")],+ config: .create()+ )+ if let message = replies.last?.message {+ print("First session reasoning: \(message.reasoning() ?? "")")+ print("First session text: \(message.text() ?? "")")+ }++ // The second session reuses the already-loaded weights instead of loading the model again.+ let secondSession = try await engine.chatWithInstance(instance: chatInstance)+ let secondReplies = try await secondSession.reply(+ input: [ChatMessage.user().withText(text: "What is the capital of France?")],+ config: .create()+ )+ if let message = secondReplies.last?.message {+ print("\nSecond session reasoning: \(message.reasoning() ?? "")")+ print("Second session text: \(message.text() ?? "")")+ }+}+```+### Chat with structured outputSometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:diff --git a/bindings/swift/Sources/Examples/CLI.swift b/bindings/swift/Sources/Examples/CLI.swiftindex 1177245e3..be33c4640 100644--- a/bindings/swift/Sources/Examples/CLI.swift+++ b/bindings/swift/Sources/Examples/CLI.swift@@ -12,7 +12,7 @@ struct Example: AsyncParsableCommand {@Argument(help:- "Mode: chat | chat-cloud | chat-structured-output | classification | quick-start | tool-calls",+ "Mode: chat | chat-cloud | chat-shared-instance | chat-structured-output | classification | quick-start | tool-calls",transform: { $0.lowercased() })var mode: String = "chat"@@ -24,6 +24,8 @@ struct Example: AsyncParsableCommand {try await runChat()case "chat-cloud":try await runChatCloud()+ case "chat-shared-instance":+ try await runChatSharedInstance()case "chat-structured-output":try await runChatStructuredOutput()case "classification":diff --git a/bindings/swift/Sources/Examples/ChatSharedInstance.swift b/bindings/swift/Sources/Examples/ChatSharedInstance.swiftnew file mode 100644index 000000000..1b22e5ae5--- /dev/null+++ b/bindings/swift/Sources/Examples/ChatSharedInstance.swift@@ -0,0 +1,40 @@+import Foundation+import Uzu++public func runChatSharedInstance() async throws {+ let engineConfig = EngineConfig.create()+ let engine = try await Engine.create(config: engineConfig)++ guard let model = try await engine.model(identifier: "alibaba:qwen3.5:0.8b:mirai:mirai-m:4") else {+ return+ }+ for try await update in try await engine.download(model: model).iterator() {+ print(String(format: "\r\u{001B}[2KDownload progress: %.2f%%", update.progress() * 100), terminator: "")+ fflush(stdout)+ }+ print()++ // The chatInstance owns the loaded model and can be shared between sessions.+ let chatInstance = try await engine.chatInstance(model: model, config: .create())++ let firstSession = try await engine.chatWithInstance(instance: chatInstance)+ let replies = try await firstSession.reply(+ input: [ChatMessage.user().withText(text: "Tell me a short, funny story about a robot")],+ config: .create()+ )+ if let message = replies.last?.message {+ print("First session reasoning: \(message.reasoning() ?? "")")+ print("First session text: \(message.text() ?? "")")+ }++ // The second session reuses the already-loaded weights instead of loading the model again.+ let secondSession = try await engine.chatWithInstance(instance: chatInstance)+ let secondReplies = try await secondSession.reply(+ input: [ChatMessage.user().withText(text: "What is the capital of France?")],+ config: .create()+ )+ if let message = secondReplies.last?.message {+ print("\nSecond session reasoning: \(message.reasoning() ?? "")")+ print("Second session text: \(message.text() ?? "")")+ }+}diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift b/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swiftindex 6e38ac40a..d31d20e72 100644--- a/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift+++ b/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift@@ -506,6 +506,132 @@ fileprivate struct FfiConverterString: FfiConverter {+/**+ * A loaded chat backend instance (model weights, tokenizer, configuration).+ * Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s+ * can be created from one [`ChatInstance`] without loading the model again.+ */+public protocol ChatInstanceProtocol: AnyObject, Sendable {++ func model() -> Model++}+/**+ * A loaded chat backend instance (model weights, tokenizer, configuration).+ * Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s+ * can be created from one [`ChatInstance`] without loading the model again.+ */+open class ChatInstance: ChatInstanceProtocol, @unchecked Sendable {+ fileprivate let handle: UInt64++ /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly.+#if swift(>=5.8)+ @_documentation(visibility: private)+#endif+ public struct NoHandle {+ public init() {}+ }++ // TODO: We'd like this to be `private` but for Swifty reasons,+ // we can't implement `FfiConverter` without making this `required` and we can't+ // make it `required` without making it `public`.+#if swift(>=5.8)+ @_documentation(visibility: private)+#endif+ required public init(unsafeFromHandle handle: UInt64) {+ self.handle = handle+ }++ // This constructor can be used to instantiate a fake object.+ // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject].+ //+ // - Warning:+ // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash.+#if swift(>=5.8)+ @_documentation(visibility: private)+#endif+ public init(noHandle: NoHandle) {+ self.handle = 0+ }++#if swift(>=5.8)+ @_documentation(visibility: private)+#endif+ public func uniffiCloneHandle() -> UInt64 {+ return try! rustCall { uniffi_nagare_fn_clone_chatinstance(self.handle, $0) }+ }+ // No primary constructor declared for this class.++ deinit {+ if handle == 0 {+ // Mock objects have handle=0 don't try to free them+ return+ }++ try! rustCall { uniffi_nagare_fn_free_chatinstance(handle, $0) }+ }+++++open func model() -> Model {+ return try! FfiConverterTypeModel_lift(try! rustCall() {+ uniffi_nagare_fn_method_chatinstance_model(+ self.uniffiCloneHandle(),$0+ )+})+}++++}+++#if swift(>=5.8)+@_documentation(visibility: private)+#endif+public struct FfiConverterTypeChatInstance: FfiConverter {+ typealias FfiType = UInt64+ typealias SwiftType = ChatInstance++ public static func lift(_ handle: UInt64) throws -> ChatInstance {+ return ChatInstance(unsafeFromHandle: handle)+ }++ public static func lower(_ value: ChatInstance) -> UInt64 {+ return value.uniffiCloneHandle()+ }++ public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ChatInstance {+ let handle: UInt64 = try readInt(&buf)+ return try lift(handle)+ }++ public static func write(_ value: ChatInstance, into buf: inout [UInt8]) {+ writeInt(&buf, lower(value))+ }+}+++#if swift(>=5.8)+@_documentation(visibility: private)+#endif+public func FfiConverterTypeChatInstance_lift(_ handle: UInt64) throws -> ChatInstance {+ return try FfiConverterTypeChatInstance.lift(handle)+}++#if swift(>=5.8)+@_documentation(visibility: private)+#endif+public func FfiConverterTypeChatInstance_lower(_ value: ChatInstance) -> UInt64 {+ return FfiConverterTypeChatInstance.lower(value)+}++++++public protocol ChatSessionProtocol: AnyObject, Sendable {func addForeignTool(tool: ForeignTool) async throws@@ -2667,6 +2793,9 @@ private let initializationResult: InitializationResult = {if (uniffi_nagare_checksum_method_chatsessionstream_next() != 38311) {return InitializationResult.apiChecksumMismatch}+ if (uniffi_nagare_checksum_method_chatinstance_model() != 52758) {+ return InitializationResult.apiChecksumMismatch+ }if (uniffi_nagare_checksum_method_classificationsession_classify() != 62366) {return InitializationResult.apiChecksumMismatch}diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift b/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swiftindex dffb3811b..fea4d4fbe 100644--- a/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift+++ b/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift@@ -854,6 +854,10 @@ public protocol EngineProtocol: AnyObject, Sendable {func chat(model: Model, config: ChatConfig) async throws -> ChatSession+ func chatInstance(model: Model, config: ChatConfig) async throws -> ChatInstance++ func chatWithInstance(instance: ChatInstance) async throws -> ChatSession+func classification(model: Model) async throws -> ClassificationSessionfunc download(model: Model) async throws -> DownloaderStream@@ -983,6 +987,40 @@ open func chat(model: Model, config: ChatConfig)async throws -> ChatSession {)}+open func chatInstance(model: Model, config: ChatConfig)async throws -> ChatInstance {+ return+ try await uniffiRustCallAsync(+ rustFutureFunc: {+ uniffi_uzu_fn_method_engine_chat_instance(+ self.uniffiCloneHandle(),+ FfiConverterTypeModel_lower(model),FfiConverterTypeChatConfig_lower(config)+ )+ },+ pollFunc: ffi_uzu_rust_future_poll_u64,+ completeFunc: ffi_uzu_rust_future_complete_u64,+ freeFunc: ffi_uzu_rust_future_free_u64,+ liftFunc: FfiConverterTypeChatInstance_lift,+ errorHandler: FfiConverterTypeEngineError_lift+ )+}++open func chatWithInstance(instance: ChatInstance)async throws -> ChatSession {+ return+ try await uniffiRustCallAsync(+ rustFutureFunc: {+ uniffi_uzu_fn_method_engine_chat_with_instance(+ self.uniffiCloneHandle(),+ FfiConverterTypeChatInstance_lower(instance)+ )+ },+ pollFunc: ffi_uzu_rust_future_poll_u64,+ completeFunc: ffi_uzu_rust_future_complete_u64,+ freeFunc: ffi_uzu_rust_future_free_u64,+ liftFunc: FfiConverterTypeChatSession_lift,+ errorHandler: FfiConverterTypeEngineError_lift+ )+}+open func classification(model: Model)async throws -> ClassificationSession {returntry await uniffiRustCallAsync(@@ -3689,6 +3727,12 @@ private let initializationResult: InitializationResult = {if (uniffi_uzu_checksum_method_engine_chat() != 2862) {return InitializationResult.apiChecksumMismatch}+ if (uniffi_uzu_checksum_method_engine_chat_instance() != 25064) {+ return InitializationResult.apiChecksumMismatch+ }+ if (uniffi_uzu_checksum_method_engine_chat_with_instance() != 46821) {+ return InitializationResult.apiChecksumMismatch+ }if (uniffi_uzu_checksum_method_engine_classification() != 6465) {return InitializationResult.apiChecksumMismatch}diff --git a/bindings/typescript/README.md b/bindings/typescript/README.mdindex 0d89bef67..40c9f48d8 100644--- a/bindings/typescript/README.md+++ b/bindings/typescript/README.md@@ -73,7 +73,7 @@ Everything from model downloading to inference configuration is handled automati## Examples-You can run any example via `cargo tools example` \<**typescript**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:+You can run any example via `cargo tools example` \<**typescript**\> \<**chat** | **chat-cloud** | **chat-shared-instance** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:### Chat@@ -164,6 +164,58 @@ main().catch((error) => {});```+### Chat with shared instance++This example shows how to reuse chat instance without reloading model into memory:++```ts+import { ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig } from '@trymirai/uzu';++async function main() {+ let engineConfig = EngineConfig.create();+ let engine = await Engine.create(engineConfig);++ let model = await engine.model('alibaba:qwen3.5:0.8b:mirai:mirai-m:4');+ if (!model) {+ throw new Error('Model not found');+ }+ for await (const update of await engine.download(model)) {+ process.stdout.write(`\rDownload progress: ${(update.progress * 100).toFixed(2)}%`);+ }+ console.log();++ // The chat instance owns the loaded model and can be shared between sessions.+ let chatInstance = await engine.chatInstance(model, ChatConfig.create());++ let firstSession = await engine.chatWithInstance(chatInstance);+ let replies = await firstSession.reply(+ [ChatMessage.user().withText('Tell me a short, funny story about a robot')],+ ChatReplyConfig.create(),+ );+ let reply = replies[replies.length - 1];+ if (reply) {+ console.log('First session reasoning: ', reply.message.reasoning);+ console.log('First session text: ', reply.message.text);+ }++ // The second session reuses the already-loaded weights instead of loading the model again.+ let secondSession = await engine.chatWithInstance(chatInstance);+ replies = await secondSession.reply(+ [ChatMessage.user().withText('What is the capital of France?')],+ ChatReplyConfig.create(),+ );+ reply = replies[replies.length - 1];+ if (reply) {+ console.log('\nSecond session reasoning: ', reply.message.reasoning);+ console.log('Second session text: ', reply.message.text);+ }+}++main().catch((error) => {+ console.error(error);+});+```+### Chat with structured outputSometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:diff --git a/bindings/typescript/examples/chatSharedInstance.ts b/bindings/typescript/examples/chatSharedInstance.tsnew file mode 100644index 000000000..40278c1ce--- /dev/null+++ b/bindings/typescript/examples/chatSharedInstance.ts@@ -0,0 +1,45 @@+import { ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig } from '@trymirai/uzu';++async function main() {+ let engineConfig = EngineConfig.create();+ let engine = await Engine.create(engineConfig);++ let model = await engine.model('alibaba:qwen3.5:0.8b:mirai:mirai-m:4');+ if (!model) {+ throw new Error('Model not found');+ }+ for await (const update of await engine.download(model)) {+ process.stdout.write(`\rDownload progress: ${(update.progress * 100).toFixed(2)}%`);+ }+ console.log();++ // The chat instance owns the loaded model and can be shared between sessions.+ let chatInstance = await engine.chatInstance(model, ChatConfig.create());++ let firstSession = await engine.chatWithInstance(chatInstance);+ let replies = await firstSession.reply(+ [ChatMessage.user().withText('Tell me a short, funny story about a robot')],+ ChatReplyConfig.create(),+ );+ let reply = replies[replies.length - 1];+ if (reply) {+ console.log('First session reasoning: ', reply.message.reasoning);+ console.log('First session text: ', reply.message.text);+ }++ // The second session reuses the already-loaded weights instead of loading the model again.+ let secondSession = await engine.chatWithInstance(chatInstance);+ replies = await secondSession.reply(+ [ChatMessage.user().withText('What is the capital of France?')],+ ChatReplyConfig.create(),+ );+ reply = replies[replies.length - 1];+ if (reply) {+ console.log('\nSecond session reasoning: ', reply.message.reasoning);+ console.log('Second session text: ', reply.message.text);+ }+}++main().catch((error) => {+ console.error(error);+});diff --git a/bindings/typescript/src/napi/index.d.mts b/bindings/typescript/src/napi/index.d.mtsindex 857eb7914..801692db9 100644--- a/bindings/typescript/src/napi/index.d.mts+++ b/bindings/typescript/src/napi/index.d.mts@@ -1,5 +1,14 @@/* auto-generated by NAPI-RS *//* eslint-disable */+/**+ * A loaded chat backend instance (model weights, tokenizer, configuration).+ * Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s+ * can be created from one [`ChatInstance`] without loading the model again.+ */+export declare class ChatInstance {+ get model(): Model+}+export declare class ChatSession {addTool(tool: NativeTool): Promise<undefined>addTools(tools: Array<NativeTool>): Promise<undefined>@@ -917,6 +926,8 @@ export declare class Engine {downloadState(model: Model): Promise<DownloadState | null>get downloadStates(): Promise<Record<string, DownloadState>>chat(model: Model, config: ChatConfig): Promise<ChatSession>+ chatInstance(model: Model, config: ChatConfig): Promise<ChatInstance>+ chatWithInstance(instance: ChatInstance): Promise<ChatSession>classification(model: Model): Promise<ClassificationSession>textToSpeech(model: Model): Promise<TextToSpeechSession>settings(): Promise<Settings>diff --git a/bindings/typescript/src/napi/index.d.ts b/bindings/typescript/src/napi/index.d.tsindex 857eb7914..801692db9 100644--- a/bindings/typescript/src/napi/index.d.ts+++ b/bindings/typescript/src/napi/index.d.ts@@ -1,5 +1,14 @@/* auto-generated by NAPI-RS *//* eslint-disable */+/**+ * A loaded chat backend instance (model weights, tokenizer, configuration).+ * Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s+ * can be created from one [`ChatInstance`] without loading the model again.+ */+export declare class ChatInstance {+ get model(): Model+}+export declare class ChatSession {addTool(tool: NativeTool): Promise<undefined>addTools(tools: Array<NativeTool>): Promise<undefined>@@ -917,6 +926,8 @@ export declare class Engine {downloadState(model: Model): Promise<DownloadState | null>get downloadStates(): Promise<Record<string, DownloadState>>chat(model: Model, config: ChatConfig): Promise<ChatSession>+ chatInstance(model: Model, config: ChatConfig): Promise<ChatInstance>+ chatWithInstance(instance: ChatInstance): Promise<ChatSession>classification(model: Model): Promise<ClassificationSession>textToSpeech(model: Model): Promise<TextToSpeechSession>settings(): Promise<Settings>diff --git a/bindings/typescript/src/napi/index.js b/bindings/typescript/src/napi/index.jsindex 5b1119889..f268bd461 100644--- a/bindings/typescript/src/napi/index.js+++ b/bindings/typescript/src/napi/index.js@@ -508,6 +508,7 @@ if (!nativeBinding) {}module.exports = nativeBinding+module.exports.ChatInstance = nativeBinding.ChatInstancemodule.exports.ChatSession = nativeBinding.ChatSessionmodule.exports.ChatSessionStream = nativeBinding.ChatSessionStreammodule.exports.ChatSessionStreamChunkError = nativeBinding.ChatSessionStreamChunkErrordiff --git a/bindings/typescript/src/napi/index.mjs b/bindings/typescript/src/napi/index.mjsindex b1d2d2b8b..13c7f20a7 100644--- a/bindings/typescript/src/napi/index.mjs+++ b/bindings/typescript/src/napi/index.mjs@@ -5,6 +5,7 @@ import cjs from './index.js';export default cjs;+export const ChatInstance = cjs.ChatInstance;export const ChatSession = cjs.ChatSession;export const ChatSessionStream = cjs.ChatSessionStream;export const ChatSessionStreamChunkError = cjs.ChatSessionStreamChunkError;diff --git a/crates/nagare/src/chat/chat_instance.rs b/crates/nagare/src/chat/chat_instance.rsnew file mode 100644index 000000000..a27c15f2e--- /dev/null+++ b/crates/nagare/src/chat/chat_instance.rs@@ -0,0 +1,86 @@+use std::sync::Arc;++use shoji::{+ traits::{+ Backend,+ backend::{chat_message::Instance as ChatMessageInstance, chat_token::Instance as ChatTokenInstance},+ },+ types::{+ model::{Model, ModelSpecialization},+ session::chat::ChatConfig,+ },+};++use crate::chat::{ChatSessionError, message, token};++#[derive(Clone)]+pub enum ChatInstanceKind {+ Token(Arc<dyn ChatTokenInstance>),+ Message(Arc<dyn ChatMessageInstance>),+}++/// A loaded chat backend instance (model weights, tokenizer, configuration).+/// Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s+/// can be created from one [`ChatInstance`] without loading the model again.+#[bindings::export(Class)]+#[derive(Clone)]+pub struct ChatInstance {+ kind: ChatInstanceKind,+ model: Model,+ reference: String,+}++impl ChatInstance {+ pub async fn new(+ backend: Arc<dyn Backend>,+ config: ChatConfig,+ model: Model,+ path: Option<String>,+ ) -> Result<Self, ChatSessionError> {+ if !model.specializations.contains(&ModelSpecialization::Chat {}) {+ return Err(ChatSessionError::UnsupportedModel {});+ }+ let reference = path.unwrap_or_else(|| model.identifier.clone());++ let kind = tokio::spawn({+ let reference = reference.clone();+ async move {+ if let Some(token_backend) = backend.as_chat_via_token_capable() {+ token::Session::create_instance(token_backend, config, reference).await.map(ChatInstanceKind::Token)+ } else if let Some(message_backend) = backend.as_chat_via_message_capable() {+ message::Session::create_instance(message_backend, config, reference)+ .await+ .map(ChatInstanceKind::Message)+ } else {+ Err(ChatSessionError::UnsupportedModel {})+ }+ }+ })+ .await+ .map_err(|error| ChatSessionError::Backend {+ message: error.to_string(),+ })??;++ Ok(Self {+ kind,+ model,+ reference,+ })+ }++ pub fn kind(&self) -> ChatInstanceKind {+ self.kind.clone()+ }++ pub fn reference(&self) -> String {+ self.reference.clone()+ }+}++#[bindings::export(Implementation)]+impl ChatInstance {+ #[bindings::export(Method(Getter))]+ pub fn model(&self) -> Model {+ self.model.clone()+ }+}diff --git a/crates/nagare/src/chat/message.rs b/crates/nagare/src/chat/message.rsindex 8299a66f2..f62835074 100644--- a/crates/nagare/src/chat/message.rs+++ b/crates/nagare/src/chat/message.rs@@ -1,4 +1,4 @@-use std::pin::Pin;+use std::{pin::Pin, sync::Arc};use futures::{Stream, StreamExt};use shoji::{@@ -13,19 +13,23 @@ use tokio_util::sync::CancellationToken;use crate::chat::ChatSessionError;pub struct Session {- instance: Box<dyn Instance>,+ instance: Arc<dyn Instance>,state: Box<dyn State>,}impl Session {- pub async fn new(+ pub async fn create_instance(backend: &dyn Backend,config: ChatConfig,reference: String,- ) -> Result<Self, ChatSessionError> {+ ) -> Result<Arc<dyn Instance>, ChatSessionError> {let instance = backend.instance(reference, config).await.map_err(|error| ChatSessionError::Backend {message: error.to_string(),})?;+ Ok(Arc::from(instance))+ }++ pub async fn with_instance(instance: Arc<dyn Instance>) -> Result<Self, ChatSessionError> {let state = instance.state().await.map_err(|error| ChatSessionError::Backend {message: error.to_string(),})?;diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rsindex f59cb20f9..965b9cd87 100644--- a/crates/nagare/src/chat/mod.rs+++ b/crates/nagare/src/chat/mod.rs@@ -2,12 +2,14 @@mod bindings_napi;#[cfg(feature = "bindings-pyo3")]mod bindings_pyo3;+mod chat_instance;mod error;pub mod message;pub mod token;use std::{panic::AssertUnwindSafe, sync::Arc};+pub use chat_instance::{ChatInstance, ChatInstanceKind};pub use error::ChatSessionError;use futures::{FutureExt, StreamExt};use indexmap::IndexMap;@@ -19,7 +21,7 @@ use shoji::{},types::{basic::{CancelToken, ToolCall, ToolDescription, ToolNamespace, Value},- model::{Model, ModelSpecialization},+ model::Model,session::chat::{ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason,ChatReplyPowerStats, ChatReplySpeculatorStats, ChatReplyStats, ChatRole,@@ -134,19 +136,28 @@ impl ChatSession {path: Option<String>,telemetry: Telemetry,) -> Result<Self, ChatSessionError> {- if !model.specializations.contains(&ModelSpecialization::Chat {}) {- return Err(ChatSessionError::UnsupportedModel {});- }+ let instance = ChatInstance::new(backend, config, model, path).await?;+ Self::with_instance(&instance, telemetry).await+ }++ pub async fn with_instance(+ instance: &ChatInstance,+ telemetry: Telemetry,+ ) -> Result<Self, ChatSessionError> {+ let model = instance.model();let model_id = model.identifier.clone();- let reference = path.unwrap_or_else(|| model.identifier.clone());- let instance = tokio::spawn(async move {- if let Some(token_backend) = backend.as_chat_via_token_capable() {- token::Session::new(token_backend, config, reference, &model).await.map(Instance::Token)- } else if let Some(message_backend) = backend.as_chat_via_message_capable() {- message::Session::new(message_backend, config, reference).await.map(Instance::Message)- } else {- Err(ChatSessionError::UnsupportedModel {})+ let instance = tokio::spawn({+ let instance = instance.clone();+ async move {+ match instance.kind() {+ ChatInstanceKind::Token(shared) => {+ token::Session::with_instance(shared, instance.reference(), &model).await.map(Instance::Token)+ },+ ChatInstanceKind::Message(shared) => {+ message::Session::with_instance(shared).await.map(Instance::Message)+ },+ }}}).awaitdiff --git a/crates/nagare/src/chat/token.rs b/crates/nagare/src/chat/token.rsindex 569571712..e07143e19 100644--- a/crates/nagare/src/chat/token.rs+++ b/crates/nagare/src/chat/token.rs@@ -1,6 +1,7 @@use std::{io,pin::Pin,+ sync::Arc,time::{Duration, Instant},};@@ -37,7 +38,7 @@ use crate::{};pub struct Session {- instance: Box<dyn ChatTokenBackendInstance>,+ instance: Arc<dyn ChatTokenBackendInstance>,state: Box<dyn State>,encoding: Encoding,input_tokens: Vec<u64>,@@ -46,10 +47,20 @@ pub struct Session {}impl Session {- pub async fn new(+ pub async fn create_instance(backend: &dyn Backend,config: ChatConfig,reference: String,+ ) -> Result<Arc<dyn ChatTokenBackendInstance>, ChatSessionError> {+ let instance = backend.instance(reference, config).await.map_err(|error| ChatSessionError::Backend {+ message: error.to_string(),+ })?;+ Ok(Arc::from(instance))+ }++ pub async fn with_instance(+ instance: Arc<dyn ChatTokenBackendInstance>,+ reference: String,model: &Model,) -> Result<Self, ChatSessionError> {let encoding_configs = model@@ -76,15 +87,10 @@ impl Session {};let tokenizer_location = TokenizerLocation::Directory {- path: reference.clone(),+ path: reference,name: None,};- let instance =- backend.instance(reference.clone(), config).await.map_err(|error| ChatSessionError::Backend {- message: error.to_string(),- })?;-let encoding = match encoding_config {Some(EncodingConfig::Hanashi {config,diff --git a/crates/uzu/examples/chat_shared_instance.rs b/crates/uzu/examples/chat_shared_instance.rsnew file mode 100644index 000000000..e449027dc--- /dev/null+++ b/crates/uzu/examples/chat_shared_instance.rs@@ -0,0 +1,50 @@+use std::io::{self, Write};++use uzu::{+ engine::{Engine, EngineConfig},+ types::session::chat::{ChatConfig, ChatMessage, ChatReplyConfig},+};++#[tokio::main]+async fn main() -> Result<(), Box<dyn std::error::Error>> {+ let engine_config = EngineConfig::default();+ let engine = Engine::new(engine_config).await?;++ let model = engine.model("alibaba:qwen3.5:0.8b:mirai:mirai-m:4".to_string()).await?.ok_or("Model not found")?;+ let downloader = engine.download(&model).await?;+ while let Some(update) = downloader.next().await {+ print!("\r\u{001B}[2KDownload progress: {:.2}%", update.progress() * 100.0);+ io::stdout().flush()?;+ }+ println!();++ // The chat_instance owns the loaded model and can be shared between sessions+ let chat_instance = engine.chat_instance(model, ChatConfig::default()).await?;++ let first_session = engine.chat_with_instance(&chat_instance).await?;+ let replies = first_session+ .reply(+ vec![ChatMessage::user().with_text("Tell me a short, funny story about a robot".to_string())],+ ChatReplyConfig::default(),+ )+ .await?;+ if let Some(reply) = replies.last() {+ println!("First session reasoning: {}", reply.message.reasoning().unwrap_or_default());+ println!("First session text: {}", reply.message.text().unwrap_or_default());+ }++ // The second session reuses the already-loaded weights instead of loading the model again+ let second_session = engine.chat_with_instance(&chat_instance).await?;+ let replies = second_session+ .reply(+ vec![ChatMessage::user().with_text("What is the capital of France?".to_string())],+ ChatReplyConfig::default(),+ )+ .await?;+ if let Some(reply) = replies.last() {+ println!("\nSecond session reasoning: {}", reply.message.reasoning().unwrap_or_default());+ println!("Second session text: {}", reply.message.text().unwrap_or_default());+ }++ Ok(())+}diff --git a/crates/uzu/src/engine/mod.rs b/crates/uzu/src/engine/mod.rsindex 1535c562d..4b96f6182 100644--- a/crates/uzu/src/engine/mod.rs+++ b/crates/uzu/src/engine/mod.rs@@ -17,7 +17,7 @@ use indexmap::{IndexMap, IndexSet};use kiban::rt::RuntimeHandle;use nagare::{api::Config as ClientConfig,- chat::ChatSession,+ chat::{ChatInstance, ChatSession},classification::ClassificationSession,telemetry::{Telemetry, TelemetryContext, TelemetryDevice, TelemetryEvent},text_to_speech::TextToSpeechSession,@@ -494,18 +494,36 @@ impl Engine {model: Model,config: ChatConfig,) -> Result<ChatSession, EngineError> {+ let instance = self.chat_instance(model, config).await?;+ self.chat_with_instance(&instance).await+ }++ #[bindings::export(Method)]+ pub async fn chat_instance(+ &self,+ model: Model,+ config: ChatConfig,+ ) -> Result<ChatInstance, EngineError> {let path = self.model_path(&model).await;if let Some(backend) = model.backends.first() {- let backends = self.backends.lock().await;- let backend = backends.get(&backend.identifier).ok_or(EngineError::BackendNotFound {})?;- let session =- ChatSession::new(backend.clone(), config, model, path, self.telemetry.lock().await.clone()).await?;- Ok(session)+ let backend =+ self.backends.lock().await.get(&backend.identifier).ok_or(EngineError::BackendNotFound {})?.clone();+ let instance = ChatInstance::new(backend, config, model, path).await?;+ Ok(instance)} else {Err(EngineError::BackendNotFound {})}}+ #[bindings::export(Method)]+ pub async fn chat_with_instance(+ &self,+ instance: &ChatInstance,+ ) -> Result<ChatSession, EngineError> {+ let session = ChatSession::with_instance(instance, self.telemetry.lock().await.clone()).await?;+ Ok(session)+ }+#[bindings::export(Method)]pub async fn classification(&self,diff --git a/platforms.toml b/platforms.tomlindex d279153e2..46280bedb 100644--- a/platforms.toml+++ b/platforms.toml@@ -175,6 +175,7 @@ swift_version = '[](https:/quick-start = { title = "Quick Start", description = "Minimal snippet that downloads a model and generates a single reply" }chat = { title = "Chat", description = "In this example, we will download a model and get a reply to a specific list of messages:", explanation = "Once loaded, the same `ChatSession` can be reused for multiple requests until you drop it. Each model may consume a significant amount of RAM, so it's important to keep only one session loaded at a time. For iOS apps, we recommend adding the [Increased Memory Capability](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.developer.kernel.increased-memory-limit) entitlement to ensure your app can allocate the required memory." }chat-cloud = { title = "Chat with the cloud model", description = "In this example, we will get a reply to a specific list of messages from a cloud model:" }+chat-shared-instance = { title = "Chat with shared instance", description = "This example shows how to reuse chat instance without reloading model into memory:" }chat-structured-output = { title = "Chat with structured output", description = "Sometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:" }classification = { title = "Classification", description = "In this example, we will use a classification model to determine whether the user's input is safe from a moderation perspective:" }tool-calls = { title = "Tool calls", description = "This example shows how to use external tools:" }