Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Shareable weights (#733)

Aleksandr Golokozebfb07085dcb
diff --git a/README.md b/README.md
index 5c1c4edde..480954a92 100644
--- a/README.md
+++ b/README.md
@@ -241,7 +241,7 @@ Everything from model downloading to inference configuration is handled automati
## Examples
-You can run any example via `cargo tools example` \<**rust** | **python** | **swift** | **typescript**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:
+You can run any example via `cargo tools example` \<**rust** | **python** | **swift** | **typescript**\> \<**chat** | **chat-cloud** | **chat-shared-instance** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:
### Chat
@@ -606,6 +606,222 @@ main().catch((error) => {
</details>
+### Chat with shared instance
+
+This example shows how to reuse chat instance without reloading model into memory:
+
+<details>
+<summary>Rust</summary>
+
+```rust
+use std::io::{self, Write};
+
+use uzu::{
+ engine::{Engine, EngineConfig},
+ types::session::chat::{ChatConfig, ChatMessage, ChatReplyConfig},
+};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ let engine_config = EngineConfig::default();
+ let engine = Engine::new(engine_config).await?;
+
+ let model = engine.model("alibaba:qwen3.5:0.8b:mirai:mirai-m:4".to_string()).await?.ok_or("Model not found")?;
+ let downloader = engine.download(&model).await?;
+ while let Some(update) = downloader.next().await {
+ print!("\r\u{001B}[2KDownload progress: {:.2}%", update.progress() * 100.0);
+ io::stdout().flush()?;
+ }
+ println!();
+
+ // The chat_instance owns the loaded model and can be shared between sessions
+ let chat_instance = engine.chat_instance(model, ChatConfig::default()).await?;
+
+ let first_session = engine.chat_with_instance(&chat_instance).await?;
+ let replies = first_session
+ .reply(
+ vec![ChatMessage::user().with_text("Tell me a short, funny story about a robot".to_string())],
+ ChatReplyConfig::default(),
+ )
+ .await?;
+ if let Some(reply) = replies.last() {
+ println!("First session reasoning: {}", reply.message.reasoning().unwrap_or_default());
+ println!("First session text: {}", reply.message.text().unwrap_or_default());
+ }
+
+ // The second session reuses the already-loaded weights instead of loading the model again
+ let second_session = engine.chat_with_instance(&chat_instance).await?;
+ let replies = second_session
+ .reply(
+ vec![ChatMessage::user().with_text("What is the capital of France?".to_string())],
+ ChatReplyConfig::default(),
+ )
+ .await?;
+ if let Some(reply) = replies.last() {
+ println!("\nSecond session reasoning: {}", reply.message.reasoning().unwrap_or_default());
+ println!("Second session text: {}", reply.message.text().unwrap_or_default());
+ }
+
+ Ok(())
+}
+```
+
+</details>
+
+<details>
+<summary>Python</summary>
+
+```python
+import asyncio
+
+from uzu import ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig
+
+
+async def main() -> None:
+ engine_config = EngineConfig.create()
+ engine = await Engine.create(engine_config)
+
+ model = await engine.model("alibaba:qwen3.5:0.8b:mirai:mirai-m:4")
+ if model is None:
+ raise RuntimeError("Model not found")
+
+ async for update in (await engine.download(model)).iterator():
+ print(f"\rDownload progress: {update.progress:.2%}", end="", flush=True)
+ print()
+
+ # The chat_instance owns the loaded model and can be shared between sessions.
+ chat_instance = await engine.chat_instance(model, ChatConfig.create())
+
+ first_session = await engine.chat_with_instance(chat_instance)
+ replies = await first_session.reply(
+ [ChatMessage.user().with_text("Tell me a short, funny story about a robot")],
+ ChatReplyConfig.create(),
+ )
+ if replies:
+ message = replies[-1].message
+ print(f"First session reasoning: {message.reasoning}")
+ print(f"First session text: {message.text}")
+
+ # The second session reuses the already-loaded weights instead of loading the model again.
+ second_session = await engine.chat_with_instance(chat_instance)
+ replies = await second_session.reply(
+ [ChatMessage.user().with_text("What is the capital of France?")],
+ ChatReplyConfig.create(),
+ )
+ if replies:
+ message = replies[-1].message
+ print(f"\nSecond session reasoning: {message.reasoning}")
+ print(f"Second session text: {message.text}")
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>Swift</summary>
+
+```swift
+import Foundation
+import Uzu
+
+public func runChatSharedInstance() async throws {
+ let engineConfig = EngineConfig.create()
+ let engine = try await Engine.create(config: engineConfig)
+
+ guard let model = try await engine.model(identifier: "alibaba:qwen3.5:0.8b:mirai:mirai-m:4") else {
+ return
+ }
+ for try await update in try await engine.download(model: model).iterator() {
+ print(String(format: "\r\u{001B}[2KDownload progress: %.2f%%", update.progress() * 100), terminator: "")
+ fflush(stdout)
+ }
+ print()
+
+ // The chatInstance owns the loaded model and can be shared between sessions.
+ let chatInstance = try await engine.chatInstance(model: model, config: .create())
+
+ let firstSession = try await engine.chatWithInstance(instance: chatInstance)
+ let replies = try await firstSession.reply(
+ input: [ChatMessage.user().withText(text: "Tell me a short, funny story about a robot")],
+ config: .create()
+ )
+ if let message = replies.last?.message {
+ print("First session reasoning: \(message.reasoning() ?? "")")
+ print("First session text: \(message.text() ?? "")")
+ }
+
+ // The second session reuses the already-loaded weights instead of loading the model again.
+ let secondSession = try await engine.chatWithInstance(instance: chatInstance)
+ let secondReplies = try await secondSession.reply(
+ input: [ChatMessage.user().withText(text: "What is the capital of France?")],
+ config: .create()
+ )
+ if let message = secondReplies.last?.message {
+ print("\nSecond session reasoning: \(message.reasoning() ?? "")")
+ print("Second session text: \(message.text() ?? "")")
+ }
+}
+```
+
+</details>
+
+<details>
+<summary>TypeScript</summary>
+
+```ts
+import { ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig } from '@trymirai/uzu';
+
+async function main() {
+ let engineConfig = EngineConfig.create();
+ let engine = await Engine.create(engineConfig);
+
+ let model = await engine.model('alibaba:qwen3.5:0.8b:mirai:mirai-m:4');
+ if (!model) {
+ throw new Error('Model not found');
+ }
+ for await (const update of await engine.download(model)) {
+ process.stdout.write(`\rDownload progress: ${(update.progress * 100).toFixed(2)}%`);
+ }
+ console.log();
+
+ // The chat instance owns the loaded model and can be shared between sessions.
+ let chatInstance = await engine.chatInstance(model, ChatConfig.create());
+
+ let firstSession = await engine.chatWithInstance(chatInstance);
+ let replies = await firstSession.reply(
+ [ChatMessage.user().withText('Tell me a short, funny story about a robot')],
+ ChatReplyConfig.create(),
+ );
+ let reply = replies[replies.length - 1];
+ if (reply) {
+ console.log('First session reasoning: ', reply.message.reasoning);
+ console.log('First session text: ', reply.message.text);
+ }
+
+ // The second session reuses the already-loaded weights instead of loading the model again.
+ let secondSession = await engine.chatWithInstance(chatInstance);
+ replies = await secondSession.reply(
+ [ChatMessage.user().withText('What is the capital of France?')],
+ ChatReplyConfig.create(),
+ );
+ reply = replies[replies.length - 1];
+ if (reply) {
+ console.log('\nSecond session reasoning: ', reply.message.reasoning);
+ console.log('Second session text: ', reply.message.text);
+ }
+}
+
+main().catch((error) => {
+ console.error(error);
+});
+```
+
+</details>
+
+
### Chat with structured output
Sometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:
diff --git a/bindings/python/README.md b/bindings/python/README.md
index 482867940..90550e0cb 100644
--- a/bindings/python/README.md
+++ b/bindings/python/README.md
@@ -73,7 +73,7 @@ Everything from model downloading to inference configuration is handled automati
## Examples
-You can run any example via `cargo tools example` \<**python**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:
+You can run any example via `cargo tools example` \<**python**\> \<**chat** | **chat-cloud** | **chat-shared-instance** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:
### Chat
@@ -160,6 +160,57 @@ async def main() -> None:
print(f"Text: {message.text}")
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+### Chat with shared instance
+
+This example shows how to reuse chat instance without reloading model into memory:
+
+```python
+import asyncio
+
+from uzu import ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig
+
+
+async def main() -> None:
+ engine_config = EngineConfig.create()
+ engine = await Engine.create(engine_config)
+
+ model = await engine.model("alibaba:qwen3.5:0.8b:mirai:mirai-m:4")
+ if model is None:
+ raise RuntimeError("Model not found")
+
+ async for update in (await engine.download(model)).iterator():
+ print(f"\rDownload progress: {update.progress:.2%}", end="", flush=True)
+ print()
+
+ # The chat_instance owns the loaded model and can be shared between sessions.
+ chat_instance = await engine.chat_instance(model, ChatConfig.create())
+
+ first_session = await engine.chat_with_instance(chat_instance)
+ replies = await first_session.reply(
+ [ChatMessage.user().with_text("Tell me a short, funny story about a robot")],
+ ChatReplyConfig.create(),
+ )
+ if replies:
+ message = replies[-1].message
+ print(f"First session reasoning: {message.reasoning}")
+ print(f"First session text: {message.text}")
+
+ # The second session reuses the already-loaded weights instead of loading the model again.
+ second_session = await engine.chat_with_instance(chat_instance)
+ replies = await second_session.reply(
+ [ChatMessage.user().with_text("What is the capital of France?")],
+ ChatReplyConfig.create(),
+ )
+ if replies:
+ message = replies[-1].message
+ print(f"\nSecond session reasoning: {message.reasoning}")
+ print(f"Second session text: {message.text}")
+
+
if __name__ == "__main__":
asyncio.run(main())
```
diff --git a/bindings/python/examples/chat_shared_instance.py b/bindings/python/examples/chat_shared_instance.py
new file mode 100644
index 000000000..4ec2fc870
--- /dev/null
+++ b/bindings/python/examples/chat_shared_instance.py
@@ -0,0 +1,44 @@
+import asyncio
+
+from uzu import ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig
+
+
+async def main() -> None:
+ engine_config = EngineConfig.create()
+ engine = await Engine.create(engine_config)
+
+ model = await engine.model("alibaba:qwen3.5:0.8b:mirai:mirai-m:4")
+ if model is None:
+ raise RuntimeError("Model not found")
+
+ async for update in (await engine.download(model)).iterator():
+ print(f"\rDownload progress: {update.progress:.2%}", end="", flush=True)
+ print()
+
+ # The chat_instance owns the loaded model and can be shared between sessions.
+ chat_instance = await engine.chat_instance(model, ChatConfig.create())
+
+ first_session = await engine.chat_with_instance(chat_instance)
+ replies = await first_session.reply(
+ [ChatMessage.user().with_text("Tell me a short, funny story about a robot")],
+ ChatReplyConfig.create(),
+ )
+ if replies:
+ message = replies[-1].message
+ print(f"First session reasoning: {message.reasoning}")
+ print(f"First session text: {message.text}")
+
+ # The second session reuses the already-loaded weights instead of loading the model again.
+ second_session = await engine.chat_with_instance(chat_instance)
+ replies = await second_session.reply(
+ [ChatMessage.user().with_text("What is the capital of France?")],
+ ChatReplyConfig.create(),
+ )
+ if replies:
+ message = replies[-1].message
+ print(f"\nSecond session reasoning: {message.reasoning}")
+ print(f"Second session text: {message.text}")
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/bindings/python/uzu/__init__.pyi b/bindings/python/uzu/__init__.pyi
index dd63e625e..08609d816 100644
--- a/bindings/python/uzu/__init__.pyi
+++ b/bindings/python/uzu/__init__.pyi
@@ -11,6 +11,7 @@ __all__ = [
"ChatConfig",
"ChatContentBlock",
"ChatContentBlockType",
+ "ChatInstance",
"ChatMessage",
"ChatMessageMetadata",
"ChatModelCapabilities",
@@ -239,6 +240,16 @@ class ChatContentBlock:
...
+class ChatInstance:
+ r"""
+ A loaded chat backend instance (model weights, tokenizer, configuration).
+ Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s
+ can be created from one [`ChatInstance`] without loading the model again.
+ """
+ @property
+ def model(self) -> Model: ...
+
@typing.final
class ChatMessage:
@property
@@ -667,6 +678,8 @@ class Engine:
def download_state(self, model: Model) -> collections.abc.Awaitable[DownloadState | None]: ...
def download_states(self) -> collections.abc.Awaitable[builtins.dict[builtins.str, DownloadState]]: ...
def chat(self, model: Model, config: ChatConfig) -> collections.abc.Awaitable[ChatSession]: ...
+ def chat_instance(self, model: Model, config: ChatConfig) -> collections.abc.Awaitable[ChatInstance]: ...
+ def chat_with_instance(self, instance: ChatInstance) -> collections.abc.Awaitable[ChatSession]: ...
def classification(self, model: Model) -> collections.abc.Awaitable[ClassificationSession]: ...
def text_to_speech(self, model: Model) -> collections.abc.Awaitable[TextToSpeechSession]: ...
def settings(self) -> collections.abc.Awaitable[Settings]: ...
diff --git a/bindings/swift/README.md b/bindings/swift/README.md
index d003cbabc..cd87a5342 100644
--- a/bindings/swift/README.md
+++ b/bindings/swift/README.md
@@ -72,7 +72,7 @@ Everything from model downloading to inference configuration is handled automati
## Examples
-You can run any example via `cargo tools example` \<**swift**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:
+You can run any example via `cargo tools example` \<**swift**\> \<**chat** | **chat-cloud** | **chat-shared-instance** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:
### Chat
@@ -150,6 +150,53 @@ public func runChatCloud() async throws {
}
```
+### Chat with shared instance
+
+This example shows how to reuse chat instance without reloading model into memory:
+
+```swift
+import Foundation
+import Uzu
+
+public func runChatSharedInstance() async throws {
+ let engineConfig = EngineConfig.create()
+ let engine = try await Engine.create(config: engineConfig)
+
+ guard let model = try await engine.model(identifier: "alibaba:qwen3.5:0.8b:mirai:mirai-m:4") else {
+ return
+ }
+ for try await update in try await engine.download(model: model).iterator() {
+ print(String(format: "\r\u{001B}[2KDownload progress: %.2f%%", update.progress() * 100), terminator: "")
+ fflush(stdout)
+ }
+ print()
+
+ // The chatInstance owns the loaded model and can be shared between sessions.
+ let chatInstance = try await engine.chatInstance(model: model, config: .create())
+
+ let firstSession = try await engine.chatWithInstance(instance: chatInstance)
+ let replies = try await firstSession.reply(
+ input: [ChatMessage.user().withText(text: "Tell me a short, funny story about a robot")],
+ config: .create()
+ )
+ if let message = replies.last?.message {
+ print("First session reasoning: \(message.reasoning() ?? "")")
+ print("First session text: \(message.text() ?? "")")
+ }
+
+ // The second session reuses the already-loaded weights instead of loading the model again.
+ let secondSession = try await engine.chatWithInstance(instance: chatInstance)
+ let secondReplies = try await secondSession.reply(
+ input: [ChatMessage.user().withText(text: "What is the capital of France?")],
+ config: .create()
+ )
+ if let message = secondReplies.last?.message {
+ print("\nSecond session reasoning: \(message.reasoning() ?? "")")
+ print("Second session text: \(message.text() ?? "")")
+ }
+}
+```
+
### Chat with structured output
Sometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:
diff --git a/bindings/swift/Sources/Examples/CLI.swift b/bindings/swift/Sources/Examples/CLI.swift
index 1177245e3..be33c4640 100644
--- a/bindings/swift/Sources/Examples/CLI.swift
+++ b/bindings/swift/Sources/Examples/CLI.swift
@@ -12,7 +12,7 @@ struct Example: AsyncParsableCommand {
@Argument(
help:
- "Mode: chat | chat-cloud | chat-structured-output | classification | quick-start | tool-calls",
+ "Mode: chat | chat-cloud | chat-shared-instance | chat-structured-output | classification | quick-start | tool-calls",
transform: { $0.lowercased() })
var mode: String = "chat"
@@ -24,6 +24,8 @@ struct Example: AsyncParsableCommand {
try await runChat()
case "chat-cloud":
try await runChatCloud()
+ case "chat-shared-instance":
+ try await runChatSharedInstance()
case "chat-structured-output":
try await runChatStructuredOutput()
case "classification":
diff --git a/bindings/swift/Sources/Examples/ChatSharedInstance.swift b/bindings/swift/Sources/Examples/ChatSharedInstance.swift
new file mode 100644
index 000000000..1b22e5ae5
--- /dev/null
+++ b/bindings/swift/Sources/Examples/ChatSharedInstance.swift
@@ -0,0 +1,40 @@
+import Foundation
+import Uzu
+
+public func runChatSharedInstance() async throws {
+ let engineConfig = EngineConfig.create()
+ let engine = try await Engine.create(config: engineConfig)
+
+ guard let model = try await engine.model(identifier: "alibaba:qwen3.5:0.8b:mirai:mirai-m:4") else {
+ return
+ }
+ for try await update in try await engine.download(model: model).iterator() {
+ print(String(format: "\r\u{001B}[2KDownload progress: %.2f%%", update.progress() * 100), terminator: "")
+ fflush(stdout)
+ }
+ print()
+
+ // The chatInstance owns the loaded model and can be shared between sessions.
+ let chatInstance = try await engine.chatInstance(model: model, config: .create())
+
+ let firstSession = try await engine.chatWithInstance(instance: chatInstance)
+ let replies = try await firstSession.reply(
+ input: [ChatMessage.user().withText(text: "Tell me a short, funny story about a robot")],
+ config: .create()
+ )
+ if let message = replies.last?.message {
+ print("First session reasoning: \(message.reasoning() ?? "")")
+ print("First session text: \(message.text() ?? "")")
+ }
+
+ // The second session reuses the already-loaded weights instead of loading the model again.
+ let secondSession = try await engine.chatWithInstance(instance: chatInstance)
+ let secondReplies = try await secondSession.reply(
+ input: [ChatMessage.user().withText(text: "What is the capital of France?")],
+ config: .create()
+ )
+ if let message = secondReplies.last?.message {
+ print("\nSecond session reasoning: \(message.reasoning() ?? "")")
+ print("Second session text: \(message.text() ?? "")")
+ }
+}
diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift b/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift
index 6e38ac40a..d31d20e72 100644
--- a/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift
+++ b/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift
@@ -506,6 +506,132 @@ fileprivate struct FfiConverterString: FfiConverter {
+/**
+ * A loaded chat backend instance (model weights, tokenizer, configuration).
+ * Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s
+ * can be created from one [`ChatInstance`] without loading the model again.
+ */
+public protocol ChatInstanceProtocol: AnyObject, Sendable {
+
+ func model() -> Model
+
+}
+/**
+ * A loaded chat backend instance (model weights, tokenizer, configuration).
+ * Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s
+ * can be created from one [`ChatInstance`] without loading the model again.
+ */
+open class ChatInstance: ChatInstanceProtocol, @unchecked Sendable {
+ fileprivate let handle: UInt64
+
+ /// Used to instantiate a [FFIObject] without an actual handle, for fakes in tests, mostly.
+#if swift(>=5.8)
+ @_documentation(visibility: private)
+#endif
+ public struct NoHandle {
+ public init() {}
+ }
+
+ // TODO: We'd like this to be `private` but for Swifty reasons,
+ // we can't implement `FfiConverter` without making this `required` and we can't
+ // make it `required` without making it `public`.
+#if swift(>=5.8)
+ @_documentation(visibility: private)
+#endif
+ required public init(unsafeFromHandle handle: UInt64) {
+ self.handle = handle
+ }
+
+ // This constructor can be used to instantiate a fake object.
+ // - Parameter noHandle: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject].
+ //
+ // - Warning:
+ // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing handle the FFI lower functions will crash.
+#if swift(>=5.8)
+ @_documentation(visibility: private)
+#endif
+ public init(noHandle: NoHandle) {
+ self.handle = 0
+ }
+
+#if swift(>=5.8)
+ @_documentation(visibility: private)
+#endif
+ public func uniffiCloneHandle() -> UInt64 {
+ return try! rustCall { uniffi_nagare_fn_clone_chatinstance(self.handle, $0) }
+ }
+ // No primary constructor declared for this class.
+
+ deinit {
+ if handle == 0 {
+ // Mock objects have handle=0 don't try to free them
+ return
+ }
+
+ try! rustCall { uniffi_nagare_fn_free_chatinstance(handle, $0) }
+ }
+
+
+
+
+open func model() -> Model {
+ return try! FfiConverterTypeModel_lift(try! rustCall() {
+ uniffi_nagare_fn_method_chatinstance_model(
+ self.uniffiCloneHandle(),$0
+ )
+})
+}
+
+
+
+}
+
+
+#if swift(>=5.8)
+@_documentation(visibility: private)
+#endif
+public struct FfiConverterTypeChatInstance: FfiConverter {
+ typealias FfiType = UInt64
+ typealias SwiftType = ChatInstance
+
+ public static func lift(_ handle: UInt64) throws -> ChatInstance {
+ return ChatInstance(unsafeFromHandle: handle)
+ }
+
+ public static func lower(_ value: ChatInstance) -> UInt64 {
+ return value.uniffiCloneHandle()
+ }
+
+ public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ChatInstance {
+ let handle: UInt64 = try readInt(&buf)
+ return try lift(handle)
+ }
+
+ public static func write(_ value: ChatInstance, into buf: inout [UInt8]) {
+ writeInt(&buf, lower(value))
+ }
+}
+
+
+#if swift(>=5.8)
+@_documentation(visibility: private)
+#endif
+public func FfiConverterTypeChatInstance_lift(_ handle: UInt64) throws -> ChatInstance {
+ return try FfiConverterTypeChatInstance.lift(handle)
+}
+
+#if swift(>=5.8)
+@_documentation(visibility: private)
+#endif
+public func FfiConverterTypeChatInstance_lower(_ value: ChatInstance) -> UInt64 {
+ return FfiConverterTypeChatInstance.lower(value)
+}
+
+
+
+
+
+
public protocol ChatSessionProtocol: AnyObject, Sendable {
func addForeignTool(tool: ForeignTool) async throws
@@ -2667,6 +2793,9 @@ private let initializationResult: InitializationResult = {
if (uniffi_nagare_checksum_method_chatsessionstream_next() != 38311) {
return InitializationResult.apiChecksumMismatch
}
+ if (uniffi_nagare_checksum_method_chatinstance_model() != 52758) {
+ return InitializationResult.apiChecksumMismatch
+ }
if (uniffi_nagare_checksum_method_classificationsession_classify() != 62366) {
return InitializationResult.apiChecksumMismatch
}
diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift b/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift
index dffb3811b..fea4d4fbe 100644
--- a/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift
+++ b/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift
@@ -854,6 +854,10 @@ public protocol EngineProtocol: AnyObject, Sendable {
func chat(model: Model, config: ChatConfig) async throws -> ChatSession
+ func chatInstance(model: Model, config: ChatConfig) async throws -> ChatInstance
+
+ func chatWithInstance(instance: ChatInstance) async throws -> ChatSession
+
func classification(model: Model) async throws -> ClassificationSession
func download(model: Model) async throws -> DownloaderStream
@@ -983,6 +987,40 @@ open func chat(model: Model, config: ChatConfig)async throws -> ChatSession {
)
}
+open func chatInstance(model: Model, config: ChatConfig)async throws -> ChatInstance {
+ return
+ try await uniffiRustCallAsync(
+ rustFutureFunc: {
+ uniffi_uzu_fn_method_engine_chat_instance(
+ self.uniffiCloneHandle(),
+ FfiConverterTypeModel_lower(model),FfiConverterTypeChatConfig_lower(config)
+ )
+ },
+ pollFunc: ffi_uzu_rust_future_poll_u64,
+ completeFunc: ffi_uzu_rust_future_complete_u64,
+ freeFunc: ffi_uzu_rust_future_free_u64,
+ liftFunc: FfiConverterTypeChatInstance_lift,
+ errorHandler: FfiConverterTypeEngineError_lift
+ )
+}
+
+open func chatWithInstance(instance: ChatInstance)async throws -> ChatSession {
+ return
+ try await uniffiRustCallAsync(
+ rustFutureFunc: {
+ uniffi_uzu_fn_method_engine_chat_with_instance(
+ self.uniffiCloneHandle(),
+ FfiConverterTypeChatInstance_lower(instance)
+ )
+ },
+ pollFunc: ffi_uzu_rust_future_poll_u64,
+ completeFunc: ffi_uzu_rust_future_complete_u64,
+ freeFunc: ffi_uzu_rust_future_free_u64,
+ liftFunc: FfiConverterTypeChatSession_lift,
+ errorHandler: FfiConverterTypeEngineError_lift
+ )
+}
+
open func classification(model: Model)async throws -> ClassificationSession {
return
try await uniffiRustCallAsync(
@@ -3689,6 +3727,12 @@ private let initializationResult: InitializationResult = {
if (uniffi_uzu_checksum_method_engine_chat() != 2862) {
return InitializationResult.apiChecksumMismatch
}
+ if (uniffi_uzu_checksum_method_engine_chat_instance() != 25064) {
+ return InitializationResult.apiChecksumMismatch
+ }
+ if (uniffi_uzu_checksum_method_engine_chat_with_instance() != 46821) {
+ return InitializationResult.apiChecksumMismatch
+ }
if (uniffi_uzu_checksum_method_engine_classification() != 6465) {
return InitializationResult.apiChecksumMismatch
}
diff --git a/bindings/typescript/README.md b/bindings/typescript/README.md
index 0d89bef67..40c9f48d8 100644
--- a/bindings/typescript/README.md
+++ b/bindings/typescript/README.md
@@ -73,7 +73,7 @@ Everything from model downloading to inference configuration is handled automati
## Examples
-You can run any example via `cargo tools example` \<**typescript**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:
+You can run any example via `cargo tools example` \<**typescript**\> \<**chat** | **chat-cloud** | **chat-shared-instance** | **chat-structured-output** | **classification** | **quick-start** | **tool-calls**\>:
### Chat
@@ -164,6 +164,58 @@ main().catch((error) => {
});
```
+### Chat with shared instance
+
+This example shows how to reuse chat instance without reloading model into memory:
+
+```ts
+import { ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig } from '@trymirai/uzu';
+
+async function main() {
+ let engineConfig = EngineConfig.create();
+ let engine = await Engine.create(engineConfig);
+
+ let model = await engine.model('alibaba:qwen3.5:0.8b:mirai:mirai-m:4');
+ if (!model) {
+ throw new Error('Model not found');
+ }
+ for await (const update of await engine.download(model)) {
+ process.stdout.write(`\rDownload progress: ${(update.progress * 100).toFixed(2)}%`);
+ }
+ console.log();
+
+ // The chat instance owns the loaded model and can be shared between sessions.
+ let chatInstance = await engine.chatInstance(model, ChatConfig.create());
+
+ let firstSession = await engine.chatWithInstance(chatInstance);
+ let replies = await firstSession.reply(
+ [ChatMessage.user().withText('Tell me a short, funny story about a robot')],
+ ChatReplyConfig.create(),
+ );
+ let reply = replies[replies.length - 1];
+ if (reply) {
+ console.log('First session reasoning: ', reply.message.reasoning);
+ console.log('First session text: ', reply.message.text);
+ }
+
+ // The second session reuses the already-loaded weights instead of loading the model again.
+ let secondSession = await engine.chatWithInstance(chatInstance);
+ replies = await secondSession.reply(
+ [ChatMessage.user().withText('What is the capital of France?')],
+ ChatReplyConfig.create(),
+ );
+ reply = replies[replies.length - 1];
+ if (reply) {
+ console.log('\nSecond session reasoning: ', reply.message.reasoning);
+ console.log('Second session text: ', reply.message.text);
+ }
+}
+
+main().catch((error) => {
+ console.error(error);
+});
+```
+
### Chat with structured output
Sometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:
diff --git a/bindings/typescript/examples/chatSharedInstance.ts b/bindings/typescript/examples/chatSharedInstance.ts
new file mode 100644
index 000000000..40278c1ce
--- /dev/null
+++ b/bindings/typescript/examples/chatSharedInstance.ts
@@ -0,0 +1,45 @@
+import { ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig } from '@trymirai/uzu';
+
+async function main() {
+ let engineConfig = EngineConfig.create();
+ let engine = await Engine.create(engineConfig);
+
+ let model = await engine.model('alibaba:qwen3.5:0.8b:mirai:mirai-m:4');
+ if (!model) {
+ throw new Error('Model not found');
+ }
+ for await (const update of await engine.download(model)) {
+ process.stdout.write(`\rDownload progress: ${(update.progress * 100).toFixed(2)}%`);
+ }
+ console.log();
+
+ // The chat instance owns the loaded model and can be shared between sessions.
+ let chatInstance = await engine.chatInstance(model, ChatConfig.create());
+
+ let firstSession = await engine.chatWithInstance(chatInstance);
+ let replies = await firstSession.reply(
+ [ChatMessage.user().withText('Tell me a short, funny story about a robot')],
+ ChatReplyConfig.create(),
+ );
+ let reply = replies[replies.length - 1];
+ if (reply) {
+ console.log('First session reasoning: ', reply.message.reasoning);
+ console.log('First session text: ', reply.message.text);
+ }
+
+ // The second session reuses the already-loaded weights instead of loading the model again.
+ let secondSession = await engine.chatWithInstance(chatInstance);
+ replies = await secondSession.reply(
+ [ChatMessage.user().withText('What is the capital of France?')],
+ ChatReplyConfig.create(),
+ );
+ reply = replies[replies.length - 1];
+ if (reply) {
+ console.log('\nSecond session reasoning: ', reply.message.reasoning);
+ console.log('Second session text: ', reply.message.text);
+ }
+}
+
+main().catch((error) => {
+ console.error(error);
+});
diff --git a/bindings/typescript/src/napi/index.d.mts b/bindings/typescript/src/napi/index.d.mts
index 857eb7914..801692db9 100644
--- a/bindings/typescript/src/napi/index.d.mts
+++ b/bindings/typescript/src/napi/index.d.mts
@@ -1,5 +1,14 @@
/* auto-generated by NAPI-RS */
/* eslint-disable */
+/**
+ * A loaded chat backend instance (model weights, tokenizer, configuration).
+ * Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s
+ * can be created from one [`ChatInstance`] without loading the model again.
+ */
+export declare class ChatInstance {
+ get model(): Model
+}
+
export declare class ChatSession {
addTool(tool: NativeTool): Promise<undefined>
addTools(tools: Array<NativeTool>): Promise<undefined>
@@ -917,6 +926,8 @@ export declare class Engine {
downloadState(model: Model): Promise<DownloadState | null>
get downloadStates(): Promise<Record<string, DownloadState>>
chat(model: Model, config: ChatConfig): Promise<ChatSession>
+ chatInstance(model: Model, config: ChatConfig): Promise<ChatInstance>
+ chatWithInstance(instance: ChatInstance): Promise<ChatSession>
classification(model: Model): Promise<ClassificationSession>
textToSpeech(model: Model): Promise<TextToSpeechSession>
settings(): Promise<Settings>
diff --git a/bindings/typescript/src/napi/index.d.ts b/bindings/typescript/src/napi/index.d.ts
index 857eb7914..801692db9 100644
--- a/bindings/typescript/src/napi/index.d.ts
+++ b/bindings/typescript/src/napi/index.d.ts
@@ -1,5 +1,14 @@
/* auto-generated by NAPI-RS */
/* eslint-disable */
+/**
+ * A loaded chat backend instance (model weights, tokenizer, configuration).
+ * Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s
+ * can be created from one [`ChatInstance`] without loading the model again.
+ */
+export declare class ChatInstance {
+ get model(): Model
+}
+
export declare class ChatSession {
addTool(tool: NativeTool): Promise<undefined>
addTools(tools: Array<NativeTool>): Promise<undefined>
@@ -917,6 +926,8 @@ export declare class Engine {
downloadState(model: Model): Promise<DownloadState | null>
get downloadStates(): Promise<Record<string, DownloadState>>
chat(model: Model, config: ChatConfig): Promise<ChatSession>
+ chatInstance(model: Model, config: ChatConfig): Promise<ChatInstance>
+ chatWithInstance(instance: ChatInstance): Promise<ChatSession>
classification(model: Model): Promise<ClassificationSession>
textToSpeech(model: Model): Promise<TextToSpeechSession>
settings(): Promise<Settings>
diff --git a/bindings/typescript/src/napi/index.js b/bindings/typescript/src/napi/index.js
index 5b1119889..f268bd461 100644
--- a/bindings/typescript/src/napi/index.js
+++ b/bindings/typescript/src/napi/index.js
@@ -508,6 +508,7 @@ if (!nativeBinding) {
}
module.exports = nativeBinding
+module.exports.ChatInstance = nativeBinding.ChatInstance
module.exports.ChatSession = nativeBinding.ChatSession
module.exports.ChatSessionStream = nativeBinding.ChatSessionStream
module.exports.ChatSessionStreamChunkError = nativeBinding.ChatSessionStreamChunkError
diff --git a/bindings/typescript/src/napi/index.mjs b/bindings/typescript/src/napi/index.mjs
index b1d2d2b8b..13c7f20a7 100644
--- a/bindings/typescript/src/napi/index.mjs
+++ b/bindings/typescript/src/napi/index.mjs
@@ -5,6 +5,7 @@ import cjs from './index.js';
export default cjs;
+export const ChatInstance = cjs.ChatInstance;
export const ChatSession = cjs.ChatSession;
export const ChatSessionStream = cjs.ChatSessionStream;
export const ChatSessionStreamChunkError = cjs.ChatSessionStreamChunkError;
diff --git a/crates/nagare/src/chat/chat_instance.rs b/crates/nagare/src/chat/chat_instance.rs
new file mode 100644
index 000000000..a27c15f2e
--- /dev/null
+++ b/crates/nagare/src/chat/chat_instance.rs
@@ -0,0 +1,86 @@
+use std::sync::Arc;
+
+use shoji::{
+ traits::{
+ Backend,
+ backend::{chat_message::Instance as ChatMessageInstance, chat_token::Instance as ChatTokenInstance},
+ },
+ types::{
+ model::{Model, ModelSpecialization},
+ session::chat::ChatConfig,
+ },
+};
+
+use crate::chat::{ChatSessionError, message, token};
+
+#[derive(Clone)]
+pub enum ChatInstanceKind {
+ Token(Arc<dyn ChatTokenInstance>),
+ Message(Arc<dyn ChatMessageInstance>),
+}
+
+/// A loaded chat backend instance (model weights, tokenizer, configuration).
+/// Cloning is cheap and shares the underlying instance, so multiple [`ChatSession`]s
+/// can be created from one [`ChatInstance`] without loading the model again.
+#[bindings::export(Class)]
+#[derive(Clone)]
+pub struct ChatInstance {
+ kind: ChatInstanceKind,
+ model: Model,
+ reference: String,
+}
+
+impl ChatInstance {
+ pub async fn new(
+ backend: Arc<dyn Backend>,
+ config: ChatConfig,
+ model: Model,
+ path: Option<String>,
+ ) -> Result<Self, ChatSessionError> {
+ if !model.specializations.contains(&ModelSpecialization::Chat {}) {
+ return Err(ChatSessionError::UnsupportedModel {});
+ }
+ let reference = path.unwrap_or_else(|| model.identifier.clone());
+
+ let kind = tokio::spawn({
+ let reference = reference.clone();
+ async move {
+ if let Some(token_backend) = backend.as_chat_via_token_capable() {
+ token::Session::create_instance(token_backend, config, reference).await.map(ChatInstanceKind::Token)
+ } else if let Some(message_backend) = backend.as_chat_via_message_capable() {
+ message::Session::create_instance(message_backend, config, reference)
+ .await
+ .map(ChatInstanceKind::Message)
+ } else {
+ Err(ChatSessionError::UnsupportedModel {})
+ }
+ }
+ })
+ .await
+ .map_err(|error| ChatSessionError::Backend {
+ message: error.to_string(),
+ })??;
+
+ Ok(Self {
+ kind,
+ model,
+ reference,
+ })
+ }
+
+ pub fn kind(&self) -> ChatInstanceKind {
+ self.kind.clone()
+ }
+
+ pub fn reference(&self) -> String {
+ self.reference.clone()
+ }
+}
+
+#[bindings::export(Implementation)]
+impl ChatInstance {
+ #[bindings::export(Method(Getter))]
+ pub fn model(&self) -> Model {
+ self.model.clone()
+ }
+}
diff --git a/crates/nagare/src/chat/message.rs b/crates/nagare/src/chat/message.rs
index 8299a66f2..f62835074 100644
--- a/crates/nagare/src/chat/message.rs
+++ b/crates/nagare/src/chat/message.rs
@@ -1,4 +1,4 @@
-use std::pin::Pin;
+use std::{pin::Pin, sync::Arc};
use futures::{Stream, StreamExt};
use shoji::{
@@ -13,19 +13,23 @@ use tokio_util::sync::CancellationToken;
use crate::chat::ChatSessionError;
pub struct Session {
- instance: Box<dyn Instance>,
+ instance: Arc<dyn Instance>,
state: Box<dyn State>,
}
impl Session {
- pub async fn new(
+ pub async fn create_instance(
backend: &dyn Backend,
config: ChatConfig,
reference: String,
- ) -> Result<Self, ChatSessionError> {
+ ) -> Result<Arc<dyn Instance>, ChatSessionError> {
let instance = backend.instance(reference, config).await.map_err(|error| ChatSessionError::Backend {
message: error.to_string(),
})?;
+ Ok(Arc::from(instance))
+ }
+
+ pub async fn with_instance(instance: Arc<dyn Instance>) -> Result<Self, ChatSessionError> {
let state = instance.state().await.map_err(|error| ChatSessionError::Backend {
message: error.to_string(),
})?;
diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rs
index f59cb20f9..965b9cd87 100644
--- a/crates/nagare/src/chat/mod.rs
+++ b/crates/nagare/src/chat/mod.rs
@@ -2,12 +2,14 @@
mod bindings_napi;
#[cfg(feature = "bindings-pyo3")]
mod bindings_pyo3;
+mod chat_instance;
mod error;
pub mod message;
pub mod token;
use std::{panic::AssertUnwindSafe, sync::Arc};
+pub use chat_instance::{ChatInstance, ChatInstanceKind};
pub use error::ChatSessionError;
use futures::{FutureExt, StreamExt};
use indexmap::IndexMap;
@@ -19,7 +21,7 @@ use shoji::{
},
types::{
basic::{CancelToken, ToolCall, ToolDescription, ToolNamespace, Value},
- model::{Model, ModelSpecialization},
+ model::Model,
session::chat::{
ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason,
ChatReplyPowerStats, ChatReplySpeculatorStats, ChatReplyStats, ChatRole,
@@ -134,19 +136,28 @@ impl ChatSession {
path: Option<String>,
telemetry: Telemetry,
) -> Result<Self, ChatSessionError> {
- if !model.specializations.contains(&ModelSpecialization::Chat {}) {
- return Err(ChatSessionError::UnsupportedModel {});
- }
+ let instance = ChatInstance::new(backend, config, model, path).await?;
+ Self::with_instance(&instance, telemetry).await
+ }
+
+ pub async fn with_instance(
+ instance: &ChatInstance,
+ telemetry: Telemetry,
+ ) -> Result<Self, ChatSessionError> {
+ let model = instance.model();
let model_id = model.identifier.clone();
- let reference = path.unwrap_or_else(|| model.identifier.clone());
- let instance = tokio::spawn(async move {
- if let Some(token_backend) = backend.as_chat_via_token_capable() {
- token::Session::new(token_backend, config, reference, &model).await.map(Instance::Token)
- } else if let Some(message_backend) = backend.as_chat_via_message_capable() {
- message::Session::new(message_backend, config, reference).await.map(Instance::Message)
- } else {
- Err(ChatSessionError::UnsupportedModel {})
+ let instance = tokio::spawn({
+ let instance = instance.clone();
+ async move {
+ match instance.kind() {
+ ChatInstanceKind::Token(shared) => {
+ token::Session::with_instance(shared, instance.reference(), &model).await.map(Instance::Token)
+ },
+ ChatInstanceKind::Message(shared) => {
+ message::Session::with_instance(shared).await.map(Instance::Message)
+ },
+ }
}
})
.await
diff --git a/crates/nagare/src/chat/token.rs b/crates/nagare/src/chat/token.rs
index 569571712..e07143e19 100644
--- a/crates/nagare/src/chat/token.rs
+++ b/crates/nagare/src/chat/token.rs
@@ -1,6 +1,7 @@
use std::{
io,
pin::Pin,
+ sync::Arc,
time::{Duration, Instant},
};
@@ -37,7 +38,7 @@ use crate::{
};
pub struct Session {
- instance: Box<dyn ChatTokenBackendInstance>,
+ instance: Arc<dyn ChatTokenBackendInstance>,
state: Box<dyn State>,
encoding: Encoding,
input_tokens: Vec<u64>,
@@ -46,10 +47,20 @@ pub struct Session {
}
impl Session {
- pub async fn new(
+ pub async fn create_instance(
backend: &dyn Backend,
config: ChatConfig,
reference: String,
+ ) -> Result<Arc<dyn ChatTokenBackendInstance>, ChatSessionError> {
+ let instance = backend.instance(reference, config).await.map_err(|error| ChatSessionError::Backend {
+ message: error.to_string(),
+ })?;
+ Ok(Arc::from(instance))
+ }
+
+ pub async fn with_instance(
+ instance: Arc<dyn ChatTokenBackendInstance>,
+ reference: String,
model: &Model,
) -> Result<Self, ChatSessionError> {
let encoding_configs = model
@@ -76,15 +87,10 @@ impl Session {
};
let tokenizer_location = TokenizerLocation::Directory {
- path: reference.clone(),
+ path: reference,
name: None,
};
- let instance =
- backend.instance(reference.clone(), config).await.map_err(|error| ChatSessionError::Backend {
- message: error.to_string(),
- })?;
-
let encoding = match encoding_config {
Some(EncodingConfig::Hanashi {
config,
diff --git a/crates/uzu/examples/chat_shared_instance.rs b/crates/uzu/examples/chat_shared_instance.rs
new file mode 100644
index 000000000..e449027dc
--- /dev/null
+++ b/crates/uzu/examples/chat_shared_instance.rs
@@ -0,0 +1,50 @@
+use std::io::{self, Write};
+
+use uzu::{
+ engine::{Engine, EngineConfig},
+ types::session::chat::{ChatConfig, ChatMessage, ChatReplyConfig},
+};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ let engine_config = EngineConfig::default();
+ let engine = Engine::new(engine_config).await?;
+
+ let model = engine.model("alibaba:qwen3.5:0.8b:mirai:mirai-m:4".to_string()).await?.ok_or("Model not found")?;
+ let downloader = engine.download(&model).await?;
+ while let Some(update) = downloader.next().await {
+ print!("\r\u{001B}[2KDownload progress: {:.2}%", update.progress() * 100.0);
+ io::stdout().flush()?;
+ }
+ println!();
+
+ // The chat_instance owns the loaded model and can be shared between sessions
+ let chat_instance = engine.chat_instance(model, ChatConfig::default()).await?;
+
+ let first_session = engine.chat_with_instance(&chat_instance).await?;
+ let replies = first_session
+ .reply(
+ vec![ChatMessage::user().with_text("Tell me a short, funny story about a robot".to_string())],
+ ChatReplyConfig::default(),
+ )
+ .await?;
+ if let Some(reply) = replies.last() {
+ println!("First session reasoning: {}", reply.message.reasoning().unwrap_or_default());
+ println!("First session text: {}", reply.message.text().unwrap_or_default());
+ }
+
+ // The second session reuses the already-loaded weights instead of loading the model again
+ let second_session = engine.chat_with_instance(&chat_instance).await?;
+ let replies = second_session
+ .reply(
+ vec![ChatMessage::user().with_text("What is the capital of France?".to_string())],
+ ChatReplyConfig::default(),
+ )
+ .await?;
+ if let Some(reply) = replies.last() {
+ println!("\nSecond session reasoning: {}", reply.message.reasoning().unwrap_or_default());
+ println!("Second session text: {}", reply.message.text().unwrap_or_default());
+ }
+
+ Ok(())
+}
diff --git a/crates/uzu/src/engine/mod.rs b/crates/uzu/src/engine/mod.rs
index 1535c562d..4b96f6182 100644
--- a/crates/uzu/src/engine/mod.rs
+++ b/crates/uzu/src/engine/mod.rs
@@ -17,7 +17,7 @@ use indexmap::{IndexMap, IndexSet};
use kiban::rt::RuntimeHandle;
use nagare::{
api::Config as ClientConfig,
- chat::ChatSession,
+ chat::{ChatInstance, ChatSession},
classification::ClassificationSession,
telemetry::{Telemetry, TelemetryContext, TelemetryDevice, TelemetryEvent},
text_to_speech::TextToSpeechSession,
@@ -494,18 +494,36 @@ impl Engine {
model: Model,
config: ChatConfig,
) -> Result<ChatSession, EngineError> {
+ let instance = self.chat_instance(model, config).await?;
+ self.chat_with_instance(&instance).await
+ }
+
+ #[bindings::export(Method)]
+ pub async fn chat_instance(
+ &self,
+ model: Model,
+ config: ChatConfig,
+ ) -> Result<ChatInstance, EngineError> {
let path = self.model_path(&model).await;
if let Some(backend) = model.backends.first() {
- let backends = self.backends.lock().await;
- let backend = backends.get(&backend.identifier).ok_or(EngineError::BackendNotFound {})?;
- let session =
- ChatSession::new(backend.clone(), config, model, path, self.telemetry.lock().await.clone()).await?;
- Ok(session)
+ let backend =
+ self.backends.lock().await.get(&backend.identifier).ok_or(EngineError::BackendNotFound {})?.clone();
+ let instance = ChatInstance::new(backend, config, model, path).await?;
+ Ok(instance)
} else {
Err(EngineError::BackendNotFound {})
}
}
+ #[bindings::export(Method)]
+ pub async fn chat_with_instance(
+ &self,
+ instance: &ChatInstance,
+ ) -> Result<ChatSession, EngineError> {
+ let session = ChatSession::with_instance(instance, self.telemetry.lock().await.clone()).await?;
+ Ok(session)
+ }
+
#[bindings::export(Method)]
pub async fn classification(
&self,
diff --git a/platforms.toml b/platforms.toml
index d279153e2..46280bedb 100644
--- a/platforms.toml
+++ b/platforms.toml
@@ -175,6 +175,7 @@ swift_version = '[![Swift](https://img.shields.io/badge/Swift-5.9-blue)](https:/
quick-start = { title = "Quick Start", description = "Minimal snippet that downloads a model and generates a single reply" }
chat = { title = "Chat", description = "In this example, we will download a model and get a reply to a specific list of messages:", explanation = "Once loaded, the same `ChatSession` can be reused for multiple requests until you drop it. Each model may consume a significant amount of RAM, so it's important to keep only one session loaded at a time. For iOS apps, we recommend adding the [Increased Memory Capability](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.developer.kernel.increased-memory-limit) entitlement to ensure your app can allocate the required memory." }
chat-cloud = { title = "Chat with the cloud model", description = "In this example, we will get a reply to a specific list of messages from a cloud model:" }
+chat-shared-instance = { title = "Chat with shared instance", description = "This example shows how to reuse chat instance without reloading model into memory:" }
chat-structured-output = { title = "Chat with structured output", description = "Sometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:" }
classification = { title = "Classification", description = "In this example, we will use a classification model to determine whether the user's input is safe from a moderation perspective:" }
tool-calls = { title = "Tool calls", description = "This example shows how to use external tools:" }