Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Integrate speculator (#640)

uuuvn762cba252694
diff --git a/README.md b/README.md
index feb25d35b..44e226a78 100644
--- a/README.md
+++ b/README.md
@@ -232,7 +232,7 @@ Everything from model downloading to inference configuration is handled automati
## Examples
-You can run any example via `cargo tools example` \<**rust** | **python** | **swift** | **typescript**\> \<**chat** | **chat-cloud** | **chat-speculation-classification** | **chat-speculation-summarization** | **chat-structured-output** | **classification** | **quick-start** | **text-to-speech**\>:
+You can run any example via `cargo tools example` \<**rust** | **python** | **swift** | **typescript**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **text-to-speech**\>:
### Chat
@@ -581,451 +581,6 @@ main().catch((error) => {
</details>
-### Chat using speculation preset for classification
-
-In this example, we will use the `classification` speculation preset to determine the sentiment of the user's input:
-
-<details>
-<summary>Rust</summary>
-
-```rust
-use uzu::{
- engine::{Engine, EngineConfig},
- types::{
- basic::{Feature, ReasoningEffort, SamplingMethod},
- session::chat::{ChatConfig, ChatMessage, ChatReplyConfig, ChatSpeculationPreset},
- },
-};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
- let engine_config = EngineConfig::default();
- let engine = Engine::new(engine_config).await?;
-
- let model = engine.model("Qwen/Qwen3-0.6B".to_string()).await?.ok_or("Model not found")?;
- let downloader = engine.download(&model).await?;
- while let Some(update) = downloader.next().await {
- println!("Download progress: {}", update.progress());
- }
-
- let feature = Feature {
- name: "sentiment".to_string(),
- values: vec![
- "Happy".to_string(),
- "Sad".to_string(),
- "Angry".to_string(),
- "Fearful".to_string(),
- "Surprised".to_string(),
- "Disgusted".to_string(),
- ],
- };
- let chat_config = ChatConfig::default().with_speculation_preset(Some(ChatSpeculationPreset::Classification {
- feature: feature.clone(),
- }));
- let session = engine.chat(model, chat_config).await?;
-
- let text_to_detect_feature = "Today's been awesome! Everything just feels right, and I can't stop smiling.";
- let prompt = format!(
- "Text is: \"{text_to_detect_feature}\". Choose {} from the list: {}. Answer with one word. Don't add a dot at the end.",
- feature.name,
- feature.values.join(", ")
- );
- let messages = vec![
- ChatMessage::system().with_reasoning_effort(ReasoningEffort::Disabled),
- ChatMessage::user().with_text(prompt),
- ];
-
- let chat_reply_config =
- ChatReplyConfig::default().with_token_limit(Some(32)).with_sampling_method(SamplingMethod::Greedy {});
- let replies = session.reply(messages, chat_reply_config).await?;
- if let Some(reply) = replies.first() {
- println!("Prediction: {}", reply.message.text().unwrap_or_default());
- println!("Generated tokens: {}", reply.stats.tokens_count_output.unwrap_or_default());
- }
-
- Ok(())
-}
-```
-
-</details>
-
-<details>
-<summary>Python</summary>
-
-```python
-import asyncio
-
-from uzu import (
- ChatConfig,
- ChatMessage,
- ChatReplyConfig,
- ChatSpeculationPreset,
- Engine,
- EngineConfig,
- Feature,
- ReasoningEffort,
- SamplingMethod,
-)
-
-
-async def main() -> None:
- engine_config = EngineConfig.create()
- engine = await Engine.create(engine_config)
-
- model = await engine.model("Qwen/Qwen3-0.6B")
- if model is None:
- raise RuntimeError("Model not found")
- async for update in (await engine.download(model)).iterator():
- print(f"Download progress: {update.progress}")
-
- feature = Feature(
- "sentiment",
- ["Happy", "Sad", "Angry", "Fearful", "Surprised", "Disgusted"],
- )
- chat_config = ChatConfig.create().with_speculation_preset(ChatSpeculationPreset.Classification(feature))
- session = await engine.chat(model, chat_config)
-
- text_to_detect_feature = "Today's been awesome! Everything just feels right, and I can't stop smiling."
- prompt = (
- f'Text is: "{text_to_detect_feature}". '
- f"Choose {feature.name} from the list: {', '.join(feature.values)}. "
- "Answer with one word. Don't add a dot at the end."
- )
- messages = [
- ChatMessage.system().with_reasoning_effort(ReasoningEffort.Disabled),
- ChatMessage.user().with_text(prompt),
- ]
-
- chat_reply_config = ChatReplyConfig.create().with_token_limit(32).with_sampling_method(SamplingMethod.Greedy())
- replies = await session.reply(messages, chat_reply_config)
- if replies:
- reply = replies[0]
- print(f"Prediction: {reply.message.text}")
- print(f"Generated tokens: {reply.stats.tokens_count_output}")
-
-
-if __name__ == "__main__":
- asyncio.run(main())
-```
-
-</details>
-
-<details>
-<summary>Swift</summary>
-
-```swift
-import Uzu
-
-public func runChatSpeculationClassification() async throws {
- let engineConfig = EngineConfig.create()
- let engine = try await Engine.create(config: engineConfig)
-
- guard let model = try await engine.model(identifier: "Qwen/Qwen3-0.6B") else {
- return
- }
- for try await update in try await engine.download(model: model).iterator() {
- print("Download progress: \(update.progress())")
- }
-
- let feature = Feature(name: "sentiment", values: [
- "Happy",
- "Sad",
- "Angry",
- "Fearful",
- "Surprised",
- "Disgusted",
- ])
- let chatConfig = ChatConfig.create().withSpeculationPreset(speculationPreset: .classification(feature: feature))
- let session = try await engine.chat(model: model, config: chatConfig)
-
- let textToDetectFeature =
- "Today's been awesome! Everything just feels right, and I can't stop smiling."
- let prompt = "Text is: \"\(textToDetectFeature)\". Choose \(feature.name) from the list: \(feature.values.joined(separator: ", ")). Answer with one word. Don't add a dot at the end."
- let messages = [
- ChatMessage.system().withReasoningEffort(reasoningEffort: .disabled),
- ChatMessage.user().withText(text: prompt)
- ]
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(tokenLimit: 32).withSamplingMethod(samplingMethod: .greedy)
- let replies = try await session.reply(input: messages, config: chatReplyConfig)
- guard let reply = replies.last else {
- return
- }
-
- print("Prediction: \(reply.message.text() ?? "empty")")
- print("Generated tokens: \(reply.stats.tokensCountOutput ?? 0)")
-}
-```
-
-</details>
-
-<details>
-<summary>TypeScript</summary>
-
-```ts
-import { ChatConfig, ChatMessage, ChatReplyConfig, ChatSpeculationPresetClassification, Engine, EngineConfig, Feature, ReasoningEffort, SamplingMethodGreedy } from '@trymirai/uzu';
-
-async function main() {
- let engineConfig = EngineConfig.create();
- let engine = await Engine.create(engineConfig);
-
- let model = await engine.model('Qwen/Qwen3-0.6B');
- if (!model) {
- throw new Error('Model not found');
- }
- for await (const update of await engine.download(model)) {
- console.log('Download progress:', update.progress);
- }
-
- const feature = new Feature('sentiment', [
- 'Happy',
- 'Sad',
- 'Angry',
- 'Fearful',
- 'Surprised',
- 'Disgusted',
- ]);
- let chatConfig = ChatConfig.create().withSpeculationPreset(new ChatSpeculationPresetClassification(feature));
- let session = await engine.chat(model, chatConfig);
-
- const textToDetectFeature =
- "Today's been awesome! Everything just feels right, and I can't stop smiling.";
- const prompt =
- `Text is: "${textToDetectFeature}". Choose ${feature.name} from the list: ${feature.values.join(', ')}. ` +
- "Answer with one word. Don't add a dot at the end.";
- let messages = [
- ChatMessage.system().withReasoningEffort("Disabled" as ReasoningEffort),
- ChatMessage.user().withText(prompt)
- ];
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(32).withSamplingMethod(new SamplingMethodGreedy());
- let reply = (await session.reply(messages, chatReplyConfig))[0];
-
- if (reply) {
- console.log('Prediction: ', reply.message.text);
- console.log('Generated tokens: ', reply.stats.tokensCountOutput);
- }
-}
-
-main().catch((error) => {
- console.error(error);
-});
-```
-
-</details>
-
-
-<br>You can view the stats to see that the answer will be ready immediately after the prefill step, and actual generation won’t even start due to speculative decoding, which significantly improves generation speed.
-
-### Chat using speculation preset for summarization
-
-In this example, we will use the `summarization` speculation preset to generate a summary of the input text:
-
-<details>
-<summary>Rust</summary>
-
-```rust
-use uzu::{
- engine::{Engine, EngineConfig},
- types::{
- basic::{ReasoningEffort, SamplingMethod},
- session::chat::{ChatConfig, ChatMessage, ChatReplyConfig, ChatSpeculationPreset},
- },
-};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
- let engine_config = EngineConfig::default();
- let engine = Engine::new(engine_config).await?;
-
- let model = engine.model("Qwen/Qwen3-0.6B".to_string()).await?.ok_or("Model not found")?;
- let downloader = engine.download(&model).await?;
- while let Some(update) = downloader.next().await {
- println!("Download progress: {}", update.progress());
- }
-
- let text_to_summarize = "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. \
- It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. \
- LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. \
- These models have a wide range of applications, including chatbots, content creation, translation, and code generation. \
- One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. \
- They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. \
- Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. \
- As these models grow in size and sophistication, they continue to enhance human-computer interactions, \
- making AI-powered communication more natural and effective.";
- let prompt = format!("Text is: \"{text_to_summarize}\". Write only summary itself.");
- let messages = vec![
- ChatMessage::system().with_reasoning_effort(ReasoningEffort::Disabled),
- ChatMessage::user().with_text(prompt),
- ];
-
- let chat_config = ChatConfig::default().with_speculation_preset(Some(ChatSpeculationPreset::Summarization {}));
- let session = engine.chat(model, chat_config).await?;
-
- let chat_reply_config =
- ChatReplyConfig::default().with_token_limit(Some(256)).with_sampling_method(SamplingMethod::Greedy {});
- let replies = session.reply(messages, chat_reply_config).await?;
- if let Some(reply) = replies.first() {
- println!("Summary: {}", reply.message.text().unwrap_or_default());
- println!("Generation t/s: {}", reply.stats.generate_tokens_per_second.unwrap_or_default());
- }
-
- Ok(())
-}
-```
-
-</details>
-
-<details>
-<summary>Python</summary>
-
-```python
-import asyncio
-
-from uzu import (
- ChatConfig,
- ChatMessage,
- ChatReplyConfig,
- ChatSpeculationPreset,
- Engine,
- EngineConfig,
- ReasoningEffort,
- SamplingMethod,
-)
-
-
-async def main() -> None:
- engine_config = EngineConfig.create()
- engine = await Engine.create(engine_config)
-
- model = await engine.model("Qwen/Qwen3-0.6B")
- if model is None:
- raise RuntimeError("Model not found")
- async for update in (await engine.download(model)).iterator():
- print(f"Download progress: {update.progress}")
-
- text_to_summarize = (
- "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. "
- "It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. "
- "LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. "
- "These models have a wide range of applications, including chatbots, content creation, translation, and code generation. "
- "One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. "
- "They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. "
- "Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. "
- "As these models grow in size and sophistication, they continue to enhance human-computer interactions, "
- "making AI-powered communication more natural and effective."
- )
- prompt = f'Text is: "{text_to_summarize}". Write only summary itself.'
- messages = [
- ChatMessage.system().with_reasoning_effort(ReasoningEffort.Disabled),
- ChatMessage.user().with_text(prompt),
- ]
-
- chat_config = ChatConfig.create().with_speculation_preset(ChatSpeculationPreset.Summarization())
- session = await engine.chat(model, chat_config)
-
- chat_reply_config = ChatReplyConfig.create().with_token_limit(256).with_sampling_method(SamplingMethod.Greedy())
- replies = await session.reply(messages, chat_reply_config)
- if replies:
- reply = replies[0]
- print(f"Summary: {reply.message.text}")
- print(f"Generation t/s: {reply.stats.generate_tokens_per_second}")
-
-
-if __name__ == "__main__":
- asyncio.run(main())
-```
-
-</details>
-
-<details>
-<summary>Swift</summary>
-
-```swift
-import Uzu
-
-public func runChatSpeculationSummarization() async throws {
- let engineConfig = EngineConfig.create()
- let engine = try await Engine.create(config: engineConfig)
-
- guard let model = try await engine.model(identifier: "Qwen/Qwen3-0.6B") else {
- return
- }
- for try await update in try await engine.download(model: model).iterator() {
- print("Download progress: \(update.progress())")
- }
-
- let textToSummarize = "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. These models have a wide range of applications, including chatbots, content creation, translation, and code generation. One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. As these models grow in size and sophistication, they continue to enhance human-computer interactions, making AI-powered communication more natural and effective.";
- let prompt = "Text is: \"\(textToSummarize)\". Write only summary itself."
- let messages = [
- ChatMessage.system().withReasoningEffort(reasoningEffort: .disabled),
- ChatMessage.user().withText(text: prompt)
- ]
-
- let chatConfig = ChatConfig.create().withSpeculationPreset(speculationPreset: .summarization)
- let session = try await engine.chat(model: model, config: chatConfig)
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(tokenLimit: 256).withSamplingMethod(samplingMethod: .greedy)
- let replies = try await session.reply(input: messages, config: chatReplyConfig)
- guard let reply = replies.last else {
- return
- }
-
- print("Summary: \(reply.message.text() ?? "empty")")
- print("Generation t\\s: \(reply.stats.generateTokensPerSecond ?? 0.0)")
-}
-```
-
-</details>
-
-<details>
-<summary>TypeScript</summary>
-
-```ts
-import { ChatConfig, ChatMessage, ChatReplyConfig, ChatSpeculationPresetSummarization, Engine, EngineConfig, ReasoningEffort, SamplingMethodGreedy } from '@trymirai/uzu';
-
-async function main() {
- let engineConfig = EngineConfig.create();
- let engine = await Engine.create(engineConfig);
-
- let model = await engine.model('Qwen/Qwen3-0.6B');
- if (!model) {
- throw new Error('Model not found');
- }
- for await (const update of await engine.download(model)) {
- console.log('Download progress:', update.progress);
- }
-
- const textToSummarize =
- "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. These models have a wide range of applications, including chatbots, content creation, translation, and code generation. One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. As these models grow in size and sophistication, they continue to enhance human-computer interactions, making AI-powered communication more natural and effective.";
- const prompt = `Text is: "${textToSummarize}". Write only summary itself.`;
- let messages = [
- ChatMessage.system().withReasoningEffort("Disabled" as ReasoningEffort),
- ChatMessage.user().withText(prompt)
- ];
-
- let chatConfig = ChatConfig.create().withSpeculationPreset(new ChatSpeculationPresetSummarization);
- let session = await engine.chat(model, chatConfig);
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(256).withSamplingMethod(new SamplingMethodGreedy());
- let reply = (await session.reply(messages, chatReplyConfig))[0];
-
- if (reply) {
- console.log('Summary: ', reply.message.text);
- console.log('Generation t\\s: ', reply.stats.generateTokensPerSecond);
- }
-}
-
-main().catch((error) => {
- console.error(error);
-});
-```
-
-</details>
-
-
-<br>You will notice that the model’s run count is lower than the actual number of generated tokens due to speculative decoding, which significantly improves generation speed.
-
### Chat with structured output
Sometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:
diff --git a/apps/playground/Sources/ViewModels/ChatModel.swift b/apps/playground/Sources/ViewModels/ChatModel.swift
index 87ba22fe4..49da6eaf3 100644
--- a/apps/playground/Sources/ViewModels/ChatModel.swift
+++ b/apps/playground/Sources/ViewModels/ChatModel.swift
@@ -31,18 +31,10 @@ final class ChatModel {
self.identifier = identifier
}
- @MainActor
- func isCloud(engineWrapper: EngineObservableWrapper) -> Bool {
- guard let model = engineWrapper.models.first(where: { $0.identifier == identifier }) else {
- return false
- }
- return model.isRemote()
- }
-
// MARK: - Lifecycle
@MainActor
- func loadSession(using engineWrapper: EngineObservableWrapper, turbo: Bool) {
+ func loadSession(using engineWrapper: EngineObservableWrapper) {
guard
case .idle = viewState,
let engine = engineWrapper.engine,
@@ -62,7 +54,7 @@ final class ChatModel {
let session: ChatSession?
let newState: ViewState
do {
- let config = turbo ? ChatConfig.create().withSpeculationPreset(speculationPreset: .generalChat) : ChatConfig.create()
+ let config = ChatConfig.create()
session = try await engine.chat(model: model, config: config)
newState = .idle
} catch {
diff --git a/apps/playground/Sources/ViewModels/ClassificationModel.swift b/apps/playground/Sources/ViewModels/ClassificationModel.swift
index 3aa18899a..f44100d6e 100644
--- a/apps/playground/Sources/ViewModels/ClassificationModel.swift
+++ b/apps/playground/Sources/ViewModels/ClassificationModel.swift
@@ -46,7 +46,6 @@ final class ClassificationModel {
}
self.viewState = .loading
- let feature = self.feature
loadingTask?.cancel()
loadingTask = Task.detached { [weak self] in
@@ -54,7 +53,7 @@ final class ClassificationModel {
let session: ChatSession?
let newState: ViewState
do {
- let config = ChatConfig.create().withSpeculationPreset(speculationPreset: .classification(feature: feature))
+ let config = ChatConfig.create()
session = try await engine.chat(model: model, config: config)
newState = .idle
} catch {
diff --git a/apps/playground/Sources/ViewModels/SummarizationModel.swift b/apps/playground/Sources/ViewModels/SummarizationModel.swift
index 121cdd23c..91901f3ff 100644
--- a/apps/playground/Sources/ViewModels/SummarizationModel.swift
+++ b/apps/playground/Sources/ViewModels/SummarizationModel.swift
@@ -52,7 +52,7 @@ final class SummarizationModel {
let session: ChatSession?
let newState: ViewState
do {
- let config = ChatConfig.create().withSpeculationPreset(speculationPreset: .summarization)
+ let config = ChatConfig.create()
session = try await engine.chat(model: model, config: config)
newState = .idle
} catch {
diff --git a/apps/playground/Sources/Views/ChatView.swift b/apps/playground/Sources/Views/ChatView.swift
index 46a376964..a682151d5 100644
--- a/apps/playground/Sources/Views/ChatView.swift
+++ b/apps/playground/Sources/Views/ChatView.swift
@@ -32,7 +32,6 @@ struct ChatView: View {
// MARK: - State
@State private var viewModel: ChatModel
@FocusState private var inputFocused: Bool
- @State private var turbo = false
// MARK: - Stored Properties
let identifier: String
@@ -75,36 +74,6 @@ struct ChatView: View {
!viewModel.inputText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
}
- private var turboEnabled: Bool {
- switch(viewModel.viewState) {
- case .idle:
- return true
- case .loading, .generating, .error(_):
- return false
- }
- }
-
- @ViewBuilder
- private var turboView: some View {
- HStack(alignment: .center) {
- Spacer()
- Button(action: {
- turbo.toggle()
- }) {
- HStack(alignment: .center, spacing: 8.0) {
- Text("Turbo")
- .font(.monoHeading14)
- .foregroundColor(MiraiAsset.primary.swiftUIColor)
- Image(symbol: turbo ? .checkmarkCircleFill : .circle)
- .font(.title24Light)
- .foregroundColor(MiraiAsset.primary.swiftUIColor)
- }
- .opacity(turboEnabled ? 1.0 : 0.5)
- }
- }
- .padding(EdgeInsets(top: 0.0, leading: 16.0, bottom: 0.0, trailing: 16.0))
- }
-
var body: some View {
VStack(spacing: 0) {
ScrollViewReader { proxy in
@@ -123,11 +92,6 @@ struct ChatView: View {
}
}
Divider()
- if !viewModel.isCloud(engineWrapper: engineWrapper) {
- Spacer(minLength: 16.0)
- turboView
- .disabled(!turboEnabled)
- }
inputView
.padding()
}
@@ -136,10 +100,7 @@ struct ChatView: View {
#endif
.toolbarRole(.editor)
.onAppear {
- viewModel.loadSession(using: engineWrapper, turbo: turbo)
- }
- .onChange(of: turbo) { _, _ in
- viewModel.loadSession(using: engineWrapper, turbo: turbo)
+ viewModel.loadSession(using: engineWrapper)
}
.onDisappear {
viewModel.tearDown()
diff --git a/bindings/python/README.md b/bindings/python/README.md
index 728f7463b..c3661e609 100644
--- a/bindings/python/README.md
+++ b/bindings/python/README.md
@@ -72,7 +72,7 @@ Everything from model downloading to inference configuration is handled automati
## Examples
-You can run any example via `cargo tools example` \<**python**\> \<**chat** | **chat-cloud** | **chat-speculation-classification** | **chat-speculation-summarization** | **chat-structured-output** | **classification** | **quick-start** | **text-to-speech**\>:
+You can run any example via `cargo tools example` \<**python**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **text-to-speech**\>:
### Chat
@@ -163,131 +163,6 @@ if __name__ == "__main__":
asyncio.run(main())
```
-### Chat using speculation preset for classification
-
-In this example, we will use the `classification` speculation preset to determine the sentiment of the user's input:
-
-```python
-import asyncio
-
-from uzu import (
- ChatConfig,
- ChatMessage,
- ChatReplyConfig,
- ChatSpeculationPreset,
- Engine,
- EngineConfig,
- Feature,
- ReasoningEffort,
- SamplingMethod,
-)
-
-
-async def main() -> None:
- engine_config = EngineConfig.create()
- engine = await Engine.create(engine_config)
-
- model = await engine.model("Qwen/Qwen3-0.6B")
- if model is None:
- raise RuntimeError("Model not found")
- async for update in (await engine.download(model)).iterator():
- print(f"Download progress: {update.progress}")
-
- feature = Feature(
- "sentiment",
- ["Happy", "Sad", "Angry", "Fearful", "Surprised", "Disgusted"],
- )
- chat_config = ChatConfig.create().with_speculation_preset(ChatSpeculationPreset.Classification(feature))
- session = await engine.chat(model, chat_config)
-
- text_to_detect_feature = "Today's been awesome! Everything just feels right, and I can't stop smiling."
- prompt = (
- f'Text is: "{text_to_detect_feature}". '
- f"Choose {feature.name} from the list: {', '.join(feature.values)}. "
- "Answer with one word. Don't add a dot at the end."
- )
- messages = [
- ChatMessage.system().with_reasoning_effort(ReasoningEffort.Disabled),
- ChatMessage.user().with_text(prompt),
- ]
-
- chat_reply_config = ChatReplyConfig.create().with_token_limit(32).with_sampling_method(SamplingMethod.Greedy())
- replies = await session.reply(messages, chat_reply_config)
- if replies:
- reply = replies[0]
- print(f"Prediction: {reply.message.text}")
- print(f"Generated tokens: {reply.stats.tokens_count_output}")
-
-
-if __name__ == "__main__":
- asyncio.run(main())
-```
-
-<br>You can view the stats to see that the answer will be ready immediately after the prefill step, and actual generation won’t even start due to speculative decoding, which significantly improves generation speed.
-
-### Chat using speculation preset for summarization
-
-In this example, we will use the `summarization` speculation preset to generate a summary of the input text:
-
-```python
-import asyncio
-
-from uzu import (
- ChatConfig,
- ChatMessage,
- ChatReplyConfig,
- ChatSpeculationPreset,
- Engine,
- EngineConfig,
- ReasoningEffort,
- SamplingMethod,
-)
-
-
-async def main() -> None:
- engine_config = EngineConfig.create()
- engine = await Engine.create(engine_config)
-
- model = await engine.model("Qwen/Qwen3-0.6B")
- if model is None:
- raise RuntimeError("Model not found")
- async for update in (await engine.download(model)).iterator():
- print(f"Download progress: {update.progress}")
-
- text_to_summarize = (
- "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. "
- "It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. "
- "LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. "
- "These models have a wide range of applications, including chatbots, content creation, translation, and code generation. "
- "One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. "
- "They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. "
- "Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. "
- "As these models grow in size and sophistication, they continue to enhance human-computer interactions, "
- "making AI-powered communication more natural and effective."
- )
- prompt = f'Text is: "{text_to_summarize}". Write only summary itself.'
- messages = [
- ChatMessage.system().with_reasoning_effort(ReasoningEffort.Disabled),
- ChatMessage.user().with_text(prompt),
- ]
-
- chat_config = ChatConfig.create().with_speculation_preset(ChatSpeculationPreset.Summarization())
- session = await engine.chat(model, chat_config)
-
- chat_reply_config = ChatReplyConfig.create().with_token_limit(256).with_sampling_method(SamplingMethod.Greedy())
- replies = await session.reply(messages, chat_reply_config)
- if replies:
- reply = replies[0]
- print(f"Summary: {reply.message.text}")
- print(f"Generation t/s: {reply.stats.generate_tokens_per_second}")
-
-
-if __name__ == "__main__":
- asyncio.run(main())
-```
-
-<br>You will notice that the model’s run count is lower than the actual number of generated tokens due to speculative decoding, which significantly improves generation speed.
-
### Chat with structured output
Sometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:
diff --git a/bindings/python/examples/chat_speculation_classification.py b/bindings/python/examples/chat_speculation_classification.py
deleted file mode 100644
index 2be88e865..000000000
--- a/bindings/python/examples/chat_speculation_classification.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import asyncio
-
-from uzu import (
- ChatConfig,
- ChatMessage,
- ChatReplyConfig,
- ChatSpeculationPreset,
- Engine,
- EngineConfig,
- Feature,
- ReasoningEffort,
- SamplingMethod,
-)
-
-
-async def main() -> None:
- engine_config = EngineConfig.create()
- engine = await Engine.create(engine_config)
-
- model = await engine.model("Qwen/Qwen3-0.6B")
- if model is None:
- raise RuntimeError("Model not found")
- async for update in (await engine.download(model)).iterator():
- print(f"Download progress: {update.progress}")
-
- feature = Feature(
- "sentiment",
- ["Happy", "Sad", "Angry", "Fearful", "Surprised", "Disgusted"],
- )
- chat_config = ChatConfig.create().with_speculation_preset(ChatSpeculationPreset.Classification(feature))
- session = await engine.chat(model, chat_config)
-
- text_to_detect_feature = "Today's been awesome! Everything just feels right, and I can't stop smiling."
- prompt = (
- f'Text is: "{text_to_detect_feature}". '
- f"Choose {feature.name} from the list: {', '.join(feature.values)}. "
- "Answer with one word. Don't add a dot at the end."
- )
- messages = [
- ChatMessage.system().with_reasoning_effort(ReasoningEffort.Disabled),
- ChatMessage.user().with_text(prompt),
- ]
-
- chat_reply_config = ChatReplyConfig.create().with_token_limit(32).with_sampling_method(SamplingMethod.Greedy())
- replies = await session.reply(messages, chat_reply_config)
- if replies:
- reply = replies[0]
- print(f"Prediction: {reply.message.text}")
- print(f"Generated tokens: {reply.stats.tokens_count_output}")
-
-
-if __name__ == "__main__":
- asyncio.run(main())
diff --git a/bindings/python/examples/chat_speculation_summarization.py b/bindings/python/examples/chat_speculation_summarization.py
deleted file mode 100644
index a99e57ba8..000000000
--- a/bindings/python/examples/chat_speculation_summarization.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import asyncio
-
-from uzu import (
- ChatConfig,
- ChatMessage,
- ChatReplyConfig,
- ChatSpeculationPreset,
- Engine,
- EngineConfig,
- ReasoningEffort,
- SamplingMethod,
-)
-
-
-async def main() -> None:
- engine_config = EngineConfig.create()
- engine = await Engine.create(engine_config)
-
- model = await engine.model("Qwen/Qwen3-0.6B")
- if model is None:
- raise RuntimeError("Model not found")
- async for update in (await engine.download(model)).iterator():
- print(f"Download progress: {update.progress}")
-
- text_to_summarize = (
- "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. "
- "It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. "
- "LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. "
- "These models have a wide range of applications, including chatbots, content creation, translation, and code generation. "
- "One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. "
- "They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. "
- "Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. "
- "As these models grow in size and sophistication, they continue to enhance human-computer interactions, "
- "making AI-powered communication more natural and effective."
- )
- prompt = f'Text is: "{text_to_summarize}". Write only summary itself.'
- messages = [
- ChatMessage.system().with_reasoning_effort(ReasoningEffort.Disabled),
- ChatMessage.user().with_text(prompt),
- ]
-
- chat_config = ChatConfig.create().with_speculation_preset(ChatSpeculationPreset.Summarization())
- session = await engine.chat(model, chat_config)
-
- chat_reply_config = ChatReplyConfig.create().with_token_limit(256).with_sampling_method(SamplingMethod.Greedy())
- replies = await session.reply(messages, chat_reply_config)
- if replies:
- reply = replies[0]
- print(f"Summary: {reply.message.text}")
- print(f"Generation t/s: {reply.stats.generate_tokens_per_second}")
-
-
-if __name__ == "__main__":
- asyncio.run(main())
diff --git a/bindings/python/uzu/__init__.pyi b/bindings/python/uzu/__init__.pyi
index 31b962c90..deb19f14c 100644
--- a/bindings/python/uzu/__init__.pyi
+++ b/bindings/python/uzu/__init__.pyi
@@ -24,7 +24,6 @@ __all__ = [
"ChatSessionState",
"ChatSessionStream",
"ChatSessionStreamChunk",
- "ChatSpeculationPreset",
"ClassificationLabel",
"ClassificationMessage",
"ClassificationOutput",
@@ -102,14 +101,11 @@ class ChatConfig:
def context_length(self) -> ContextLength: ...
@property
def sampling_seed(self) -> SamplingSeed: ...
- @property
- def speculation_preset(self) -> typing.Optional[ChatSpeculationPreset]: ...
- def __new__(cls, context_length: ContextLength, sampling_seed: SamplingSeed, speculation_preset: typing.Optional[ChatSpeculationPreset]) -> ChatConfig: ...
+ def __new__(cls, context_length: ContextLength, sampling_seed: SamplingSeed) -> ChatConfig: ...
@staticmethod
def create() -> ChatConfig: ...
def with_context_length(self, context_length: ContextLength) -> ChatConfig: ...
def with_sampling_seed(self, sampling_seed: SamplingSeed) -> ChatConfig: ...
- def with_speculation_preset(self, speculation_preset: typing.Optional[ChatSpeculationPreset]) -> ChatConfig: ...
class ChatContentBlock:
@typing.final
@@ -443,26 +439,6 @@ class ChatSessionStreamChunk:
...
-class ChatSpeculationPreset:
- @typing.final
- class GeneralChat(ChatSpeculationPreset):
- __match_args__ = ()
- def __new__(cls) -> ChatSpeculationPreset.GeneralChat: ...
-
- @typing.final
- class Summarization(ChatSpeculationPreset):
- __match_args__ = ()
- def __new__(cls) -> ChatSpeculationPreset.Summarization: ...
-
- @typing.final
- class Classification(ChatSpeculationPreset):
- __match_args__ = ("feature",)
- @property
- def feature(self) -> Feature: ...
- def __new__(cls, feature: Feature) -> ChatSpeculationPreset.Classification: ...
-
- ...
-
@typing.final
class ClassificationLabel:
@property
diff --git a/bindings/swift/README.md b/bindings/swift/README.md
index 6adb19569..287ce28a9 100644
--- a/bindings/swift/README.md
+++ b/bindings/swift/README.md
@@ -69,7 +69,7 @@ Everything from model downloading to inference configuration is handled automati
## Examples
-You can run any example via `cargo tools example` \<**swift**\> \<**chat** | **chat-cloud** | **chat-speculation-classification** | **chat-speculation-summarization** | **chat-structured-output** | **classification** | **quick-start** | **text-to-speech**\>:
+You can run any example via `cargo tools example` \<**swift**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **text-to-speech**\>:
### Chat
@@ -144,97 +144,6 @@ public func runChatCloud() async throws {
}
```
-### Chat using speculation preset for classification
-
-In this example, we will use the `classification` speculation preset to determine the sentiment of the user's input:
-
-```swift
-import Uzu
-
-public func runChatSpeculationClassification() async throws {
- let engineConfig = EngineConfig.create()
- let engine = try await Engine.create(config: engineConfig)
-
- guard let model = try await engine.model(identifier: "Qwen/Qwen3-0.6B") else {
- return
- }
- for try await update in try await engine.download(model: model).iterator() {
- print("Download progress: \(update.progress())")
- }
-
- let feature = Feature(name: "sentiment", values: [
- "Happy",
- "Sad",
- "Angry",
- "Fearful",
- "Surprised",
- "Disgusted",
- ])
- let chatConfig = ChatConfig.create().withSpeculationPreset(speculationPreset: .classification(feature: feature))
- let session = try await engine.chat(model: model, config: chatConfig)
-
- let textToDetectFeature =
- "Today's been awesome! Everything just feels right, and I can't stop smiling."
- let prompt = "Text is: \"\(textToDetectFeature)\". Choose \(feature.name) from the list: \(feature.values.joined(separator: ", ")). Answer with one word. Don't add a dot at the end."
- let messages = [
- ChatMessage.system().withReasoningEffort(reasoningEffort: .disabled),
- ChatMessage.user().withText(text: prompt)
- ]
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(tokenLimit: 32).withSamplingMethod(samplingMethod: .greedy)
- let replies = try await session.reply(input: messages, config: chatReplyConfig)
- guard let reply = replies.last else {
- return
- }
-
- print("Prediction: \(reply.message.text() ?? "empty")")
- print("Generated tokens: \(reply.stats.tokensCountOutput ?? 0)")
-}
-```
-
-<br>You can view the stats to see that the answer will be ready immediately after the prefill step, and actual generation won’t even start due to speculative decoding, which significantly improves generation speed.
-
-### Chat using speculation preset for summarization
-
-In this example, we will use the `summarization` speculation preset to generate a summary of the input text:
-
-```swift
-import Uzu
-
-public func runChatSpeculationSummarization() async throws {
- let engineConfig = EngineConfig.create()
- let engine = try await Engine.create(config: engineConfig)
-
- guard let model = try await engine.model(identifier: "Qwen/Qwen3-0.6B") else {
- return
- }
- for try await update in try await engine.download(model: model).iterator() {
- print("Download progress: \(update.progress())")
- }
-
- let textToSummarize = "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. These models have a wide range of applications, including chatbots, content creation, translation, and code generation. One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. As these models grow in size and sophistication, they continue to enhance human-computer interactions, making AI-powered communication more natural and effective.";
- let prompt = "Text is: \"\(textToSummarize)\". Write only summary itself."
- let messages = [
- ChatMessage.system().withReasoningEffort(reasoningEffort: .disabled),
- ChatMessage.user().withText(text: prompt)
- ]
-
- let chatConfig = ChatConfig.create().withSpeculationPreset(speculationPreset: .summarization)
- let session = try await engine.chat(model: model, config: chatConfig)
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(tokenLimit: 256).withSamplingMethod(samplingMethod: .greedy)
- let replies = try await session.reply(input: messages, config: chatReplyConfig)
- guard let reply = replies.last else {
- return
- }
-
- print("Summary: \(reply.message.text() ?? "empty")")
- print("Generation t\\s: \(reply.stats.generateTokensPerSecond ?? 0.0)")
-}
-```
-
-<br>You will notice that the model’s run count is lower than the actual number of generated tokens due to speculative decoding, which significantly improves generation speed.
-
### Chat with structured output
Sometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:
diff --git a/bindings/swift/Sources/Examples/CLI.swift b/bindings/swift/Sources/Examples/CLI.swift
index 63435bb8f..e51545a1b 100644
--- a/bindings/swift/Sources/Examples/CLI.swift
+++ b/bindings/swift/Sources/Examples/CLI.swift
@@ -26,10 +26,6 @@ struct Example: AsyncParsableCommand {
try await runChatCloud()
case "chat-structured-output":
try await runChatStructuredOutput()
- case "chat-speculation-summarization":
- try await runChatSpeculationSummarization()
- case "chat-speculation-classification":
- try await runChatSpeculationClassification()
case "classification":
try await runClassification()
case "text-to-speech":
diff --git a/bindings/swift/Sources/Examples/ChatSpeculationClassification.swift b/bindings/swift/Sources/Examples/ChatSpeculationClassification.swift
deleted file mode 100644
index 32d424545..000000000
--- a/bindings/swift/Sources/Examples/ChatSpeculationClassification.swift
+++ /dev/null
@@ -1,41 +0,0 @@
-import Uzu
-
-public func runChatSpeculationClassification() async throws {
- let engineConfig = EngineConfig.create()
- let engine = try await Engine.create(config: engineConfig)
-
- guard let model = try await engine.model(identifier: "Qwen/Qwen3-0.6B") else {
- return
- }
- for try await update in try await engine.download(model: model).iterator() {
- print("Download progress: \(update.progress())")
- }
-
- let feature = Feature(name: "sentiment", values: [
- "Happy",
- "Sad",
- "Angry",
- "Fearful",
- "Surprised",
- "Disgusted",
- ])
- let chatConfig = ChatConfig.create().withSpeculationPreset(speculationPreset: .classification(feature: feature))
- let session = try await engine.chat(model: model, config: chatConfig)
-
- let textToDetectFeature =
- "Today's been awesome! Everything just feels right, and I can't stop smiling."
- let prompt = "Text is: \"\(textToDetectFeature)\". Choose \(feature.name) from the list: \(feature.values.joined(separator: ", ")). Answer with one word. Don't add a dot at the end."
- let messages = [
- ChatMessage.system().withReasoningEffort(reasoningEffort: .disabled),
- ChatMessage.user().withText(text: prompt)
- ]
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(tokenLimit: 32).withSamplingMethod(samplingMethod: .greedy)
- let replies = try await session.reply(input: messages, config: chatReplyConfig)
- guard let reply = replies.last else {
- return
- }
-
- print("Prediction: \(reply.message.text() ?? "empty")")
- print("Generated tokens: \(reply.stats.tokensCountOutput ?? 0)")
-}
diff --git a/bindings/swift/Sources/Examples/ChatSpeculationSummarization.swift b/bindings/swift/Sources/Examples/ChatSpeculationSummarization.swift
deleted file mode 100644
index 877d8a7b8..000000000
--- a/bindings/swift/Sources/Examples/ChatSpeculationSummarization.swift
+++ /dev/null
@@ -1,32 +0,0 @@
-import Uzu
-
-public func runChatSpeculationSummarization() async throws {
- let engineConfig = EngineConfig.create()
- let engine = try await Engine.create(config: engineConfig)
-
- guard let model = try await engine.model(identifier: "Qwen/Qwen3-0.6B") else {
- return
- }
- for try await update in try await engine.download(model: model).iterator() {
- print("Download progress: \(update.progress())")
- }
-
- let textToSummarize = "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. These models have a wide range of applications, including chatbots, content creation, translation, and code generation. One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. As these models grow in size and sophistication, they continue to enhance human-computer interactions, making AI-powered communication more natural and effective.";
- let prompt = "Text is: \"\(textToSummarize)\". Write only summary itself."
- let messages = [
- ChatMessage.system().withReasoningEffort(reasoningEffort: .disabled),
- ChatMessage.user().withText(text: prompt)
- ]
-
- let chatConfig = ChatConfig.create().withSpeculationPreset(speculationPreset: .summarization)
- let session = try await engine.chat(model: model, config: chatConfig)
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(tokenLimit: 256).withSamplingMethod(samplingMethod: .greedy)
- let replies = try await session.reply(input: messages, config: chatReplyConfig)
- guard let reply = replies.last else {
- return
- }
-
- print("Summary: \(reply.message.text() ?? "empty")")
- print("Generation t\\s: \(reply.stats.generateTokensPerSecond ?? 0.0)")
-}
diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
index 23a3fd675..146d6d5b6 100644
--- a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
+++ b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
@@ -658,14 +658,12 @@ public func FfiConverterTypeCancelToken_lower(_ value: CancelToken) -> UInt64 {
public struct ChatConfig: Equatable, Hashable, Codable {
public var contextLength: ContextLength
public var samplingSeed: SamplingSeed
- public var speculationPreset: ChatSpeculationPreset?
// Default memberwise initializers are never public by default, so we
// declare one manually.
- public init(contextLength: ContextLength, samplingSeed: SamplingSeed, speculationPreset: ChatSpeculationPreset?) {
+ public init(contextLength: ContextLength, samplingSeed: SamplingSeed) {
self.contextLength = contextLength
self.samplingSeed = samplingSeed
- self.speculationPreset = speculationPreset
}
@@ -687,15 +685,6 @@ public func withSamplingSeed(samplingSeed: SamplingSeed) -> ChatConfig {
})
}
-public func withSpeculationPreset(speculationPreset: ChatSpeculationPreset?) -> ChatConfig {
- return try! FfiConverterTypeChatConfig_lift(try! rustCall() {
- uniffi_shoji_fn_method_chatconfig_with_speculation_preset(
- FfiConverterTypeChatConfig_lower(self),
- FfiConverterOptionTypeChatSpeculationPreset.lower(speculationPreset),$0
- )
-})
-}
-
}
@@ -712,15 +701,13 @@ public struct FfiConverterTypeChatConfig: FfiConverterRustBuffer {
return
try ChatConfig(
contextLength: FfiConverterTypeContextLength.read(from: &buf),
- samplingSeed: FfiConverterTypeSamplingSeed.read(from: &buf),
- speculationPreset: FfiConverterOptionTypeChatSpeculationPreset.read(from: &buf)
+ samplingSeed: FfiConverterTypeSamplingSeed.read(from: &buf)
)
}
public static func write(_ value: ChatConfig, into buf: inout [UInt8]) {
FfiConverterTypeContextLength.write(value.contextLength, into: &buf)
FfiConverterTypeSamplingSeed.write(value.samplingSeed, into: &buf)
- FfiConverterOptionTypeChatSpeculationPreset.write(value.speculationPreset, into: &buf)
}
}
@@ -3758,83 +3745,6 @@ public func FfiConverterTypeChatRole_lower(_ value: ChatRole) -> RustBuffer {
}
-// Note that we don't yet support `indirect` for enums.
-// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.
-
-public enum ChatSpeculationPreset: Equatable, Hashable, Codable {
-
- case generalChat
- case summarization
- case classification(feature: Feature
- )
-
-
-
-
-
-}
-
-#if compiler(>=6)
-extension ChatSpeculationPreset: Sendable {}
-#endif
-
-#if swift(>=5.8)
-@_documentation(visibility: private)
-#endif
-public struct FfiConverterTypeChatSpeculationPreset: FfiConverterRustBuffer {
- typealias SwiftType = ChatSpeculationPreset
-
- public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ChatSpeculationPreset {
- let variant: Int32 = try readInt(&buf)
- switch variant {
-
- case 1: return .generalChat
-
- case 2: return .summarization
-
- case 3: return .classification(feature: try FfiConverterTypeFeature.read(from: &buf)
- )
-
- default: throw UniffiInternalError.unexpectedEnumCase
- }
- }
-
- public static func write(_ value: ChatSpeculationPreset, into buf: inout [UInt8]) {
- switch value {
-
-
- case .generalChat:
- writeInt(&buf, Int32(1))
-
-
- case .summarization:
- writeInt(&buf, Int32(2))
-
-
- case let .classification(feature):
- writeInt(&buf, Int32(3))
- FfiConverterTypeFeature.write(feature, into: &buf)
-
- }
- }
-}
-
-
-#if swift(>=5.8)
-@_documentation(visibility: private)
-#endif
-public func FfiConverterTypeChatSpeculationPreset_lift(_ buf: RustBuffer) throws -> ChatSpeculationPreset {
- return try FfiConverterTypeChatSpeculationPreset.lift(buf)
-}
-
-#if swift(>=5.8)
-@_documentation(visibility: private)
-#endif
-public func FfiConverterTypeChatSpeculationPreset_lower(_ value: ChatSpeculationPreset) -> RustBuffer {
- return FfiConverterTypeChatSpeculationPreset.lower(value)
-}
-
-
// Note that we don't yet support `indirect` for enums.
// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.
@@ -5315,30 +5225,6 @@ fileprivate struct FfiConverterOptionTypeChatReplyFinishReason: FfiConverterRust
}
}
-#if swift(>=5.8)
-@_documentation(visibility: private)
-#endif
-fileprivate struct FfiConverterOptionTypeChatSpeculationPreset: FfiConverterRustBuffer {
- typealias SwiftType = ChatSpeculationPreset?
-
- public static func write(_ value: SwiftType, into buf: inout [UInt8]) {
- guard let value = value else {
- writeInt(&buf, Int8(0))
- return
- }
- writeInt(&buf, Int8(1))
- FfiConverterTypeChatSpeculationPreset.write(value, into: &buf)
- }
-
- public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {
- switch try readInt(&buf) as Int8 {
- case 0: return nil
- case 1: return try FfiConverterTypeChatSpeculationPreset.read(from: &buf)
- default: throw UniffiInternalError.unexpectedOptionalTag
- }
- }
-}
-
#if swift(>=5.8)
@_documentation(visibility: private)
#endif
diff --git a/bindings/typescript/README.md b/bindings/typescript/README.md
index 4a17fa62b..c655e819e 100644
--- a/bindings/typescript/README.md
+++ b/bindings/typescript/README.md
@@ -72,7 +72,7 @@ Everything from model downloading to inference configuration is handled automati
## Examples
-You can run any example via `cargo tools example` \<**typescript**\> \<**chat** | **chat-cloud** | **chat-speculation-classification** | **chat-speculation-summarization** | **chat-structured-output** | **classification** | **quick-start** | **text-to-speech**\>:
+You can run any example via `cargo tools example` \<**typescript**\> \<**chat** | **chat-cloud** | **chat-structured-output** | **classification** | **quick-start** | **text-to-speech**\>:
### Chat
@@ -154,108 +154,6 @@ main().catch((error) => {
});
```
-### Chat using speculation preset for classification
-
-In this example, we will use the `classification` speculation preset to determine the sentiment of the user's input:
-
-```ts
-import { ChatConfig, ChatMessage, ChatReplyConfig, ChatSpeculationPresetClassification, Engine, EngineConfig, Feature, ReasoningEffort, SamplingMethodGreedy } from '@trymirai/uzu';
-
-async function main() {
- let engineConfig = EngineConfig.create();
- let engine = await Engine.create(engineConfig);
-
- let model = await engine.model('Qwen/Qwen3-0.6B');
- if (!model) {
- throw new Error('Model not found');
- }
- for await (const update of await engine.download(model)) {
- console.log('Download progress:', update.progress);
- }
-
- const feature = new Feature('sentiment', [
- 'Happy',
- 'Sad',
- 'Angry',
- 'Fearful',
- 'Surprised',
- 'Disgusted',
- ]);
- let chatConfig = ChatConfig.create().withSpeculationPreset(new ChatSpeculationPresetClassification(feature));
- let session = await engine.chat(model, chatConfig);
-
- const textToDetectFeature =
- "Today's been awesome! Everything just feels right, and I can't stop smiling.";
- const prompt =
- `Text is: "${textToDetectFeature}". Choose ${feature.name} from the list: ${feature.values.join(', ')}. ` +
- "Answer with one word. Don't add a dot at the end.";
- let messages = [
- ChatMessage.system().withReasoningEffort("Disabled" as ReasoningEffort),
- ChatMessage.user().withText(prompt)
- ];
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(32).withSamplingMethod(new SamplingMethodGreedy());
- let reply = (await session.reply(messages, chatReplyConfig))[0];
-
- if (reply) {
- console.log('Prediction: ', reply.message.text);
- console.log('Generated tokens: ', reply.stats.tokensCountOutput);
- }
-}
-
-main().catch((error) => {
- console.error(error);
-});
-```
-
-<br>You can view the stats to see that the answer will be ready immediately after the prefill step, and actual generation won’t even start due to speculative decoding, which significantly improves generation speed.
-
-### Chat using speculation preset for summarization
-
-In this example, we will use the `summarization` speculation preset to generate a summary of the input text:
-
-```ts
-import { ChatConfig, ChatMessage, ChatReplyConfig, ChatSpeculationPresetSummarization, Engine, EngineConfig, ReasoningEffort, SamplingMethodGreedy } from '@trymirai/uzu';
-
-async function main() {
- let engineConfig = EngineConfig.create();
- let engine = await Engine.create(engineConfig);
-
- let model = await engine.model('Qwen/Qwen3-0.6B');
- if (!model) {
- throw new Error('Model not found');
- }
- for await (const update of await engine.download(model)) {
- console.log('Download progress:', update.progress);
- }
-
- const textToSummarize =
- "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. These models have a wide range of applications, including chatbots, content creation, translation, and code generation. One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. As these models grow in size and sophistication, they continue to enhance human-computer interactions, making AI-powered communication more natural and effective.";
- const prompt = `Text is: "${textToSummarize}". Write only summary itself.`;
- let messages = [
- ChatMessage.system().withReasoningEffort("Disabled" as ReasoningEffort),
- ChatMessage.user().withText(prompt)
- ];
-
- let chatConfig = ChatConfig.create().withSpeculationPreset(new ChatSpeculationPresetSummarization);
- let session = await engine.chat(model, chatConfig);
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(256).withSamplingMethod(new SamplingMethodGreedy());
- let reply = (await session.reply(messages, chatReplyConfig))[0];
-
- if (reply) {
- console.log('Summary: ', reply.message.text);
- console.log('Generation t\\s: ', reply.stats.generateTokensPerSecond);
- }
-}
-
-main().catch((error) => {
- console.error(error);
-});
-```
-
-<br>You will notice that the model’s run count is lower than the actual number of generated tokens due to speculative decoding, which significantly improves generation speed.
-
### Chat with structured output
Sometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:
diff --git a/bindings/typescript/examples/chatSpeculationClassification.ts b/bindings/typescript/examples/chatSpeculationClassification.ts
deleted file mode 100644
index 7a41528cb..000000000
--- a/bindings/typescript/examples/chatSpeculationClassification.ts
+++ /dev/null
@@ -1,47 +0,0 @@
-import { ChatConfig, ChatMessage, ChatReplyConfig, ChatSpeculationPresetClassification, Engine, EngineConfig, Feature, ReasoningEffort, SamplingMethodGreedy } from '@trymirai/uzu';
-
-async function main() {
- let engineConfig = EngineConfig.create();
- let engine = await Engine.create(engineConfig);
-
- let model = await engine.model('Qwen/Qwen3-0.6B');
- if (!model) {
- throw new Error('Model not found');
- }
- for await (const update of await engine.download(model)) {
- console.log('Download progress:', update.progress);
- }
-
- const feature = new Feature('sentiment', [
- 'Happy',
- 'Sad',
- 'Angry',
- 'Fearful',
- 'Surprised',
- 'Disgusted',
- ]);
- let chatConfig = ChatConfig.create().withSpeculationPreset(new ChatSpeculationPresetClassification(feature));
- let session = await engine.chat(model, chatConfig);
-
- const textToDetectFeature =
- "Today's been awesome! Everything just feels right, and I can't stop smiling.";
- const prompt =
- `Text is: "${textToDetectFeature}". Choose ${feature.name} from the list: ${feature.values.join(', ')}. ` +
- "Answer with one word. Don't add a dot at the end.";
- let messages = [
- ChatMessage.system().withReasoningEffort("Disabled" as ReasoningEffort),
- ChatMessage.user().withText(prompt)
- ];
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(32).withSamplingMethod(new SamplingMethodGreedy());
- let reply = (await session.reply(messages, chatReplyConfig))[0];
-
- if (reply) {
- console.log('Prediction: ', reply.message.text);
- console.log('Generated tokens: ', reply.stats.tokensCountOutput);
- }
-}
-
-main().catch((error) => {
- console.error(error);
-});
diff --git a/bindings/typescript/examples/chatSpeculationSummarization.ts b/bindings/typescript/examples/chatSpeculationSummarization.ts
deleted file mode 100644
index 9597d9914..000000000
--- a/bindings/typescript/examples/chatSpeculationSummarization.ts
+++ /dev/null
@@ -1,37 +0,0 @@
-import { ChatConfig, ChatMessage, ChatReplyConfig, ChatSpeculationPresetSummarization, Engine, EngineConfig, ReasoningEffort, SamplingMethodGreedy } from '@trymirai/uzu';
-
-async function main() {
- let engineConfig = EngineConfig.create();
- let engine = await Engine.create(engineConfig);
-
- let model = await engine.model('Qwen/Qwen3-0.6B');
- if (!model) {
- throw new Error('Model not found');
- }
- for await (const update of await engine.download(model)) {
- console.log('Download progress:', update.progress);
- }
-
- const textToSummarize =
- "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. These models have a wide range of applications, including chatbots, content creation, translation, and code generation. One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. As these models grow in size and sophistication, they continue to enhance human-computer interactions, making AI-powered communication more natural and effective.";
- const prompt = `Text is: "${textToSummarize}". Write only summary itself.`;
- let messages = [
- ChatMessage.system().withReasoningEffort("Disabled" as ReasoningEffort),
- ChatMessage.user().withText(prompt)
- ];
-
- let chatConfig = ChatConfig.create().withSpeculationPreset(new ChatSpeculationPresetSummarization);
- let session = await engine.chat(model, chatConfig);
-
- let chatReplyConfig = ChatReplyConfig.create().withTokenLimit(256).withSamplingMethod(new SamplingMethodGreedy());
- let reply = (await session.reply(messages, chatReplyConfig))[0];
-
- if (reply) {
- console.log('Summary: ', reply.message.text);
- console.log('Generation t\\s: ', reply.stats.generateTokensPerSecond);
- }
-}
-
-main().catch((error) => {
- console.error(error);
-});
diff --git a/bindings/typescript/src/napi/index.d.mts b/bindings/typescript/src/napi/index.d.mts
index 3e7ec8b24..eedca445e 100644
--- a/bindings/typescript/src/napi/index.d.mts
+++ b/bindings/typescript/src/napi/index.d.mts
@@ -113,13 +113,11 @@ export declare class CancelToken {
export declare class ChatConfig {
contextLength: ContextLength
samplingSeed: SamplingSeed
- speculationPreset?: ChatSpeculationPreset
- constructor(contextLength: ContextLength, samplingSeed: SamplingSeed, speculationPreset?: ChatSpeculationPreset)
+ constructor(contextLength: ContextLength, samplingSeed: SamplingSeed)
static create(): ChatConfig
withContextLength(contextLength: ContextLength): ChatConfig
withSamplingSeed(samplingSeed: SamplingSeed): ChatConfig
- withSpeculationPreset(speculationPreset?: ChatSpeculationPreset | undefined | null): ChatConfig
}
export declare class ChatContentBlockAudio {
@@ -326,21 +324,6 @@ export declare class ChatRoleUser {
constructor()
}
-export declare class ChatSpeculationPresetClassification {
- feature: Feature
- constructor(feature: Feature)
-}
-
-export declare class ChatSpeculationPresetGeneralChat {
-
- constructor()
-}
-
-export declare class ChatSpeculationPresetSummarization {
-
- constructor()
-}
-
export declare class ClassificationLabel {
index: number
label: string
@@ -739,9 +722,6 @@ export declare const enum ChatReplyFinishReason {
export type ChatRole =
ChatRoleUser | ChatRoleAssistant | ChatRoleSystem | ChatRoleDeveloper | ChatRoleTool | ChatRoleCustom
-export type ChatSpeculationPreset =
- ChatSpeculationPresetGeneralChat | ChatSpeculationPresetSummarization | ChatSpeculationPresetClassification
-
export interface ClassificationOutputProbabilities {
values: Record<string, number>
}
diff --git a/bindings/typescript/src/napi/index.d.ts b/bindings/typescript/src/napi/index.d.ts
index 3e7ec8b24..eedca445e 100644
--- a/bindings/typescript/src/napi/index.d.ts
+++ b/bindings/typescript/src/napi/index.d.ts
@@ -113,13 +113,11 @@ export declare class CancelToken {
export declare class ChatConfig {
contextLength: ContextLength
samplingSeed: SamplingSeed
- speculationPreset?: ChatSpeculationPreset
- constructor(contextLength: ContextLength, samplingSeed: SamplingSeed, speculationPreset?: ChatSpeculationPreset)
+ constructor(contextLength: ContextLength, samplingSeed: SamplingSeed)
static create(): ChatConfig
withContextLength(contextLength: ContextLength): ChatConfig
withSamplingSeed(samplingSeed: SamplingSeed): ChatConfig
- withSpeculationPreset(speculationPreset?: ChatSpeculationPreset | undefined | null): ChatConfig
}
export declare class ChatContentBlockAudio {
@@ -326,21 +324,6 @@ export declare class ChatRoleUser {
constructor()
}
-export declare class ChatSpeculationPresetClassification {
- feature: Feature
- constructor(feature: Feature)
-}
-
-export declare class ChatSpeculationPresetGeneralChat {
-
- constructor()
-}
-
-export declare class ChatSpeculationPresetSummarization {
-
- constructor()
-}
-
export declare class ClassificationLabel {
index: number
label: string
@@ -739,9 +722,6 @@ export declare const enum ChatReplyFinishReason {
export type ChatRole =
ChatRoleUser | ChatRoleAssistant | ChatRoleSystem | ChatRoleDeveloper | ChatRoleTool | ChatRoleCustom
-export type ChatSpeculationPreset =
- ChatSpeculationPresetGeneralChat | ChatSpeculationPresetSummarization | ChatSpeculationPresetClassification
-
export interface ClassificationOutputProbabilities {
values: Record<string, number>
}
diff --git a/bindings/typescript/src/napi/index.js b/bindings/typescript/src/napi/index.js
index b3447f3e2..8817a639d 100644
--- a/bindings/typescript/src/napi/index.js
+++ b/bindings/typescript/src/napi/index.js
@@ -552,9 +552,6 @@ module.exports.ChatRoleDeveloper = nativeBinding.ChatRoleDeveloper
module.exports.ChatRoleSystem = nativeBinding.ChatRoleSystem
module.exports.ChatRoleTool = nativeBinding.ChatRoleTool
module.exports.ChatRoleUser = nativeBinding.ChatRoleUser
-module.exports.ChatSpeculationPresetClassification = nativeBinding.ChatSpeculationPresetClassification
-module.exports.ChatSpeculationPresetGeneralChat = nativeBinding.ChatSpeculationPresetGeneralChat
-module.exports.ChatSpeculationPresetSummarization = nativeBinding.ChatSpeculationPresetSummarization
module.exports.ClassificationLabel = nativeBinding.ClassificationLabel
module.exports.ClassificationMessage = nativeBinding.ClassificationMessage
module.exports.ClassificationOutput = nativeBinding.ClassificationOutput
diff --git a/bindings/typescript/src/napi/index.mjs b/bindings/typescript/src/napi/index.mjs
index 79d673d55..ffff55107 100644
--- a/bindings/typescript/src/napi/index.mjs
+++ b/bindings/typescript/src/napi/index.mjs
@@ -49,9 +49,6 @@ export const ChatRoleDeveloper = cjs.ChatRoleDeveloper;
export const ChatRoleSystem = cjs.ChatRoleSystem;
export const ChatRoleTool = cjs.ChatRoleTool;
export const ChatRoleUser = cjs.ChatRoleUser;
-export const ChatSpeculationPresetClassification = cjs.ChatSpeculationPresetClassification;
-export const ChatSpeculationPresetGeneralChat = cjs.ChatSpeculationPresetGeneralChat;
-export const ChatSpeculationPresetSummarization = cjs.ChatSpeculationPresetSummarization;
export const ClassificationLabel = cjs.ClassificationLabel;
export const ClassificationMessage = cjs.ClassificationMessage;
export const ClassificationOutput = cjs.ClassificationOutput;
diff --git a/crates/backend-uzu/src/backends/common/kernel/radix_top_k_small.rs b/crates/backend-uzu/src/backends/common/kernel/radix_top_k_small.rs
index c8767fa00..2d67a4bca 100644
--- a/crates/backend-uzu/src/backends/common/kernel/radix_top_k_small.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/radix_top_k_small.rs
@@ -2,7 +2,7 @@ use crate::backends::common::{Allocation, Backend, Encoder};
pub const MAX_K: u32 = 512;
-pub trait RadixTopKSmall<B: Backend>: Sized {
+pub trait RadixTopKSmall<B: Backend>: Sized + Send + Sync {
fn new(
context: &B::Context,
columns: u32,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
index 24cbcd0eb..d03f8d74a 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
@@ -1,18 +1,24 @@
use half::bf16;
use proc_macros::kernel;
-use crate::backends::common::gpu_types::weaver::CANDIDATES_MAX;
+use crate::{
+ backends::common::gpu_types::weaver::{CANDIDATES_MAX, MetadataIdx},
+ encodable_block::sampling::{gumbel_float, revidx},
+};
#[kernel(WeaverTopChildren)]
pub fn weaver_top_children(
residual_logits: *const bf16,
candidate_logits: *const f32,
candidate_ids: *const u32,
+ depth_seeds: *const u64,
+ node_metadata: *const u32,
output_token_ids: *mut u32,
output_model_logprobs: *mut f32,
rows: u32,
candidates: u32,
children_per_node: u32,
+ vocab_size: u32,
) {
let rows = rows as usize;
let candidates = candidates as usize;
@@ -22,20 +28,24 @@ pub fn weaver_top_children(
}
for row in 0..rows {
let base = row * candidates;
+ let depth = unsafe { *node_metadata.add(MetadataIdx::Depth as usize * rows + row) } as usize;
+ let seed = unsafe { *depth_seeds.add(depth) };
+ let token = |index: usize| unsafe { *candidate_ids.add(base + index) };
let logits = (0..candidates)
.map(|index| unsafe { *candidate_logits.add(base + index) + (*residual_logits.add(base + index)).to_f32() })
.collect::<Vec<_>>();
let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let log_sum = logits.iter().map(|value| (value - max).exp()).sum::<f32>().ln() + max;
+ let perturbed = (0..candidates)
+ .map(|index| logits[index] + gumbel_float(seed, revidx(token(index), vocab_size)))
+ .collect::<Vec<_>>();
let mut indices = (0..candidates).collect::<Vec<_>>();
indices.sort_by(|&left, &right| {
- logits[right]
- .total_cmp(&logits[left])
- .then_with(|| unsafe { (*candidate_ids.add(base + left)).cmp(&*candidate_ids.add(base + right)) })
+ perturbed[right].total_cmp(&perturbed[left]).then_with(|| token(left).cmp(&token(right)))
});
for (rank, index) in indices.into_iter().take(children_per_node).enumerate() {
unsafe {
- *output_token_ids.add(row * children_per_node + rank) = *candidate_ids.add(base + index);
+ *output_token_ids.add(row * children_per_node + rank) = token(index);
*output_model_logprobs.add(row * children_per_node + rank) = logits[index] - log_sum;
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal
index e80de0bc5..7613b0fe0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal
@@ -1,11 +1,31 @@
#include <metal_stdlib>
+#include "../common/defines.h"
#include "../common/dsl.h"
#include "../common/thread_context.h"
#include "../common/top_k.h"
+#include "../rng.h"
#include "weaver_frontier.h"
using namespace metal;
+#define GUMBEL_THREADGROUP_SIZE 1024u
+#define GUMBEL_WORDS_PER_OFFSET 4u
+
+METAL_FUNC uint2 gumbel_revidx(uint logit_idx, uint vocab_size) {
+ const uint thread_idx = logit_idx % GUMBEL_THREADGROUP_SIZE;
+ const uint thread_offset = div_ceil(vocab_size, GUMBEL_THREADGROUP_SIZE * GUMBEL_WORDS_PER_OFFSET) * thread_idx;
+ const uint block_idx = logit_idx / GUMBEL_THREADGROUP_SIZE;
+ return uint2(thread_offset + block_idx / GUMBEL_WORDS_PER_OFFSET, block_idx % GUMBEL_WORDS_PER_OFFSET);
+}
+
+METAL_FUNC float gumbel_noise(uint64_t seed, uint logit_idx, uint vocab_size) {
+ const uint2 offset_word = gumbel_revidx(logit_idx, vocab_size);
+ PhiloxState rng;
+ philox_init(&rng, seed, offset_word.x);
+ const float uniform = float(rng.output[offset_word.y]) * (1.0f / 4294967296.0f);
+ return -log(-log(uniform));
+}
+
METAL_FUNC bool weaver_better(uint score, uint token, uint index, uint best_score, uint best_token, uint best_index) {
return score > best_score ||
(score == best_score && (token < best_token || (token == best_token && index < best_index)));
@@ -15,18 +35,20 @@ PUBLIC KERNEL(WeaverTopChildren)(
const device bfloat* residual_logits,
const device float* candidate_logits,
const device uint* candidate_ids,
+ const device uint64_t* depth_seeds,
+ const device uint* node_metadata,
device uint* output_token_ids,
device float* output_model_logprobs,
constant uint& rows,
constant uint& candidates,
constant uint& children_per_node,
+ constant uint& vocab_size,
threadgroup float reduce_float[TOP_CHILDREN_SIMDGROUPS],
threadgroup uint reduce_score[TOP_CHILDREN_SIMDGROUPS],
threadgroup uint reduce_token[TOP_CHILDREN_SIMDGROUPS],
threadgroup uint reduce_index[TOP_CHILDREN_SIMDGROUPS],
threadgroup float& logit_max,
threadgroup float& log_sum,
- threadgroup uint& winner_score,
threadgroup uint& winner_token,
threadgroup uint& winner_index,
const ThreadContext thread_context,
@@ -38,6 +60,8 @@ PUBLIC KERNEL(WeaverTopChildren)(
}
const uint base = row * candidates;
+ const uint depth = node_metadata[uint(MetadataIdx::Depth) * rows + row];
+ const uint64_t seed = depth_seeds[depth];
const uint first_index = lid;
const uint second_index = lid + TOP_CHILDREN_THREADS;
const bool first_valid = first_index < candidates;
@@ -48,8 +72,10 @@ PUBLIC KERNEL(WeaverTopChildren)(
second_valid ? candidate_logits[base + second_index] + float(residual_logits[base + second_index]) : -INFINITY;
const uint first_token = first_valid ? uint(candidate_ids[base + first_index]) : 0xffffffffu;
const uint second_token = second_valid ? uint(candidate_ids[base + second_index]) : 0xffffffffu;
- const uint first_score = first_valid ? top_k_score_key(first_logit) : 0u;
- const uint second_score = second_valid ? top_k_score_key(second_logit) : 0u;
+ const uint first_score =
+ first_valid ? top_k_score_key(first_logit + gumbel_noise(seed, first_token, vocab_size)) : 0u;
+ const uint second_score =
+ second_valid ? top_k_score_key(second_logit + gumbel_noise(seed, second_token, vocab_size)) : 0u;
bool first_active = first_valid;
bool second_active = second_valid;
@@ -125,7 +151,6 @@ PUBLIC KERNEL(WeaverTopChildren)(
: 0xffffffffu;
const uint selected_index = simd_min(group_index);
if (thread_context.simd_lane_id == 0) {
- winner_score = selected_score;
winner_token = selected_token;
winner_index = selected_index;
}
@@ -133,7 +158,7 @@ PUBLIC KERNEL(WeaverTopChildren)(
threadgroup_barrier(mem_flags::mem_threadgroup);
if (lid == 0) {
- const float winner_logit = top_k_score_from_key(winner_score);
+ const float winner_logit = candidate_logits[base + winner_index] + float(residual_logits[base + winner_index]);
output_token_ids[row * children_per_node + child] = winner_token;
output_model_logprobs[row * children_per_node + child] = winner_logit - log_sum;
}
diff --git a/crates/backend-uzu/src/bridge/backend.rs b/crates/backend-uzu/src/bridge/backend.rs
index 67b8c2e94..a8e3dda60 100644
--- a/crates/backend-uzu/src/bridge/backend.rs
+++ b/crates/backend-uzu/src/bridge/backend.rs
@@ -52,12 +52,17 @@ impl ChatTokenBackend for UzuLlmBackend {
&'a self,
reference: String,
config: ChatConfig,
- tokenizer: &'a Tokenizer,
+ #[cfg_attr(not(grammar), allow(unused_variables))] tokenizer: &'a Tokenizer,
) -> Pin<Box<dyn Future<Output = Result<Box<dyn ChatTokenInstance>, BackendError>> + Send + 'a>> {
Box::pin(async move {
let instance = select_backend!(
- UzuChatTokenBackendInstance::<B>::new(reference, config, tokenizer)
- .map(|i| Box::new(i) as Box<dyn ChatTokenInstance>),
+ UzuChatTokenBackendInstance::<B>::new(
+ reference,
+ config,
+ #[cfg(grammar)]
+ tokenizer,
+ )
+ .map(|i| Box::new(i) as Box<dyn ChatTokenInstance>),
BackendError::from("Unable to open any backend")
)?;
Ok(instance)
diff --git a/crates/backend-uzu/src/bridge/chat_token_backend.rs b/crates/backend-uzu/src/bridge/chat_token_backend.rs
index bf9a1f000..eae510420 100644
--- a/crates/backend-uzu/src/bridge/chat_token_backend.rs
+++ b/crates/backend-uzu/src/bridge/chat_token_backend.rs
@@ -21,6 +21,7 @@ use shoji::{
},
types::session::chat::{ChatConfig, ChatReplyConfig},
};
+#[cfg(grammar)]
use tokenizers::Tokenizer;
use tokio_util::sync::CancellationToken;
@@ -30,17 +31,16 @@ use crate::{
backends::common::Backend,
bridge::{
chat_token_state::UzuChatTokenBackendInstanceState,
- helpers::{error_stream, get_max_context_length, get_sampling_method, get_speculator},
+ helpers::{error_stream, get_max_context_length, get_sampling_method},
},
engine::{
Engine,
language_model::{
LanguageModel,
state::LanguageModelState,
- stream::{LanguageModelStream, LanguageModelStreamOptions, LanguageModelStreamSpeculatorOptions},
+ stream::{LanguageModelStream, LanguageModelStreamOptions},
},
},
- speculators::speculator::Speculator,
};
pub struct UzuChatTokenBackendInstance<B: Backend> {
@@ -50,7 +50,6 @@ pub struct UzuChatTokenBackendInstance<B: Backend> {
#[cfg(grammar)]
tokenizer: Tokenizer,
stop_token_ids: Vec<i32>,
- speculator: Option<(Box<dyn Speculator>, usize)>,
max_context_length: Option<usize>,
}
@@ -58,7 +57,7 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
pub fn new(
model_path: String,
config: ChatConfig,
- tokenizer: &Tokenizer,
+ #[cfg(grammar)] tokenizer: &Tokenizer,
) -> Result<Self, BackendError> {
let engine = Engine::<B>::new().map_err(|err| err.to_string())?;
let model_path = PathBuf::from(model_path);
@@ -66,12 +65,6 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
let stop_token_ids = model.generation_config().stop_token_ids.iter().map(|id| *id as i32).collect();
- let speculator = if let Some(preset) = config.speculation_preset.as_ref() {
- get_speculator(preset, tokenizer)?
- } else {
- None
- };
-
let max_context_length = get_max_context_length(&model, config.context_length.clone());
Ok(Self {
@@ -81,7 +74,6 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
#[cfg(grammar)]
tokenizer: tokenizer.clone(),
stop_token_ids,
- speculator,
max_context_length,
})
}
@@ -146,11 +138,6 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
sampling_method: get_sampling_method::<B>(&model_guard, &config.sampling_policy),
#[cfg(grammar)]
grammar,
- speculator: self.speculator.as_ref().map(|(speculator, budget)| LanguageModelStreamSpeculatorOptions {
- speculator: speculator.as_ref(),
- speculation_budget: *budget,
- trie_creation_config: Default::default(),
- }),
};
let stream = match model_guard.stream(input, &mut state_guard, options) {
diff --git a/crates/backend-uzu/src/bridge/helpers.rs b/crates/backend-uzu/src/bridge/helpers.rs
index 81f67e14f..c52d2375e 100644
--- a/crates/backend-uzu/src/bridge/helpers.rs
+++ b/crates/backend-uzu/src/bridge/helpers.rs
@@ -5,23 +5,16 @@ use futures::{Stream, stream};
use shoji::types::basic::Grammar as ShojiGrammar;
use shoji::{
traits::backend::{Error as BackendError, chat_token::StreamOutput as ChatTokenStreamOutput},
- types::{
- basic::{ContextLength, SamplingMethod as ShojiSamplingMethod, SamplingPolicy as ShojiSamplingPolicy},
- session::chat::ChatSpeculationPreset,
- },
+ types::basic::{ContextLength, SamplingMethod as ShojiSamplingMethod, SamplingPolicy as ShojiSamplingPolicy},
};
+#[cfg(grammar)]
use tokenizers::Tokenizer;
#[cfg(grammar)]
use crate::engine::language_model::grammar::{Grammar as UzuGrammar, GrammarConfig, GrammarError};
use crate::{
- backends::common::Backend,
- encodable_block::sampling::SamplingMethod as UzuSamplingMethod,
+ backends::common::Backend, encodable_block::sampling::SamplingMethod as UzuSamplingMethod,
engine::language_model::LanguageModel,
- speculators::{
- fixed_token_speculator::FixedTokensSpeculator, prompt_lookup_speculator::PromptLookupSpeculator,
- speculator::Speculator,
- },
};
pub fn error_stream<'a>(
@@ -101,36 +94,3 @@ pub fn get_sampling_method<B: Backend>(
},
}
}
-
-pub fn get_speculator(
- preset: &ChatSpeculationPreset,
- tokenizer: &Tokenizer,
-) -> Result<Option<(Box<dyn Speculator>, usize)>, BackendError> {
- match preset {
- ChatSpeculationPreset::GeneralChat {
- ..
- } => Ok(None),
- ChatSpeculationPreset::Summarization {
- ..
- } => {
- let speculator = Box::new(PromptLookupSpeculator::new_with_params(3));
- Ok(Some((speculator, 16)))
- },
- ChatSpeculationPreset::Classification {
- feature,
- } => {
- let proposals = feature
- .values
- .iter()
- .map(|value| {
- let encoding = tokenizer.encode(value.as_str(), false)?;
- let ids = encoding.get_ids().iter().map(|&id| id as u64).collect::<Vec<_>>();
- Ok(ids)
- })
- .collect::<Result<Vec<Vec<u64>>, BackendError>>()?;
- let speculator = Box::new(FixedTokensSpeculator::new(proposals));
- let budget = speculator.max_trie_nodes() + 1;
- Ok(Some((speculator, budget)))
- },
- }
-}
diff --git a/crates/backend-uzu/src/config/weaver.rs b/crates/backend-uzu/src/config/weaver.rs
index fbf4444e7..48f38cb2f 100644
--- a/crates/backend-uzu/src/config/weaver.rs
+++ b/crates/backend-uzu/src/config/weaver.rs
@@ -1,6 +1,6 @@
use proc_macros::uzu_config;
-use crate::config::normalization::NormalizationConfig;
+use crate::config::{linear::LinearConfig, normalization::NormalizationConfig};
#[uzu_config]
pub struct WeaverConfig {
@@ -12,5 +12,6 @@ pub struct WeaverConfig {
pub hidden_dim: usize,
pub max_depth: usize,
pub candidate_pool_size: usize,
+ pub linear_config: LinearConfig,
pub norm_config: NormalizationConfig,
}
diff --git a/crates/backend-uzu/src/encodable_block/classifier.rs b/crates/backend-uzu/src/encodable_block/classifier.rs
index 335869492..9c3991a3a 100644
--- a/crates/backend-uzu/src/encodable_block/classifier.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier.rs
@@ -131,7 +131,7 @@ impl<B: Backend> Classifier<B> {
.collect::<Box<[TrieNode]>>();
let hidden = self
.transformer
- .encode(hidden, None, &BatchTopology::new(&nodes, true), Some(0..batch_dim), None, encoder, &[])
+ .encode(hidden, None, &BatchTopology::new(&nodes, true), Some(0..batch_dim), None, None, encoder)
.map_err(ClassifierError::Backend)?
.output
.unwrap();
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index 0a0dd9b95..a33d7cc84 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -38,7 +38,7 @@ pub struct Decoder<B: Backend> {
pub struct DecoderEncodeOutput<B: Backend> {
pub logits: Option<Allocation<B>>,
#[allow(dead_code)]
- pub hidden_features: Box<[Allocation<B>]>,
+ pub hidden_features: Option<Box<[Allocation<B>]>>,
#[allow(dead_code)]
pub final_hidden: Option<Allocation<B>>,
}
@@ -120,9 +120,9 @@ impl<B: Backend> Decoder<B> {
token_ids: &Allocation<B>,
batch_dim: &BatchTopology,
output_range: Option<Range<usize>>,
+ hidden_feature_layer_indices: Option<&[usize]>,
state: &mut TransformerState<B>,
encoder: &mut Encoder<B>,
- hidden_feature_layer_indices: &[usize],
) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {
let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), encoder)?;
@@ -143,9 +143,9 @@ impl<B: Backend> Decoder<B> {
per_layer_inputs.as_ref(),
batch_dim,
output_range.clone(),
+ hidden_feature_layer_indices,
Some(state),
encoder,
- hidden_feature_layer_indices,
)
.map_err(DecoderError::Backend)?;
@@ -155,7 +155,7 @@ impl<B: Backend> Decoder<B> {
} else {
None
};
- let final_hidden = if hidden_feature_layer_indices.is_empty() {
+ let final_hidden = if hidden_feature_layer_indices.is_none() {
None
} else {
transformer_output.output
diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rs
index 74c3198da..55abe8edc 100644
--- a/crates/backend-uzu/src/encodable_block/dflash.rs
+++ b/crates/backend-uzu/src/encodable_block/dflash.rs
@@ -3,7 +3,7 @@ use thiserror::Error;
use crate::{
array::size_for_shape,
backends::common::{
- Allocation, AllocationType, Backend, Context, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
gpu_types::trie::TrieNode,
kernel::{TensorAddSwapKernel, radix_top_k_small::RadixTopKSmall},
},
@@ -36,12 +36,11 @@ pub struct DFlashState<B: Backend> {
layer_states: Box<[AttentionState<B>]>,
context_length: usize,
context_capacity: usize,
- target_output_norm: Option<Allocation<B>>,
}
impl<B: Backend> DFlashState<B> {
- pub(crate) fn target_output_norm(&self) -> Option<&Allocation<B>> {
- self.target_output_norm.as_ref()
+ pub(crate) fn context_length(&self) -> usize {
+ self.context_length
}
}
@@ -192,7 +191,6 @@ impl<B: Backend> DFlashDraft<B> {
layer_states,
context_length: 0,
context_capacity,
- target_output_norm: None,
})
}
@@ -200,30 +198,28 @@ impl<B: Backend> DFlashDraft<B> {
&self,
state: &mut DFlashState<B>,
target_features: &[Allocation<B>],
- num_tokens: usize,
- target_output_norm: &Allocation<B>,
+ accepted_indices: &[usize],
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- if num_tokens == 0 {
+ if accepted_indices.is_empty() {
return Ok(());
}
+ let num_tokens = accepted_indices.len();
let expected_feature_count = self.target_feature_width / self.model_dim;
assert_eq!(target_features.len(), expected_feature_count);
- let row_size = self.model_dim * self.data_type.size_in_bytes();
- let layer_feature_size = num_tokens * self.model_dim * self.data_type.size_in_bytes();
- assert!(target_features.iter().all(|features| features.size() == layer_feature_size));
- assert_eq!(target_output_norm.size(), row_size);
+ let layer_feature_bytes = self.model_dim * self.data_type.size_in_bytes();
+ assert!(target_features.iter().all(|features| features.size() % layer_feature_bytes == 0));
let context_length = state.context_length + num_tokens;
assert!(context_length <= state.context_capacity, "DFlash state capacity exceeded");
assert!(context_length <= self.max_context_length, "DFlash context exceeds configured RoPE capacity");
let mut projection_input =
encoder.allocate_scratch(num_tokens * self.target_feature_width * self.data_type.size_in_bytes())?;
- let layer_feature_bytes = self.model_dim * self.data_type.size_in_bytes();
for (layer_index, features) in target_features.iter().enumerate() {
- for token_index in 0..num_tokens {
- let source_start = token_index * layer_feature_bytes;
+ for (token_index, &accepted_index) in accepted_indices.iter().enumerate() {
+ let source_start = accepted_index * layer_feature_bytes;
+ assert!(source_start + layer_feature_bytes <= features.size(), "accepted index out of feature bounds");
let destination_start = (token_index * target_features.len() + layer_index) * layer_feature_bytes;
encoder.encode_copy(
features,
@@ -233,10 +229,6 @@ impl<B: Backend> DFlashDraft<B> {
);
}
}
- let mut stored_hidden = encoder.context().create_allocation(row_size, AllocationType::Global)?;
- encoder.encode_copy(target_output_norm, .., &mut stored_hidden, ..);
- state.target_output_norm = Some(stored_hidden);
-
let projected = self.context_projection.encode(projection_input, num_tokens, encoder)?;
let projected = self.context_norm.encode(&projected, 0, num_tokens, None, encoder)?;
let token_positions = (state.context_length..state.context_length + num_tokens).collect::<Box<[_]>>();
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 081716246..2b5350cc4 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -5,9 +5,10 @@ use crate::{
array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder, Kernels,
- gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode},
+ gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder, QuantizationMethod, QuantizationMode},
kernel::{
- FullPrecisionEmbeddingLookupKernel, LogitSoftCapKernel, QuantizedEmbeddingLookupKernel,
+ FullPrecisionEmbeddingLookupKernel, HadamardTransformKernel, LogitSoftCapKernel,
+ QuantizedEmbeddingLookupKernel,
matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
@@ -85,9 +86,15 @@ enum UntiedEmbeddingReadoutType<B: Backend> {
zero_points_or_biases: Option<Allocation<B>>,
readout: Mutex<<B::Kernels as Kernels>::MatmulKernel>,
readout_config: ReadoutQuantConfig,
+ input_hadamard: Option<InputHadamard<B>>,
},
}
+struct InputHadamard<B: Backend> {
+ factors: Allocation<B>,
+ kernel: <B::Kernels as Kernels>::HadamardTransformKernel,
+}
+
enum EmbeddingTying<B: Backend> {
Tied {
ty: TiedEmbeddingType<B>,
@@ -338,6 +345,52 @@ impl<B: Backend> Embedding<B> {
lookup,
}
},
+ AnyWeightMatrixSpec::HybridSpec(HybridSpec {
+ quantization_spec,
+ adapter_spec: None,
+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ incoherence_processing_mode: IncoherenceProcessingMode::Output,
+ ..
+ }) => {
+ let (embedding_quantization_mode, group_size, quantization_method) =
+ input_quantization_from_spec(*quantization_spec)?;
+ let quantized_tree = input_embedding_tree.subtree("quantized")?;
+ let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(
+ &quantized_tree,
+ vocab_size as usize,
+ model_dim as usize,
+ data_type,
+ embedding_quantization_mode,
+ quantization_method,
+ group_size,
+ )?;
+
+ let incoherence_signs_tree = input_embedding_tree.subtree("incoherence_signs")?;
+ let output_hadamard_factors = Some(
+ incoherence_signs_tree
+ .leaf("output_signs")?
+ .validate(&[model_dim as usize], DataType::I32)?
+ .read_allocation()?,
+ );
+ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
+ context,
+ data_type,
+ group_size as u32,
+ embedding_quantization_mode,
+ quantization_method,
+ true,
+ )
+ .map_err(EmbeddingError::BackendError)?;
+
+ UntiedEmbeddingLookupType::Quantized {
+ weights,
+ scales,
+ zero_points_or_biases,
+ quantization_method,
+ output_hadamard_factors,
+ lookup,
+ }
+ },
spec => return Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),
};
@@ -389,6 +442,62 @@ impl<B: Backend> Embedding<B> {
zero_points_or_biases,
readout,
readout_config,
+ input_hadamard: None,
+ }
+ },
+ AnyWeightMatrixSpec::HybridSpec(HybridSpec {
+ quantization_spec,
+ adapter_spec: None,
+ incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ incoherence_processing_mode: IncoherenceProcessingMode::Input,
+ ..
+ }) => {
+ let (embedding_quantization_mode, group_size, quantization_method) =
+ output_quantization_from_spec(*quantization_spec)?;
+ let quantized_tree = output_embedding_tree.subtree("quantized")?;
+ let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(
+ &quantized_tree,
+ vocab_size as usize,
+ model_dim as usize,
+ data_type,
+ embedding_quantization_mode,
+ quantization_method,
+ group_size,
+ )?;
+ let (readout, readout_config) = quantized_readout(
+ context,
+ data_type,
+ embedding_quantization_mode,
+ quantization_method,
+ group_size,
+ )?;
+
+ // Input-side incoherence is applied privately to the readout
+ // input: the shared hidden state must stay untransformed
+ // (e.g. for the speculator).
+ let incoherence_signs_tree = output_embedding_tree.subtree("incoherence_signs")?;
+ let factors = incoherence_signs_tree
+ .leaf("input_signs")?
+ .validate(&[model_dim as usize], DataType::I32)?
+ .read_allocation()?;
+ let kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(
+ context,
+ data_type,
+ HadamardTransformOrder::Input,
+ )
+ .map_err(EmbeddingError::BackendError)?;
+ let input_hadamard = Some(InputHadamard {
+ factors,
+ kernel,
+ });
+
+ UntiedEmbeddingReadoutType::Quantized {
+ weights,
+ scales,
+ zero_points_or_biases,
+ readout,
+ readout_config,
+ input_hadamard,
}
},
spec => return Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),
@@ -525,6 +634,17 @@ impl<B: Backend> Embedding<B> {
) -> Result<Allocation<B>, EmbeddingError<B>> {
assert!(batch_dim > 0, "Embedding readout requires at least one row");
let native_output = output_data_type == self.data_type;
+ let input_hadamard = match &self.tying {
+ EmbeddingTying::Untied {
+ output_ty:
+ UntiedEmbeddingReadoutType::Quantized {
+ input_hadamard,
+ ..
+ },
+ ..
+ } => input_hadamard.as_ref(),
+ _ => None,
+ };
let mut output_allocation = encoder
.allocate_scratch(size_for_shape(&[batch_dim, self.vocab_size as usize], output_data_type))
.map_err(EmbeddingError::BackendError)?;
@@ -598,58 +718,54 @@ impl<B: Backend> Embedding<B> {
zero_points_or_biases,
readout,
readout_config,
+ ..
},
} => {
- let b: MatmulB<'_, B> = match readout_config.method {
- QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {
- b: weights,
- scales,
- biases: zero_points_or_biases.as_ref().expect("ScaleBias quantization requires biases"),
- mode: readout_config.mode,
- group_size: readout_config.group_size,
- },
- QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {
- b: weights,
- scales,
- zero_points: zero_points_or_biases
- .as_ref()
- .expect("ScaleZeroPoint quantization requires zero_points"),
- mode: readout_config.mode,
- group_size: readout_config.group_size,
+ let b = quantized_matmul_b(weights, scales, zero_points_or_biases.as_ref(), readout_config);
+ let mut rht_input: Option<Allocation<B>> = None;
+ let a = match input_hadamard {
+ Some(input_hadamard) => {
+ let mut transformed =
+ encoder.allocate_scratch(input_allocation.size()).map_err(EmbeddingError::BackendError)?;
+ encoder.encode_copy(input_allocation, .., &mut transformed, ..);
+ input_hadamard.kernel.encode(
+ &mut transformed,
+ &input_hadamard.factors,
+ self.model_dim,
+ batch_dim as u32,
+ encoder,
+ );
+ rht_input.insert(transformed)
},
- QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {
- b: weights,
- scales,
- mode: readout_config.mode,
- group_size: readout_config.group_size,
+ None => input_allocation,
+ };
+ let arguments = MatmulArguments {
+ a: MatmulA::FullPrecision {
+ values: a,
+ offset: 0,
},
+ b,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut output_allocation,
+ d_transform: MatmulDOps::none(),
+ gather_indices: None,
+ m: batch_dim as u32,
+ n: self.vocab_size,
+ k: self.model_dim,
};
- if !native_output {
- return Err(EmbeddingError::UnsupportedConfiguration(
- "widened readout requires full-precision LM-head weights".to_string(),
- ));
- }
- readout
- .lock()
- .encode(
- MatmulArguments {
- a: MatmulA::FullPrecision {
- values: input_allocation,
- offset: 0,
- },
- b,
- b_leading_dimension: None,
- b_transpose: true,
- d: &mut output_allocation,
- d_transform: MatmulDOps::none(),
- gather_indices: None,
- m: batch_dim as u32,
- n: self.vocab_size,
- k: self.model_dim,
- },
- encoder,
+ if native_output {
+ readout.lock().encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;
+ } else {
+ let mut widened = <B::Kernels as Kernels>::MatmulKernel::new(
+ encoder.context(),
+ self.data_type,
+ self.data_type,
+ output_data_type,
)
.map_err(EmbeddingError::BackendError)?;
+ widened.encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;
+ }
},
};
@@ -668,7 +784,7 @@ impl<B: Backend> Embedding<B> {
}
/// Per-row candidate readout via the GEMV B-row gather: `out[r][j] == dense[r][token_ids[r][j]]`,
- /// soft-capped when configured, one dispatch. Full-precision weights; caller guarantees `token_ids < vocab_size`.
+ /// soft-capped when configured, one dispatch. Caller guarantees `token_ids < vocab_size`.
pub(crate) fn encode_readout_sparse(
&self,
input: &Allocation<B>,
@@ -678,7 +794,7 @@ impl<B: Backend> Embedding<B> {
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, EmbeddingError<B>> {
assert!(rows > 0 && ids_per_row > 0);
- let (weights, readout) = match &self.tying {
+ let (b, readout, input_hadamard) = match &self.tying {
EmbeddingTying::Tied {
ty:
TiedEmbeddingType::FullPrecision {
@@ -686,7 +802,13 @@ impl<B: Backend> Embedding<B> {
readout,
..
},
- } => (weights, readout),
+ } => (
+ MatmulB::FullPrecision {
+ b: weights,
+ },
+ readout,
+ None,
+ ),
EmbeddingTying::Untied {
output_ty:
UntiedEmbeddingReadoutType::FullPrecision {
@@ -694,30 +816,73 @@ impl<B: Backend> Embedding<B> {
readout,
},
..
- } => (weights, readout),
- _ => {
- return Err(EmbeddingError::UnsupportedConfiguration(
- "fused sparse readout requires full-precision embedding weights".to_string(),
- ));
- },
+ } => (
+ MatmulB::FullPrecision {
+ b: weights,
+ },
+ readout,
+ None,
+ ),
+ EmbeddingTying::Tied {
+ ty:
+ TiedEmbeddingType::Quantized {
+ weights,
+ scales,
+ zero_points_or_biases,
+ readout,
+ readout_config,
+ ..
+ },
+ } => (quantized_matmul_b(weights, scales, zero_points_or_biases.as_ref(), readout_config), readout, None),
+ EmbeddingTying::Untied {
+ output_ty:
+ UntiedEmbeddingReadoutType::Quantized {
+ weights,
+ scales,
+ zero_points_or_biases,
+ readout,
+ readout_config,
+ input_hadamard,
+ },
+ ..
+ } => (
+ quantized_matmul_b(weights, scales, zero_points_or_biases.as_ref(), readout_config),
+ readout,
+ input_hadamard.as_ref(),
+ ),
};
let mut output = encoder
.allocate_scratch(size_for_shape(&[rows, ids_per_row], self.data_type))
.map_err(EmbeddingError::BackendError)?;
+ let mut rht_input: Option<Allocation<B>> = None;
+ let a = match input_hadamard {
+ Some(input_hadamard) => {
+ let mut transformed = encoder.allocate_scratch(input.size()).map_err(EmbeddingError::BackendError)?;
+ encoder.encode_copy(input, .., &mut transformed, ..);
+ input_hadamard.kernel.encode(
+ &mut transformed,
+ &input_hadamard.factors,
+ self.model_dim,
+ rows as u32,
+ encoder,
+ );
+ rht_input.insert(transformed)
+ },
+ None => input,
+ };
+
let soft_cap = self.logit_soft_cap.as_ref().map(|cap| cap.value);
readout
.lock()
.encode(
MatmulArguments {
a: MatmulA::FullPrecision {
- values: input,
+ values: a,
offset: 0,
},
- b: MatmulB::FullPrecision {
- b: weights,
- },
+ b,
b_leading_dimension: None,
b_transpose: true,
d: &mut output,
@@ -738,6 +903,36 @@ impl<B: Backend> Embedding<B> {
}
}
+fn quantized_matmul_b<'a, B: Backend>(
+ weights: &'a Allocation<B>,
+ scales: &'a Allocation<B>,
+ zero_points_or_biases: Option<&'a Allocation<B>>,
+ readout_config: &ReadoutQuantConfig,
+) -> MatmulB<'a, B> {
+ match readout_config.method {
+ QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {
+ b: weights,
+ scales,
+ biases: zero_points_or_biases.expect("ScaleBias quantization requires biases"),
+ mode: readout_config.mode,
+ group_size: readout_config.group_size,
+ },
+ QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {
+ b: weights,
+ scales,
+ zero_points: zero_points_or_biases.expect("ScaleZeroPoint quantization requires zero_points"),
+ mode: readout_config.mode,
+ group_size: readout_config.group_size,
+ },
+ QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {
+ b: weights,
+ scales,
+ mode: readout_config.mode,
+ group_size: readout_config.group_size,
+ },
+ }
+}
+
fn input_quantization_from_spec<B: Backend>(
spec: AnyWeightMatrixSpec
) -> Result<(QuantizationMode, usize, QuantizationMethod), EmbeddingError<B>> {
diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rs
index db7d59ec6..e6426d7dd 100644
--- a/crates/backend-uzu/src/encodable_block/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mod.rs
@@ -1,7 +1,7 @@
pub mod batch_topology;
pub mod classifier;
pub mod decoder;
-pub(crate) mod dflash;
+pub mod dflash;
pub mod embedding;
pub mod linear;
pub mod mixer;
diff --git a/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs b/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs
index ad74d12ae..cd7fb2096 100644
--- a/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs
+++ b/crates/backend-uzu/src/encodable_block/sampling/gumbel.rs
@@ -1,5 +1,3 @@
-use std::collections::HashMap;
-
const PHILOX_W32_0: u32 = 0x9E3779B9;
const PHILOX_W32_1: u32 = 0xBB67AE85;
const PHILOX_M4X32_0: u32 = 0xD2511F53;
@@ -74,15 +72,3 @@ pub fn revidx(
(thread_offset + block_idx_offset, block_idx_word)
}
-
-pub fn speculator_sample(
- seed: u64,
- vocab_size: usize,
- speculator_probs: &HashMap<u64, f32>,
-) -> Option<u64> {
- speculator_probs
- .iter()
- .map(|(&k, &v)| (k, f32::ln(v) + gumbel_float(seed, revidx(k as u32, vocab_size as u32))))
- .max_by(|(_, av), (_, bv)| f32::total_cmp(av, bv))
- .map(|(k, _)| k)
-}
diff --git a/crates/backend-uzu/src/encodable_block/sampling/mod.rs b/crates/backend-uzu/src/encodable_block/sampling/mod.rs
index 63eef2a73..2f322b98f 100644
--- a/crates/backend-uzu/src/encodable_block/sampling/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/sampling/mod.rs
@@ -15,10 +15,10 @@ use crate::{
encodable_block::batch_topology::BatchTopology,
};
+#[cfg(backend = "cpu")]
mod gumbel;
mod prng;
-pub use gumbel::speculator_sample;
#[cfg(backend = "cpu")]
pub use gumbel::{gumbel_float, revidx};
pub use prng::PRng;
diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rs
index 5cfe976ad..37459df24 100644
--- a/crates/backend-uzu/src/encodable_block/transformer.rs
+++ b/crates/backend-uzu/src/encodable_block/transformer.rs
@@ -28,7 +28,7 @@ pub struct TransformerState<B: Backend> {
pub struct TransformerEncodeOutput<B: Backend> {
pub output: Option<Allocation<B>>,
- pub hidden_features: Box<[Allocation<B>]>,
+ pub hidden_features: Option<Box<[Allocation<B>]>>,
}
impl<B: Backend> TransformerState<B> {
@@ -224,19 +224,20 @@ impl<B: Backend> Transformer<B> {
per_layer_inputs: Option<&Allocation<B>>,
batch_dim: &BatchTopology,
output_range: Option<Range<usize>>,
+ hidden_feature_layer_indices: Option<&[usize]>,
mut state: Option<&mut TransformerState<B>>,
encoder: &mut Encoder<B>,
- hidden_feature_layer_indices: &[usize],
) -> Result<TransformerEncodeOutput<B>, B::Error> {
let mut hidden = input;
- let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_empty() {
+ let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_none() {
self.prefill_cache_layer_count()
} else {
self.layers.len()
};
let mut shortcut = encoder.allocate_scratch(hidden.size())?;
- let mut hidden_features = (0..hidden_feature_layer_indices.len()).map(|_| None).collect::<Vec<_>>();
+ let mut hidden_features =
+ hidden_feature_layer_indices.map(|indices| (0..indices.len()).map(|_| None).collect::<Vec<_>>());
let context_length = state.as_ref().map(|state| state.context_length).unwrap_or(0);
let token_positions =
@@ -276,23 +277,30 @@ impl<B: Backend> Transformer<B> {
encoder,
)?;
- for (feature_index, &layer_index) in hidden_feature_layer_indices.iter().enumerate() {
- if layer_index == layer.layer_index {
- let feature = self.capture_residual(&shortcut, &hidden, batch_dim.size(), encoder)?;
- hidden_features[feature_index] = Some(feature);
+ if let (Some(hidden_features), Some(indices)) = (&mut hidden_features, hidden_feature_layer_indices) {
+ for (feature_index, &layer_index) in indices.iter().enumerate() {
+ if layer_index == layer.layer_index {
+ let feature = self.capture_residual(&shortcut, &hidden, batch_dim.size(), encoder)?;
+ hidden_features[feature_index] = Some(feature);
+ }
}
}
}
- let hidden_features: Box<[Allocation<B>]> = hidden_features
- .into_iter()
- .enumerate()
- .map(|(feature_index, feature)| {
- feature.unwrap_or_else(|| {
- panic!("requested hidden feature for missing layer {}", hidden_feature_layer_indices[feature_index])
+ let hidden_features = hidden_features.map(|hidden_features| {
+ hidden_features
+ .into_iter()
+ .enumerate()
+ .map(|(feature_index, feature)| {
+ feature.unwrap_or_else(|| {
+ panic!(
+ "requested hidden feature for missing layer {}",
+ hidden_feature_layer_indices.unwrap()[feature_index]
+ )
+ })
})
- })
- .collect();
+ .collect::<Box<[_]>>()
+ });
let Some(output_range) = output_range else {
return Ok(TransformerEncodeOutput {
diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rs
index c57281cf1..344a1838c 100644
--- a/crates/backend-uzu/src/encodable_block/weaver.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver.rs
@@ -4,10 +4,13 @@ use crate::{
array::size_for_shape,
backends::common::{
Allocation, AllocationType, AsBufferRangeMut, Backend, Context, Encoder, Kernels,
- gpu_types::weaver::{CANDIDATES_MAX, MetadataIdx},
+ gpu_types::weaver::{
+ CANDIDATES_MAX, FRONTIER_MAX_SLOTS, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx,
+ },
kernel::{
ActivationKernel, AncestorAttentionKernel, AttentionPrepareKernel, TensorAddBiasKernel,
- WeaverNodeCacheWriteKernel, WeaverTopChildrenKernel,
+ WeaverFrontierScatterKernel, WeaverFrontierSelectKernel, WeaverNodeCacheWriteKernel,
+ WeaverTopChildrenKernel,
},
},
config::{normalization::NormalizationConfig, weaver::WeaverConfig},
@@ -26,9 +29,9 @@ use crate::{
const DATA_TYPE: DataType = DataType::BF16;
-pub(crate) struct TopKChildren<B: Backend> {
- pub(crate) token_ids: Allocation<B>,
- pub(crate) logprobs: Allocation<B>,
+struct TopKChildren<B: Backend> {
+ token_ids: Allocation<B>,
+ logprobs: Allocation<B>,
}
struct PrefixOutput<B: Backend> {
@@ -41,25 +44,57 @@ struct PrefixAttentionInputs<B: Backend> {
kv_cache: Allocation<B>,
}
-pub(crate) struct WeaverPrefixKvCache<B: Backend> {
+struct WeaverPrefixKvCache<B: Backend> {
layer_kv: Box<[Allocation<B>]>,
token_count: usize,
}
-pub(crate) struct WeaverNodeKvCache<B: Backend> {
+struct WeaverNodeKvCache<B: Backend> {
layer_kv: Box<[Allocation<B>]>,
node_capacity: u32,
}
-pub(crate) struct WeaverStepBatch<'a, B: Backend> {
- pub node_count: usize,
- pub candidates_per_node: usize,
- pub ancestor_stride: usize,
- pub node_token_ids: &'a Allocation<B>,
- pub candidate_ids: &'a Allocation<B>,
- pub candidate_logits: &'a Allocation<B>,
- pub ancestor_indices: &'a Allocation<B>,
- pub node_metadata: &'a Allocation<B>,
+struct WeaverStepBatch<'a, B: Backend> {
+ node_count: usize,
+ candidates_per_node: usize,
+ ancestor_stride: usize,
+ node_token_ids: &'a Allocation<B>,
+ candidate_ids: &'a Allocation<B>,
+ candidate_logits: &'a Allocation<B>,
+ ancestor_indices: &'a Allocation<B>,
+ node_metadata: &'a Allocation<B>,
+ depth_seeds: &'a Allocation<B>,
+}
+
+pub(crate) struct WeaverTreeInput<'a, B: Backend> {
+ pub(crate) target_hidden: &'a Allocation<B>,
+ pub(crate) draft_hidden: &'a Allocation<B>,
+ pub(crate) target_embedding: &'a Embedding<B>,
+ pub(crate) candidate_ids: &'a Allocation<B>,
+ pub(crate) candidate_scores: &'a Allocation<B>,
+ pub(crate) candidate_rows: usize,
+ pub(crate) candidates_per_row: usize,
+ pub(crate) depth_seeds: &'a [u64],
+ pub(crate) root_token_id: u32,
+ pub(crate) tree_budget: usize,
+ pub(crate) frontier_width: usize,
+ pub(crate) children_per_node: usize,
+}
+
+pub(crate) struct EncodedWeaverTree<B: Backend> {
+ slots: Allocation<B>,
+}
+
+pub(crate) struct ProposalNode {
+ pub(crate) token: u32,
+ pub(crate) depth: usize,
+ pub(crate) children: Vec<usize>,
+}
+
+impl<B: Backend> EncodedWeaverTree<B> {
+ pub(crate) fn decode(self) -> Vec<ProposalNode> {
+ decode_tree_slots(&self.slots.copyout::<u32>())
+ }
}
pub(crate) struct Weaver<B: Backend> {
@@ -74,6 +109,8 @@ pub(crate) struct Weaver<B: Backend> {
prefix_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
node_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
top_children: <B::Kernels as Kernels>::WeaverTopChildrenKernel,
+ frontier_select: <B::Kernels as Kernels>::WeaverFrontierSelectKernel,
+ frontier_scatter: <B::Kernels as Kernels>::WeaverFrontierScatterKernel,
model_dim: usize,
target_model_dim: usize,
max_depth: usize,
@@ -127,6 +164,8 @@ pub enum WeaverEncodeError<B: Backend> {
Backend(#[source] B::Error),
#[error("embedding error: {0}")]
Embedding(#[from] EmbeddingError<B>),
+ #[error("invalid Weaver tree input")]
+ InvalidTreeInput,
}
impl<B: Backend> Weaver<B> {
@@ -224,6 +263,10 @@ impl<B: Backend> Weaver<B> {
.map_err(WeaverNewError::Backend)?;
let top_children =
<B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context).map_err(WeaverNewError::Backend)?;
+ let frontier_select =
+ <B::Kernels as Kernels>::WeaverFrontierSelectKernel::new(context).map_err(WeaverNewError::Backend)?;
+ let frontier_scatter =
+ <B::Kernels as Kernels>::WeaverFrontierScatterKernel::new(context).map_err(WeaverNewError::Backend)?;
Ok(Self {
token_embedding_norm,
token_embedding_projection,
@@ -236,13 +279,152 @@ impl<B: Backend> Weaver<B> {
prefix_position_add,
node_position_add,
top_children,
+ frontier_select,
+ frontier_scatter,
model_dim: config.model_dim,
target_model_dim: config.target_model_dim,
max_depth: config.max_depth,
})
}
- pub(crate) fn build_prefix(
+ pub(crate) fn encode_tree(
+ &self,
+ input: WeaverTreeInput<'_, B>,
+ context: &B::Context,
+ encoder: &mut Encoder<B>,
+ ) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {
+ let slot_count = input.tree_budget + 1;
+ let frontier_capacity = slot_count * input.children_per_node;
+ let lookahead_count = self.max_depth.min(input.candidate_rows);
+ if input.tree_budget == 0
+ || input.frontier_width == 0
+ || input.children_per_node == 0
+ || input.children_per_node > input.candidates_per_row
+ || frontier_capacity > FRONTIER_MAX_SLOTS
+ || lookahead_count == 0
+ || input.depth_seeds.len() != self.max_depth
+ {
+ return Err(WeaverEncodeError::InvalidTreeInput);
+ }
+
+ let prefix = self.build_prefix(input.target_hidden, input.draft_hidden, lookahead_count, encoder)?;
+ let mut node_cache = self.create_node_kv_cache(slot_count, context)?;
+
+ let mut tree_values = vec![0u32; TreeIdx::COUNT * slot_count];
+ for slot in 0..slot_count {
+ tree_values[TreeIdx::ParentSlot as usize * slot_count + slot] = FRONTIER_NO_WINNER;
+ }
+ tree_values[TreeIdx::TokenId as usize * slot_count] = input.root_token_id;
+ tree_values[TreeIdx::Valid as usize * slot_count] = 1;
+
+ let mut tree = encoder.allocate_constant_from_slice(&tree_values).map_err(WeaverEncodeError::Backend)?;
+ let mut frontier = encoder
+ .allocate_constant_from_slice(&vec![0u32; FrontierIdx::COUNT * frontier_capacity])
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut slot_ancestors = encoder
+ .allocate_constant_from_slice(&vec![0u32; slot_count * self.max_depth])
+ .map_err(WeaverEncodeError::Backend)?;
+
+ let mut round_token_id_values = vec![0u32; input.frontier_width];
+ round_token_id_values[0] = input.root_token_id;
+ let mut round_valid_values = vec![0u32; input.frontier_width];
+ round_valid_values[0] = 1;
+ let mut round_token_ids =
+ encoder.allocate_constant_from_slice(&round_token_id_values).map_err(WeaverEncodeError::Backend)?;
+ let mut round_metadata = encoder
+ .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * input.frontier_width])
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut round_ancestors = encoder
+ .allocate_constant_from_slice(&vec![0u32; input.frontier_width * self.max_depth])
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut round_valid =
+ encoder.allocate_constant_from_slice(&round_valid_values).map_err(WeaverEncodeError::Backend)?;
+ let mut round_candidate_ids = encoder
+ .allocate_constant_from_slice(&vec![0u32; input.frontier_width * input.candidates_per_row])
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut round_candidate_scores = encoder
+ .allocate_constant_from_slice(&vec![0.0f32; input.frontier_width * input.candidates_per_row])
+ .map_err(WeaverEncodeError::Backend)?;
+ let depth_seeds =
+ encoder.allocate_constant_from_slice(input.depth_seeds).map_err(WeaverEncodeError::Backend)?;
+
+ let mut slot_start = 0;
+ while slot_start < slot_count {
+ let node_count = if slot_start == 0 {
+ 1
+ } else {
+ input.frontier_width.min(slot_count - slot_start)
+ };
+ if slot_start > 0 {
+ self.frontier_select.encode(
+ &mut frontier,
+ &mut tree,
+ &mut slot_ancestors,
+ &mut round_token_ids,
+ &mut round_metadata,
+ &mut round_ancestors,
+ &mut round_valid,
+ input.candidate_ids,
+ input.candidate_scores,
+ &mut round_candidate_ids,
+ &mut round_candidate_scores,
+ frontier_capacity as u32,
+ slot_count as u32,
+ node_count as u32,
+ slot_start as u32,
+ self.max_depth as u32,
+ self.max_depth as u32,
+ lookahead_count as u32,
+ input.candidate_rows as u32,
+ input.candidates_per_row as u32,
+ encoder,
+ );
+ }
+ let (candidate_ids, candidate_scores) = if slot_start == 0 {
+ (input.candidate_ids, input.candidate_scores)
+ } else {
+ (&round_candidate_ids, &round_candidate_scores)
+ };
+ let batch = WeaverStepBatch {
+ node_count,
+ candidates_per_node: input.candidates_per_row,
+ ancestor_stride: self.max_depth,
+ node_token_ids: &round_token_ids,
+ candidate_ids,
+ candidate_logits: candidate_scores,
+ ancestor_indices: &round_ancestors,
+ node_metadata: &round_metadata,
+ depth_seeds: &depth_seeds,
+ };
+ let children = self.encode_step_batch(
+ &prefix,
+ &batch,
+ &mut node_cache,
+ input.children_per_node,
+ input.target_embedding,
+ encoder,
+ )?;
+ self.frontier_scatter.encode(
+ &tree,
+ &round_metadata,
+ &round_valid,
+ &children.token_ids,
+ &children.logprobs,
+ &mut frontier,
+ frontier_capacity as u32,
+ slot_count as u32,
+ node_count as u32,
+ input.children_per_node as u32,
+ encoder,
+ );
+ slot_start += node_count;
+ }
+ Ok(EncodedWeaverTree {
+ slots: tree,
+ })
+ }
+
+ fn build_prefix(
&self,
target_hidden: &Allocation<B>,
draft_hidden: &Allocation<B>,
@@ -304,7 +486,7 @@ impl<B: Backend> Weaver<B> {
})
}
- pub(crate) fn create_node_kv_cache(
+ fn create_node_kv_cache(
&self,
capacity: usize,
context: &B::Context,
@@ -321,7 +503,7 @@ impl<B: Backend> Weaver<B> {
})
}
- pub(crate) fn encode_step_batch(
+ fn encode_step_batch(
&self,
prefix: &WeaverPrefixKvCache<B>,
batch: &WeaverStepBatch<'_, B>,
@@ -404,11 +586,14 @@ impl<B: Backend> Weaver<B> {
&residual_logits,
batch.candidate_logits,
batch.candidate_ids,
+ batch.depth_seeds,
+ batch.node_metadata,
&mut token_ids,
&mut logprobs,
batch.node_count as u32,
batch.candidates_per_node as u32,
children_per_node as u32,
+ target_embedding.vocab_size() as u32,
encoder,
);
Ok(TopKChildren {
@@ -668,3 +853,33 @@ impl<B: Backend> WeaverBlock<B> {
self.down_projection.encode(mlp_hidden, row_count, encoder)
}
}
+
+fn decode_tree_slots(tree: &[u32]) -> Vec<ProposalNode> {
+ assert!(tree.len().is_multiple_of(TreeIdx::COUNT), "tree array must contain {} equal-length lanes", TreeIdx::COUNT);
+ let slot_count = tree.len() / TreeIdx::COUNT;
+ let field = |field: TreeIdx, slot: usize| tree[field as usize * slot_count + slot];
+ let mut slot_to_node = vec![usize::MAX; slot_count];
+ let mut nodes: Vec<ProposalNode> = Vec::with_capacity(slot_count);
+ for slot in 0..slot_count {
+ if field(TreeIdx::Valid, slot) == 0 {
+ continue;
+ }
+ let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;
+ let parent = (parent_slot >= 0).then(|| {
+ let parent = slot_to_node[parent_slot as usize];
+ assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");
+ parent
+ });
+ let index = nodes.len();
+ slot_to_node[slot] = index;
+ if let Some(parent) = parent {
+ nodes[parent].children.push(index);
+ }
+ nodes.push(ProposalNode {
+ token: field(TreeIdx::TokenId, slot),
+ depth: field(TreeIdx::Depth, slot) as usize,
+ children: Vec::new(),
+ });
+ }
+ nodes
+}
diff --git a/crates/backend-uzu/src/engine/dflash_speculator.rs b/crates/backend-uzu/src/engine/dflash_speculator.rs
deleted file mode 100644
index 3fe5dbb4d..000000000
--- a/crates/backend-uzu/src/engine/dflash_speculator.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-use std::{
- fs::File,
- io::{self, BufReader},
- path::Path,
-};
-
-use thiserror::Error;
-
-use crate::{
- backends::common::Backend,
- config::speculator::{AnySpeculatorConfig, model::SpeculatorModelConfig},
- data_type::DataType,
- encodable_block::{
- dflash::{DFlashDraft, DFlashDraftNewError},
- weaver::{Weaver, WeaverNewError},
- },
- engine::Engine,
- parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},
- speculators::dflash_speculator::DFlashSpeculator,
-};
-
-#[derive(Debug, Error)]
-pub enum EngineLoadDFlashSpeculatorError<B: Backend> {
- #[error("I/O error: {0}")]
- IO(#[from] io::Error),
- #[error("Serde error: {0}")]
- Serde(#[from] serde_json::Error),
- #[error("HeaderLoading error: {0}")]
- HeaderLoading(#[from] HeaderLoadingError),
- #[error("ParameterLoader error: {0}")]
- ParameterLoader(#[from] ParameterLoaderError<B>),
- #[error("DFlash draft error: {0}")]
- Draft(#[from] DFlashDraftNewError<B>),
- #[error("Weaver error: {0}")]
- Weaver(#[from] WeaverNewError<B>),
- #[error("DFlash mask_token_id {mask_token_id} is outside vocabulary size {vocab_size}")]
- InvalidMaskTokenId {
- mask_token_id: u64,
- vocab_size: usize,
- },
-}
-
-impl<B: Backend> Engine<B> {
- pub fn load_dflash_speculator(
- &self,
- model_path: &Path,
- ) -> Result<DFlashSpeculator<B>, EngineLoadDFlashSpeculatorError<B>> {
- let config: SpeculatorModelConfig =
- serde_json::from_reader(BufReader::new(File::open(model_path.join("config.json"))?))?;
- let AnySpeculatorConfig::DFlashSpeculatorConfig(speculator_config) = config.speculator_config;
- let draft_config = &speculator_config.draft_config;
- if draft_config.mask_token_id >= draft_config.vocab_size as u64 {
- return Err(EngineLoadDFlashSpeculatorError::InvalidMaskTokenId {
- mask_token_id: draft_config.mask_token_id,
- vocab_size: draft_config.vocab_size,
- });
- }
-
- let data_type = DataType::BF16;
-
- let weights_file = File::open(model_path.join("model.safetensors"))?;
- let weight_loader = ParameterLoader::new(&weights_file, &*self.context)?;
- let speculator_tree = weight_loader.tree().subtree("speculator")?;
- let draft_tree = speculator_tree.subtree("draft_model")?;
- let draft_model = DFlashDraft::new(&*self.context, &speculator_config.draft_config, &draft_tree, data_type)?;
- let weaver = speculator_config
- .weaver_config
- .as_ref()
- .map(|weaver_config| Weaver::new(&*self.context, weaver_config, &speculator_tree.subtree("weaver")?))
- .transpose()?;
-
- weight_loader.tree().assert_all_tensors_validated()?;
-
- Ok(DFlashSpeculator::new(self.context.clone(), draft_model, weaver, speculator_config))
- }
-}
diff --git a/crates/backend-uzu/src/engine/language_model/mod.rs b/crates/backend-uzu/src/engine/language_model/mod.rs
index c78afca07..7343cbcaf 100644
--- a/crates/backend-uzu/src/engine/language_model/mod.rs
+++ b/crates/backend-uzu/src/engine/language_model/mod.rs
@@ -12,6 +12,7 @@ use crate::{
},
engine::Engine,
parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},
+ speculators::dflash_speculator::{DFlashSpeculator, DFlashSpeculatorLoadError},
};
pub mod state;
@@ -23,9 +24,11 @@ pub mod grammar;
pub struct LanguageModel<B: Backend> {
context: Arc<B::Context>,
decoder: Decoder<B>,
+ speculator: Option<DFlashSpeculator<B>>,
sampling: Sampling<B>,
context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,
generation_config: GenerationConfig,
+ #[cfg(grammar)]
vocab_size: usize,
}
@@ -43,6 +46,8 @@ pub enum EngineLoadLanguageModelError<B: Backend> {
Backend(#[source] B::Error),
#[error("Decoder error: {0}")]
Decoder(#[from] DecoderError<B>),
+ #[error("Speculator error: {0}")]
+ Speculator(#[from] DFlashSpeculatorLoadError<B>),
}
impl<B: Backend> Engine<B> {
@@ -56,9 +61,14 @@ impl<B: Backend> Engine<B> {
let weights_file = File::open(model_path.join("model.safetensors"))?;
let weight_loader = ParameterLoader::new(&weights_file, &*self.context)?;
- self.build_language_model(config, &weight_loader)
+ // TODO
+ let speculator_path = model_path.join("speculator");
+ let speculator_path = speculator_path.exists().then_some(speculator_path);
+
+ self.build_language_model(config, &weight_loader, speculator_path.as_deref())
}
+ // TODO: better design
pub fn load_language_model_random(
&self,
config_path: &Path,
@@ -70,13 +80,14 @@ impl<B: Backend> Engine<B> {
let header_file = File::open(header_path)?;
let weight_loader = ParameterLoader::new_random(&header_file, &*self.context, seed)?;
- self.build_language_model(config, &weight_loader)
+ self.build_language_model(config, &weight_loader, None)
}
fn build_language_model(
&self,
config: LanguageModelConfig,
weight_loader: &ParameterLoader<B>,
+ speculator_path: Option<&Path>,
) -> Result<LanguageModel<B>, EngineLoadLanguageModelError<B>> {
let context = self.context.clone();
@@ -89,6 +100,15 @@ impl<B: Backend> Engine<B> {
data_type,
)?;
+ assert!(
+ speculator_path.is_none() || decoder.speculation_supported(),
+ "attempted to load speculator for a model that doesn't support one"
+ );
+
+ let speculator = speculator_path
+ .map(|speculator_path| DFlashSpeculator::load(speculator_path, context.clone()))
+ .transpose()?;
+
let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);
let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&context)
@@ -98,24 +118,23 @@ impl<B: Backend> Engine<B> {
let generation_config = config.generation_config;
+ #[cfg(grammar)]
let vocab_size = config.decoder_config.vocab_size;
Ok(LanguageModel {
context,
decoder,
+ speculator,
sampling,
context_ring_update,
generation_config,
+ #[cfg(grammar)]
vocab_size,
})
}
}
impl<B: Backend> LanguageModel<B> {
- pub(crate) fn embedding(&self) -> &crate::encodable_block::embedding::Embedding<B> {
- self.decoder.embedding()
- }
-
pub fn max_context_length(&self) -> Option<usize> {
self.decoder.max_context_length()
}
diff --git a/crates/backend-uzu/src/engine/language_model/state.rs b/crates/backend-uzu/src/engine/language_model/state.rs
index 291e50073..d796779c0 100644
--- a/crates/backend-uzu/src/engine/language_model/state.rs
+++ b/crates/backend-uzu/src/engine/language_model/state.rs
@@ -2,7 +2,7 @@ use thiserror::Error;
use crate::{
backends::common::Backend,
- encodable_block::{sampling::PRng, transformer::TransformerState},
+ encodable_block::{dflash::DFlashState, sampling::PRng, transformer::TransformerState},
engine::language_model::LanguageModel,
};
@@ -11,6 +11,7 @@ pub struct LanguageModelState<B: Backend> {
pub(super) last_output_token: Option<u64>, // TODO: this leaks previous LanguageModelStreamOptions
pub(super) prng: PRng,
pub(super) transformer_state: TransformerState<B>,
+ pub(super) speculator_state: Option<DFlashState<B>>,
pub(super) max_context_length: Option<usize>,
}
@@ -41,11 +42,21 @@ impl<B: Backend> LanguageModel<B> {
.create_empty_state(max_context_length, &self.context)
.map_err(LanguageModelCreateEmptyStateError::Backend)?;
+ let speculator_state = self
+ .speculator
+ .as_ref()
+ .map(|speculator| {
+ speculator.empty_state(max_context_length.expect("speculator doesn't support unlimited state capacity"))
+ })
+ .transpose()
+ .map_err(LanguageModelCreateEmptyStateError::Backend)?;
+
Ok(LanguageModelState {
tokens,
last_output_token,
prng,
transformer_state,
+ speculator_state,
max_context_length,
})
}
diff --git a/crates/backend-uzu/src/engine/language_model/stream/mod.rs b/crates/backend-uzu/src/engine/language_model/stream/mod.rs
index bb60b0298..a6bc22a76 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/mod.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/mod.rs
@@ -6,26 +6,18 @@ use crate::{
backends::common::Backend,
encodable_block::decoder::DecoderError,
engine::language_model::{LanguageModel, state::LanguageModelState},
- speculators::speculator::Speculator,
+ speculators::dflash_speculator::DFlashTreeError,
};
pub use crate::{
encodable_block::sampling::SamplingMethod, engine::language_model::stream::stream::LanguageModelStream,
- trie::TrieCreationConfig,
};
mod stream;
-pub struct LanguageModelStreamSpeculatorOptions<'a> {
- pub speculator: &'a dyn Speculator,
- pub speculation_budget: usize,
- pub trie_creation_config: TrieCreationConfig,
-}
-
-pub struct LanguageModelStreamOptions<'a> {
+pub struct LanguageModelStreamOptions {
pub sampling_method: SamplingMethod,
#[cfg(grammar)]
pub grammar: Option<Grammar>,
- pub speculator: Option<LanguageModelStreamSpeculatorOptions<'a>>,
}
#[derive(Debug, Error)]
@@ -39,8 +31,8 @@ pub enum LanguageModelStreamError<B: Backend> {
Grammar(#[from] GrammarError),
#[error("Trie accept error: {0}")]
TrieAccept(#[from] crate::trie::TrieAcceptError),
- #[error("Speculators are not supported by this model")]
- SpeculatorsNotSupported,
+ #[error("Speculator error: {0}")]
+ Speculator(#[from] DFlashTreeError<B>),
#[error("No seed token (both state and input are empty)")]
NoSeedToken,
#[error("Context overflow")]
@@ -48,12 +40,11 @@ pub enum LanguageModelStreamError<B: Backend> {
}
impl<B: Backend> LanguageModel<B> {
- pub fn default_stream_options<'a>(&'a self) -> LanguageModelStreamOptions<'a> {
+ pub fn default_stream_options(&self) -> LanguageModelStreamOptions {
LanguageModelStreamOptions {
sampling_method: self.default_sampling_method(),
#[cfg(grammar)]
grammar: None,
- speculator: None,
}
}
@@ -61,7 +52,7 @@ impl<B: Backend> LanguageModel<B> {
&'a self,
input: &[u64],
state: &'a mut LanguageModelState<B>,
- options: LanguageModelStreamOptions<'a>,
+ options: LanguageModelStreamOptions,
) -> Result<LanguageModelStream<'a, B>, LanguageModelStreamError<B>> {
LanguageModelStream::new(self, input, state, options)
}
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index 671a07ab0..eaa6ad021 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -20,37 +20,55 @@ use crate::{
state::LanguageModelState,
stream::{LanguageModelStreamError, LanguageModelStreamOptions},
},
+ speculators::dflash_speculator::DFlashTreeOptions,
trie::TrieNode,
};
enum ForwardPassChaining<B: Backend> {
- Constant(u64),
+ Constant {
+ token: u64,
+ output_norm: Option<Allocation<B>>,
+ },
InFlight(DecodingStatePending<B>),
}
impl<B: Backend> ForwardPassChaining<B> {
- fn resolve(
- &mut self,
+ fn resolve<'a>(
+ &'a mut self,
tokens: &mut Vec<u64>,
#[cfg(grammar)] grammar: Option<&mut Grammar>,
- ) -> Result<u64, LanguageModelStreamError<B>> {
+ ) -> Result<(u64, Option<&'a Allocation<B>>), LanguageModelStreamError<B>> {
match self {
- Self::Constant(token_id) => Ok(*token_id),
+ Self::Constant {
+ token,
+ output_norm,
+ } => Ok((*token, output_norm.as_ref())),
Self::InFlight(in_flight) => {
assert!(in_flight.full_accept);
for pending in replace(&mut in_flight.pending, Box::new([])) {
pending.wait_until_completed().map_err(LanguageModelStreamError::Backend)?;
}
- let output = in_flight.output.as_slice::<u32>();
- assert_eq!(output.len(), 1);
- let token_id = output[0] as u64;
- *self = Self::Constant(token_id);
+ let output_tokens = in_flight.output_tokens.as_slice::<u32>();
+ assert_eq!(output_tokens.len(), 1);
+ let token_id = output_tokens[0] as u64;
+ let output_norm = in_flight.output_norm.take();
+ *self = Self::Constant {
+ token: token_id,
+ output_norm,
+ };
tokens.push(token_id);
#[cfg(grammar)]
if let Some(grammar) = grammar {
grammar.accept_token(token_id)?;
}
- Ok(token_id)
+ let Self::Constant {
+ output_norm,
+ ..
+ } = self
+ else {
+ unreachable!()
+ };
+ Ok((token_id, output_norm.as_ref()))
},
}
}
@@ -60,7 +78,9 @@ struct DecodingStatePending<B: Backend> {
input_trie: TrieNode,
full_accept: bool,
pending: Box<[Pending<B>]>,
- output: Allocation<B>,
+ hidden_features: Option<Box<[Allocation<B>]>>,
+ output_norm: Option<Allocation<B>>,
+ output_tokens: Allocation<B>,
}
enum DecodingState<B: Backend> {
@@ -71,6 +91,8 @@ enum DecodingState<B: Backend> {
Accepting {
full: Box<[(usize, u64, u64)]>,
num_accepted: usize,
+ hidden_features: Option<Box<[Allocation<B>]>>,
+ output_norm: Option<(Allocation<B>, usize)>,
},
Halted,
Invalid,
@@ -92,7 +114,7 @@ fn prefill_chunk_parts(
pub struct LanguageModelStream<'a, B: Backend> {
model: &'a LanguageModel<B>,
model_state: &'a mut LanguageModelState<B>,
- options: LanguageModelStreamOptions<'a>,
+ options: LanguageModelStreamOptions,
allocation_pool: Arc<AllocationPool<B>>,
context_ring: Option<Allocation<B>>,
decoding_state: DecodingState<B>,
@@ -104,7 +126,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
model: &'a LanguageModel<B>,
input: &[u64],
model_state: &'a mut LanguageModelState<B>,
- options: LanguageModelStreamOptions<'a>,
+ options: LanguageModelStreamOptions,
) -> Result<Self, LanguageModelStreamError<B>> {
#[cfg(grammar)]
let mut options = options;
@@ -112,10 +134,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
return Err(LanguageModelStreamError::NoSeedToken);
};
- if options.speculator.is_some() && !model.speculation_supported() {
- return Err(LanguageModelStreamError::SpeculatorsNotSupported);
- }
-
if model_state
.max_context_length
.is_some_and(|max_context_length| model_state.tokens.len() + input.len() > max_context_length)
@@ -173,8 +191,11 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let mut encoder = Encoder::<B>::new_with_pool(&model.context, allocation_pool.clone())
.map_err(LanguageModelStreamError::Backend)?;
- let mut output = None;
+ let mut output_tokens = None;
+ let mut output_norm = None;
let split_logits_row = model.decoder.prefill_cache_skips_trailing_layers();
+ let hidden_feature_layer_indices =
+ model.speculator.as_ref().map(|speculator| speculator.hidden_feature_layer_indices());
for (input_chunk, sample_last) in input
.chunks(max_batch_size)
@@ -195,17 +216,15 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let input_flat_trie_nodes = input_flat_trie.token_subtrie_ranges().collect::<Box<[GpuTrieNode]>>();
let batch_dim = BatchTopology::new(&input_flat_trie_nodes, true);
- let logits = model
- .decoder
- .encode(
- &token_ids,
- &batch_dim,
- sample_last.then(|| (input_chunk.len() - 1)..input_chunk.len()),
- &mut model_state.transformer_state,
- &mut encoder,
- &[],
- )?
- .logits;
+ let decoder_output = model.decoder.encode(
+ &token_ids,
+ &batch_dim,
+ sample_last.then(|| (input_chunk.len() - 1)..input_chunk.len()),
+ hidden_feature_layer_indices,
+ &mut model_state.transformer_state,
+ &mut encoder,
+ )?;
+ let logits = decoder_output.logits;
if sample_last {
let logits = logits.unwrap();
@@ -241,7 +260,8 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
#[cfg(not(grammar))]
let bitmask = None;
- output = Some(
+ output_norm = decoder_output.final_hidden;
+ output_tokens = Some(
model
.sampling
.encode(
@@ -264,6 +284,18 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.encode_accept(&(0..input_chunk.len()).collect::<Box<[usize]>>(), &mut encoder)
.map_err(LanguageModelStreamError::Backend)?;
+ if let Some(speculator) = model.speculator.as_ref() {
+ let speculator_state = model_state.speculator_state.as_mut().unwrap();
+ speculator
+ .append_state(
+ speculator_state,
+ decoder_output.hidden_features.as_ref().unwrap(),
+ &(0..input_chunk.len()).collect::<Box<[usize]>>(),
+ &mut encoder,
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
+ }
+
if let Some(suffix_repetition_length) = options.sampling_method.suffix_repetition_length() {
model.context_ring_update.encode(
&token_ids,
@@ -288,7 +320,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
input_trie: TrieNode::new(0, 0),
full_accept: true,
pending,
- output: output.unwrap(),
+ hidden_features: None,
+ output_norm,
+ output_tokens: output_tokens.unwrap(),
})
} else {
// TODO: this leaks previous LanguageModelStreamOptions
@@ -320,7 +354,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let _ = grammar.accept_token(seed_token); // TODO: this should not be ignored
}
self.metrics.num_tokens_returned += 1;
- (ForwardPassChaining::Constant(seed_token), None)
+ (
+ ForwardPassChaining::Constant {
+ token: seed_token,
+ output_norm: None,
+ },
+ None,
+ )
},
DecodingState::ForwardPassPending(forward_pass_pending) => {
if forward_pass_pending.full_accept {
@@ -331,20 +371,27 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
pending.wait_until_completed().map_err(LanguageModelStreamError::Backend)?;
}
let sampled_tokens = forward_pass_pending
- .output
+ .output_tokens
.as_slice::<u32>()
.iter()
.map(|x| *x as u64)
.collect::<Box<[u64]>>();
- let full = forward_pass_pending.input_trie.linearize().accept(
+ let flat_trie = forward_pass_pending.input_trie.linearize();
+ let full = flat_trie.accept(
&sampled_tokens,
#[cfg(grammar)]
self.options.grammar.as_mut(),
)?;
+ let output_norm = forward_pass_pending.output_norm.map(|norm| {
+ let row_bytes = norm.size() / flat_trie.len();
+ (norm, row_bytes)
+ });
self.metrics.num_tokens_accepted += full.len();
self.decoding_state = DecodingState::Accepting {
full,
num_accepted: 0,
+ hidden_features: forward_pass_pending.hidden_features,
+ output_norm,
};
return self.generate();
}
@@ -352,6 +399,8 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
DecodingState::Accepting {
full,
num_accepted,
+ hidden_features,
+ output_norm,
} => {
let output_token_id = full[num_accepted].2;
@@ -361,6 +410,8 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.decoding_state = DecodingState::Accepting {
full,
num_accepted: num_accepted + 1,
+ hidden_features,
+ output_norm,
};
return Ok(Some(output_token_id));
} else {
@@ -374,6 +425,25 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.transformer_state
.encode_accept(&accepted_token_indicies, &mut encoder)
.map_err(LanguageModelStreamError::Backend)?;
+ if let Some(speculator) = self.model.speculator.as_ref() {
+ speculator
+ .append_state(
+ self.model_state.speculator_state.as_mut().unwrap(),
+ hidden_features.as_deref().unwrap(),
+ &accepted_token_indicies,
+ &mut encoder,
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
+ }
+ let output_norm = if let Some((final_hidden, row_bytes)) = output_norm {
+ let row = full.last().unwrap().0;
+ let mut norm =
+ encoder.allocate_scratch(row_bytes).map_err(LanguageModelStreamError::Backend)?;
+ encoder.encode_copy(&final_hidden, row * row_bytes..(row + 1) * row_bytes, &mut norm, ..);
+ Some(norm)
+ } else {
+ None
+ };
if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length()
{
let mut accepted_input_token_ids_const = encoder
@@ -394,7 +464,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
);
}
self.model_state.tokens.extend(accepted_output_token_ids);
- (ForwardPassChaining::Constant(output_token_id), Some(encoder))
+ (
+ ForwardPassChaining::Constant {
+ token: output_token_id,
+ output_norm,
+ },
+ Some(encoder),
+ )
}
},
DecodingState::Halted => return Ok(None),
@@ -405,11 +481,15 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if self.model_state.max_context_length.is_some_and(|max_context_length| context_length >= max_context_length) {
self.decoding_state = DecodingState::Halted;
- return Ok(Some(prev_output.resolve(
- &mut self.model_state.tokens,
- #[cfg(grammar)]
- self.options.grammar.as_mut(),
- )?));
+ return Ok(Some(
+ prev_output
+ .resolve(
+ &mut self.model_state.tokens,
+ #[cfg(grammar)]
+ self.options.grammar.as_mut(),
+ )?
+ .0,
+ ));
}
let mut pending = Vec::new();
@@ -420,30 +500,43 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.map_err(LanguageModelStreamError::Backend)?
};
- let (input_trie, chain_copy, full_accept) = if let Some(speculator) = &self.options.speculator {
- prev_output.resolve(
+ let speculation_batch = self
+ .model_state
+ .max_context_length
+ .map_or(32, |max_context_length| 32.min(max_context_length - context_length));
+
+ let (input_trie, chain_copy, full_accept) = if speculation_batch > 1
+ && let Some(speculator) = &self.model.speculator
+ && let (root_token, Some(output_norm)) = prev_output.resolve(
&mut self.model_state.tokens,
#[cfg(grammar)]
self.options.grammar.as_mut(),
- )?;
-
- let input_trie = TrieNode::from_speculator(
- &self.model_state.tokens,
+ )? {
+ pending.push(encoder.end_encoding().submit());
+ encoder = Encoder::<B>::new_with_pool(&self.model.context, self.allocation_pool.clone())
+ .map_err(LanguageModelStreamError::Backend)?;
+ let trie = speculator.propose_tree(
+ self.model_state.speculator_state.as_mut().unwrap(),
+ output_norm,
+ root_token as u32,
+ self.model.decoder.embedding(),
&self.model_state.prng,
#[cfg(grammar)]
self.options.grammar.as_mut(),
- speculator.speculator,
- self.model.vocab_size,
- &speculator.trie_creation_config,
- speculator.speculation_budget,
- self.model_state.max_context_length.map(|max_context_length| max_context_length - context_length),
- );
-
- (input_trie, None, false)
+ DFlashTreeOptions {
+ budget: speculation_batch - 1,
+ frontier_width: 4,
+ children_per_node: 4,
+ },
+ )?;
+ (trie, None, false)
} else {
let (token, chain_copy) = match &prev_output {
- ForwardPassChaining::Constant(token) => (*token, None),
- ForwardPassChaining::InFlight(pending) => (0, Some(&pending.output)),
+ ForwardPassChaining::Constant {
+ token,
+ ..
+ } => (*token, None),
+ ForwardPassChaining::InFlight(pending) => (0, Some(&pending.output_tokens)),
};
(TrieNode::new(token, self.model_state.prng.derive(context_length as u64)), chain_copy, true)
};
@@ -470,19 +563,18 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.prepare(self.model_state.transformer_state.context_length(), batch_dim.size(), &self.model.context)
.map_err(LanguageModelStreamError::Backend)?;
- let logits = self
- .model
- .decoder
- .encode(
- &token_ids,
- &batch_dim,
- Some(0..batch_dim.size()),
- &mut self.model_state.transformer_state,
- &mut encoder,
- &[],
- )?
- .logits
- .unwrap();
+ let hidden_feature_layer_indices =
+ self.model.speculator.as_ref().map(|speculator| speculator.hidden_feature_layer_indices());
+
+ let decoder_output = self.model.decoder.encode(
+ &token_ids,
+ &batch_dim,
+ Some(0..batch_dim.size()),
+ hidden_feature_layer_indices,
+ &mut self.model_state.transformer_state,
+ &mut encoder,
+ )?;
+ let logits = decoder_output.logits.unwrap();
#[cfg(grammar)]
let (bitmask, mut encoder) = if let Some(grammar) = self.options.grammar.as_mut() {
@@ -535,7 +627,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
None
};
- let output = self
+ let output_tokens = self
.model
.sampling
.encode(
@@ -561,6 +653,18 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.encode_accept(&(0..batch_dim.size()).collect::<Box<[usize]>>(), &mut encoder)
.map_err(LanguageModelStreamError::Backend)?;
+ if let Some(speculator) = self.model.speculator.as_ref() {
+ let speculator_state = self.model_state.speculator_state.as_mut().unwrap();
+ speculator
+ .append_state(
+ speculator_state,
+ decoder_output.hidden_features.as_ref().unwrap(),
+ &(0..batch_dim.size()).collect::<Box<[usize]>>(),
+ &mut encoder,
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
+ }
+
if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {
self.model.context_ring_update.encode(
&token_ids,
@@ -586,14 +690,24 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
input_trie,
full_accept,
pending: pending.into_boxed_slice(),
- output,
+ hidden_features: if full_accept {
+ None
+ } else {
+ decoder_output.hidden_features
+ },
+ output_norm: decoder_output.final_hidden,
+ output_tokens,
});
- Ok(Some(prev_output.resolve(
- &mut self.model_state.tokens,
- #[cfg(grammar)]
- self.options.grammar.as_mut(),
- )?))
+ Ok(Some(
+ prev_output
+ .resolve(
+ &mut self.model_state.tokens,
+ #[cfg(grammar)]
+ self.options.grammar.as_mut(),
+ )?
+ .0,
+ ))
}
pub fn metrics(&self) -> &TokenStreamMetrics {
@@ -624,26 +738,44 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
let mut encoder =
Encoder::<B>::new_with_pool(&self.model.context, self.allocation_pool.clone()).unwrap();
self.model_state.transformer_state.encode_accept(&[0], &mut encoder).unwrap();
+ if let Some(speculator) = self.model.speculator.as_ref() {
+ speculator
+ .append_state(
+ self.model_state.speculator_state.as_mut().unwrap(),
+ in_flight.hidden_features.as_deref().unwrap(),
+ &[0],
+ &mut encoder,
+ )
+ .unwrap();
+ }
encoder.end_encoding().submit().wait_until_completed().unwrap();
}
- Some(in_flight.output.as_slice::<u32>()[0] as u64)
+ Some(in_flight.output_tokens.as_slice::<u32>()[0] as u64)
},
DecodingState::Accepting {
full,
num_accepted,
+ hidden_features,
+ output_norm: _,
} => {
assert!(num_accepted > 0 && num_accepted < full.len());
let mut encoder =
Encoder::<B>::new_with_pool(&self.model.context, self.allocation_pool.clone()).unwrap();
- self.model_state
- .transformer_state
- .encode_accept(
- &full.iter().take(num_accepted + 1).map(|(i, _, _)| *i).collect::<Box<[usize]>>(),
- &mut encoder,
- )
- .unwrap();
+ let accepted_token_indicies =
+ full.iter().take(num_accepted + 1).map(|(i, _, _)| *i).collect::<Box<[usize]>>();
+ self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut encoder).unwrap();
+ if let Some(speculator) = self.model.speculator.as_ref() {
+ speculator
+ .append_state(
+ self.model_state.speculator_state.as_mut().unwrap(),
+ hidden_features.as_deref().unwrap(),
+ &accepted_token_indicies,
+ &mut encoder,
+ )
+ .unwrap();
+ }
encoder.end_encoding().submit().wait_until_completed().unwrap();
self.model_state.tokens.extend(full.iter().take(num_accepted).map(|(_, _, t)| *t));
diff --git a/crates/backend-uzu/src/engine/mod.rs b/crates/backend-uzu/src/engine/mod.rs
index b8cc6667a..e6c733ad6 100644
--- a/crates/backend-uzu/src/engine/mod.rs
+++ b/crates/backend-uzu/src/engine/mod.rs
@@ -5,7 +5,6 @@ use thiserror::Error;
use crate::backends::common::{Backend, Context};
pub mod classifier_model;
-pub mod dflash_speculator;
pub mod language_model;
pub struct Engine<B: Backend> {
diff --git a/crates/backend-uzu/src/lib.rs b/crates/backend-uzu/src/lib.rs
index 954a99876..fff204135 100644
--- a/crates/backend-uzu/src/lib.rs
+++ b/crates/backend-uzu/src/lib.rs
@@ -5,7 +5,7 @@ mod array;
mod config;
mod encodable_block;
mod parameters;
-pub mod speculators;
+mod speculators;
mod trie;
mod utils;
diff --git a/crates/backend-uzu/src/speculators/chain_acceptance.rs b/crates/backend-uzu/src/speculators/chain_acceptance.rs
deleted file mode 100644
index ddc51e75c..000000000
--- a/crates/backend-uzu/src/speculators/chain_acceptance.rs
+++ /dev/null
@@ -1,88 +0,0 @@
-pub struct ChainProposal {
- pub token_ids: Box<[u64]>,
- pub token_positions: Box<[usize]>,
-}
-
-pub struct AcceptedProposal {
- pub committed_token_ids: Vec<u64>,
- pub committed_token_positions: Vec<usize>,
- pub num_verified_nodes: usize,
-}
-
-impl ChainProposal {
- /// `sampled_token_ids[i]` is the target's next token after node `i`.
- pub fn accept(
- &self,
- sampled_token_ids: &[u64],
- remaining_length: usize,
- eos_token_ids: &[u64],
- ) -> AcceptedProposal {
- let num_nodes = self.token_ids.len();
- debug_assert!(num_nodes > 0, "chain proposal must contain at least the last committed token");
- debug_assert_eq!(self.token_positions.len(), num_nodes, "token_positions length must match token_ids");
- debug_assert_eq!(sampled_token_ids.len(), num_nodes, "expected one sampled token per proposal node");
-
- let num_verified_drafts =
- (0..num_nodes - 1).take_while(|&i| self.token_ids[i + 1] == sampled_token_ids[i]).count();
- let num_verified_nodes = num_verified_drafts + 1;
-
- let mut committed_token_ids = Vec::with_capacity(num_verified_nodes);
- let mut committed_token_positions = Vec::with_capacity(num_verified_nodes);
- let mut has_draft_eos = false;
- for draft in 0..num_verified_drafts {
- let token_id = self.token_ids[draft + 1];
- committed_token_ids.push(token_id);
- committed_token_positions.push(self.token_positions[draft + 1]);
- if eos_token_ids.contains(&token_id) {
- has_draft_eos = true;
- break;
- }
- }
-
- // A draft EOS suppresses the bonus token; a bonus EOS is kept.
- if !has_draft_eos {
- committed_token_ids.push(sampled_token_ids[num_verified_drafts]);
- committed_token_positions.push(self.token_positions[num_verified_drafts] + 1);
- }
-
- committed_token_ids.truncate(remaining_length);
- committed_token_positions.truncate(remaining_length);
-
- AcceptedProposal {
- committed_token_ids,
- committed_token_positions,
- num_verified_nodes,
- }
- }
-}
-
-impl AcceptedProposal {
- pub fn next_context_nodes(&self) -> std::ops::Range<usize> {
- 0..self.num_verified_nodes
- }
-
- pub fn last_token_id(&self) -> Option<u64> {
- self.committed_token_ids.last().copied()
- }
-
- /// One behind the last committed token (the bonus, not yet forward-passed).
- pub fn last_covered_position(&self) -> Option<usize> {
- if let Some(position) = self.committed_token_positions.last() {
- debug_assert!(*position >= 1, "committed token positions are 1-based");
- Some(position - 1)
- } else {
- None
- }
- }
-
- pub fn has_eos(
- &self,
- eos_token_ids: &[u64],
- ) -> bool {
- self.committed_token_ids.iter().any(|token| eos_token_ids.contains(token))
- }
-}
-
-#[cfg(test)]
-#[path = "../../tests/unit/speculators/chain_acceptance.rs"]
-mod tests;
diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rs
index 20591d36d..b187148c8 100644
--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs
+++ b/crates/backend-uzu/src/speculators/dflash_speculator.rs
@@ -1,22 +1,27 @@
-use std::sync::Arc;
+use std::{
+ fs::File,
+ io::{self, BufReader},
+ path::Path,
+ sync::Arc,
+};
use thiserror::Error;
pub use crate::encodable_block::dflash::DFlashState;
+#[cfg(grammar)]
+use crate::engine::language_model::grammar::Grammar;
use crate::{
- backends::common::{
- Allocation, AllocationType, Backend, Context, Encoder, Kernels,
- gpu_types::weaver::{FRONTIER_MAX_SLOTS, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx},
- kernel::{WeaverFrontierScatterKernel, WeaverFrontierSelectKernel},
- },
- config::speculator::dflash::DFlashSpeculatorConfig,
+ backends::common::{Allocation, Backend, Encoder},
+ config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},
data_type::DataType,
encodable_block::{
- dflash::DFlashDraft,
+ dflash::{DFlashDraft, DFlashDraftNewError},
embedding::{Embedding, EmbeddingError},
- weaver::{Weaver, WeaverEncodeError, WeaverNodeKvCache, WeaverPrefixKvCache, WeaverStepBatch},
+ sampling::PRng,
+ weaver::{ProposalNode, Weaver, WeaverEncodeError, WeaverNewError, WeaverTreeInput},
},
- engine::language_model::LanguageModel,
+ parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},
+ trie::TrieNode,
};
#[derive(Clone, Copy, Debug)]
@@ -30,25 +35,6 @@ const MAX_TREE_BUDGET: usize = 4096;
const MAX_TREE_FRONTIER_WIDTH: usize = 8;
const MAX_CHILDREN_PER_NODE: usize = 8;
-impl Default for DFlashTreeOptions {
- fn default() -> Self {
- Self {
- budget: 128,
- frontier_width: 8,
- children_per_node: 8,
- }
- }
-}
-
-pub struct TreeProposal<B: Backend> {
- /// DFS order; slot 0 is the bonus root.
- pub token_ids: Allocation<B>,
- pub parents: Allocation<B>,
- pub depths: Allocation<B>,
- pub draft_logprobs: Allocation<B>,
- pub length: usize,
-}
-
#[derive(Debug, Error)]
pub enum DFlashTreeError<B: Backend> {
#[error("backend error: {0}")]
@@ -57,20 +43,10 @@ pub enum DFlashTreeError<B: Backend> {
Embedding(#[from] EmbeddingError<B>),
#[error("Weaver error: {0}")]
Weaver(#[from] WeaverEncodeError<B>),
- #[error("DFlash tree construction requires target hidden features in the state")]
- MissingTargetHidden,
#[error("invalid tree options")]
InvalidOptions,
}
-struct HostTreeNode {
- token: u32,
- parent: Option<usize>,
- depth: usize,
- logprob: f32,
- children: Vec<usize>,
-}
-
struct DFlashChainOutput<B: Backend> {
pool_ids: Allocation<B>,
pool_scores: Allocation<B>,
@@ -78,41 +54,104 @@ struct DFlashChainOutput<B: Backend> {
draft_hidden: Allocation<B>,
}
-pub struct DFlashSpeculator<B: Backend> {
- pub(crate) context: Arc<B::Context>,
- pub(crate) model: DFlashDraft<B>,
- pub(crate) weaver: Option<Weaver<B>>,
- pub(crate) config: DFlashSpeculatorConfig,
+#[derive(Debug, Error)]
+pub enum DFlashSpeculatorLoadError<B: Backend> {
+ #[error("I/O error: {0}")]
+ IO(#[from] io::Error),
+ #[error("Serde error: {0}")]
+ Serde(#[from] serde_json::Error),
+ #[error("HeaderLoading error: {0}")]
+ HeaderLoading(#[from] HeaderLoadingError),
+ #[error("ParameterLoader error: {0}")]
+ ParameterLoader(#[from] ParameterLoaderError<B>),
+ #[error("DFlash draft error: {0}")]
+ Draft(#[from] DFlashDraftNewError<B>),
+ #[error("Weaver error: {0}")]
+ Weaver(#[from] WeaverNewError<B>),
+ #[error("DFlash mask_token_id {mask_token_id} is outside vocabulary size {vocab_size}")]
+ InvalidMaskTokenId {
+ mask_token_id: u64,
+ vocab_size: usize,
+ },
}
-#[derive(Clone, Copy)]
-pub struct DFlashTfm<'a, B: Backend> {
- speculator: &'a DFlashSpeculator<B>,
- target_embedding: &'a Embedding<B>,
+pub struct DFlashSpeculator<B: Backend> {
+ context: Arc<B::Context>,
+ model: DFlashDraft<B>,
+ weaver: Option<Weaver<B>>,
+ config: DFlashSpeculatorConfig,
}
impl<B: Backend> DFlashSpeculator<B> {
- pub(crate) const fn new(
+ pub fn load(
+ model_path: &Path,
context: Arc<B::Context>,
- model: DFlashDraft<B>,
- weaver: Option<Weaver<B>>,
- config: DFlashSpeculatorConfig,
- ) -> Self {
- Self {
+ ) -> Result<Self, DFlashSpeculatorLoadError<B>> {
+ let config: SpeculatorModelConfig =
+ serde_json::from_reader(BufReader::new(File::open(model_path.join("config.json"))?))?;
+ let AnySpeculatorConfig::DFlashSpeculatorConfig(config) = config.speculator_config;
+ let draft_config = &config.draft_config;
+ if draft_config.mask_token_id >= draft_config.vocab_size as u64 {
+ return Err(DFlashSpeculatorLoadError::InvalidMaskTokenId {
+ mask_token_id: draft_config.mask_token_id,
+ vocab_size: draft_config.vocab_size,
+ });
+ }
+
+ let data_type = DataType::BF16;
+
+ let weights_file = File::open(model_path.join("model.safetensors"))?;
+ let weight_loader = ParameterLoader::new(&weights_file, &*context)?;
+ let speculator_tree = weight_loader.tree().subtree("speculator")?;
+ let model =
+ DFlashDraft::new(&*context, &config.draft_config, &speculator_tree.subtree("draft_model")?, data_type)?;
+ let weaver = config
+ .weaver_config
+ .as_ref()
+ .map(|weaver_config| Weaver::new(&*context, weaver_config, &speculator_tree.subtree("weaver")?))
+ .transpose()?;
+
+ weight_loader.tree().assert_all_tensors_validated()?;
+
+ Ok(Self {
context,
model,
weaver,
config,
- }
+ })
}
- fn propose_tree(
+ pub fn hidden_feature_layer_indices(&self) -> &[usize] {
+ &self.config.draft_config.target_layer_ids
+ }
+
+ pub fn empty_state(
+ &self,
+ context_capacity: usize,
+ ) -> Result<DFlashState<B>, B::Error> {
+ self.model.empty_state(context_capacity, &self.context)
+ }
+
+ pub fn append_state(
+ &self,
+ state: &mut DFlashState<B>,
+ target_features: &[Allocation<B>],
+ accepted_indices: &[usize],
+ encoder: &mut Encoder<B>,
+ ) -> Result<(), B::Error> {
+ self.model.append_state(state, target_features, accepted_indices, encoder)
+ }
+
+ pub fn propose_tree(
&self,
state: &mut DFlashState<B>,
- bonus_token: u32,
+ target_output_norm: &Allocation<B>,
+ target_output_token: u32,
target_embedding: &Embedding<B>,
+ prng: &PRng,
+ #[cfg(grammar)] grammar: Option<&mut Grammar>,
options: DFlashTreeOptions,
- ) -> Result<TreeProposal<B>, DFlashTreeError<B>> {
+ ) -> Result<TrieNode, DFlashTreeError<B>> {
if options.budget == 0
|| options.budget > MAX_TREE_BUDGET
|| options.frontier_width == 0
@@ -127,7 +166,7 @@ impl<B: Backend> DFlashSpeculator<B> {
let vocab_size = self.config.draft_config.vocab_size;
let pool_size =
self.config.weaver_config.as_ref().map_or(1, |config| config.candidate_pool_size.min(vocab_size));
- if bonus_token as usize >= vocab_size
+ if target_output_token as usize >= vocab_size
|| pool_size == 0
|| options.children_per_node > pool_size
|| target_embedding.vocab_size() != self.config.draft_config.vocab_size
@@ -136,6 +175,7 @@ impl<B: Backend> DFlashSpeculator<B> {
{
return Err(DFlashTreeError::InvalidOptions);
}
+ let root_position = state.context_length();
let mut encoder = Encoder::new(&*self.context).map_err(DFlashTreeError::Backend)?;
let DFlashChainOutput {
@@ -146,7 +186,8 @@ impl<B: Backend> DFlashSpeculator<B> {
} = self.encode_dflash_chain(
&mut encoder,
state,
- bonus_token,
+ target_output_norm,
+ target_output_token,
target_embedding,
block_size,
target_model_dim,
@@ -155,98 +196,99 @@ impl<B: Backend> DFlashSpeculator<B> {
if let Some(weaver) = self.weaver.as_ref() {
let max_depth = self.config.weaver_config.as_ref().expect("a Weaver implies a Weaver config").max_depth;
- let lookahead_count = max_depth.min(block_size.saturating_sub(1));
- // The frontier holds one slot per (tree slot, child) pair; the select
- // kernel silently no-ops past its capacity.
- if (options.budget + 1) * options.children_per_node > FRONTIER_MAX_SLOTS {
- return Err(DFlashTreeError::InvalidOptions);
- }
- let target_hidden = state.target_output_norm().ok_or(DFlashTreeError::MissingTargetHidden)?;
- let prefix = weaver.build_prefix(target_hidden, &draft_hidden, lookahead_count, &mut encoder)?;
- drop(draft_hidden);
- drop(draft_logits);
- let mut weaver_state = weaver.create_node_kv_cache(options.budget + 1, &self.context)?;
- let arguments = TreeEncodingArguments {
- weaver,
- prefix: &prefix,
- target_embedding,
- pool_ids: &pool_ids,
- pool_scores: &pool_scores,
- pool_rows: block_size - 1,
- pool_size,
- options,
- max_depth,
- lookahead_count,
- bonus_token,
- };
- let tree = self.encode_tree(&mut encoder, &arguments, &mut weaver_state)?;
+ let depth_seeds =
+ (0..max_depth).map(|depth| prng.derive((root_position + depth) as u64)).collect::<Box<[u64]>>();
+ let tree = weaver.encode_tree(
+ WeaverTreeInput {
+ target_hidden: target_output_norm,
+ draft_hidden: &draft_hidden,
+ target_embedding,
+ candidate_ids: &pool_ids,
+ candidate_scores: &pool_scores,
+ candidate_rows: block_size - 1,
+ candidates_per_row: pool_size,
+ depth_seeds: &depth_seeds,
+ root_token_id: target_output_token,
+ tree_budget: options.budget,
+ frontier_width: options.frontier_width,
+ children_per_node: options.children_per_node,
+ },
+ &self.context,
+ &mut encoder,
+ )?;
let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
- let slots = tree.copyout::<u32>();
- drop(tree);
+ let nodes = tree.decode();
drop(pool_ids);
drop(pool_scores);
+ drop(draft_logits);
+ drop(draft_hidden);
drop(completed);
- return Self::finish_tree(&self.context, tree_from_slots(&slots));
+ return Ok(Self::finish_tree(
+ nodes,
+ prng,
+ root_position,
+ #[cfg(grammar)]
+ grammar,
+ ));
}
let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
let pool_id_values = pool_ids.copyout::<u32>();
- let host_logits = draft_logits.copyout::<f32>();
drop(pool_ids);
drop(pool_scores);
drop(draft_logits);
drop(draft_hidden);
drop(completed);
- let mut nodes = vec![HostTreeNode {
- token: bonus_token,
- parent: None,
+ let mut nodes = vec![ProposalNode {
+ token: target_output_token,
depth: 0,
- logprob: 0.0,
children: Vec::new(),
}];
for depth in 0..options.budget.min(block_size.saturating_sub(1)) {
let token = pool_id_values[depth * pool_size];
- let row_start = depth * vocab_size;
- let logprob = log_softmax(&host_logits[row_start..row_start + vocab_size])[token as usize];
let parent = nodes.len() - 1;
let index = nodes.len();
- nodes.push(HostTreeNode {
+ nodes.push(ProposalNode {
token,
- parent: Some(parent),
depth: depth + 1,
- logprob,
children: Vec::new(),
});
nodes[parent].children.push(index);
}
- Self::finish_tree(&self.context, nodes)
+ Ok(Self::finish_tree(
+ nodes,
+ prng,
+ root_position,
+ #[cfg(grammar)]
+ grammar,
+ ))
}
fn encode_dflash_chain(
&self,
encoder: &mut Encoder<B>,
state: &mut DFlashState<B>,
- bonus_token: u32,
+ target_output_norm: &Allocation<B>,
+ target_output_token: u32,
target_embedding: &Embedding<B>,
block_size: usize,
target_model_dim: usize,
pool_size: usize,
) -> Result<DFlashChainOutput<B>, DFlashTreeError<B>> {
- let target_hidden_size = state.target_output_norm().ok_or(DFlashTreeError::MissingTargetHidden)?.size();
- if target_hidden_size != target_model_dim * DataType::BF16.size_in_bytes() {
+ if target_output_norm.size() != target_model_dim * DataType::BF16.size_in_bytes() {
return Err(DFlashTreeError::InvalidOptions);
}
let mut noise_ids =
encoder.allocate_constant(block_size * DataType::U32.size_in_bytes()).map_err(DFlashTreeError::Backend)?;
let mut noise = vec![self.config.draft_config.mask_token_id as u32; block_size];
- noise[0] = bonus_token;
+ noise[0] = target_output_token;
noise_ids.copyin(&noise);
let token_embeddings = target_embedding.encode_lookup(&noise_ids, block_size, encoder)?;
let draft_hidden =
self.model.encode_block(state, token_embeddings, encoder).map_err(DFlashTreeError::Backend)?;
- // The first block row is the bonus token; only the lookahead rows are ranked.
+ // The first block row is the target's output token; only the lookahead rows are ranked.
let row_bytes = target_embedding.model_dim() * DataType::BF16.size_in_bytes();
let mut lookahead_hidden =
encoder.allocate_scratch((block_size - 1) * row_bytes).map_err(DFlashTreeError::Backend)?;
@@ -268,274 +310,50 @@ impl<B: Backend> DFlashSpeculator<B> {
}
fn finish_tree(
- context: &B::Context,
- nodes: Vec<HostTreeNode>,
- ) -> Result<TreeProposal<B>, DFlashTreeError<B>> {
- let mut order = Vec::with_capacity(nodes.len());
- let mut stack = vec![0usize];
- while let Some(index) = stack.pop() {
- order.push(index);
- for &child in nodes[index].children.iter().rev() {
- stack.push(child);
- }
- }
- let mut expansion_to_dfs = vec![u32::MAX; order.len()];
- for (dfs, &expansion) in order.iter().enumerate() {
- expansion_to_dfs[expansion] = dfs as u32;
- }
- let mut tokens = vec![0u32; order.len()];
- let mut parents = vec![-1i32; order.len()];
- let mut depths = vec![0u32; order.len()];
- let mut draft_logprobs = vec![0.0f32; order.len()];
- for (dfs, &index) in order.iter().enumerate() {
- tokens[dfs] = nodes[index].token;
- parents[dfs] = nodes[index].parent.map_or(-1, |parent| expansion_to_dfs[parent] as i32);
- depths[dfs] = nodes[index].depth as u32;
- draft_logprobs[dfs] = nodes[index].logprob;
- }
- let tokens = global_allocation::<B, _>(context, &tokens).map_err(DFlashTreeError::Backend)?;
- let parents = global_allocation::<B, _>(context, &parents).map_err(DFlashTreeError::Backend)?;
- let depths = global_allocation::<B, _>(context, &depths).map_err(DFlashTreeError::Backend)?;
- let draft_logprobs = global_allocation::<B, _>(context, &draft_logprobs).map_err(DFlashTreeError::Backend)?;
- Ok(TreeProposal {
- token_ids: tokens,
- parents,
- depths,
- draft_logprobs,
- length: order.len(),
- })
- }
-
- fn encode_tree(
- &self,
- encoder: &mut Encoder<B>,
- params: &TreeEncodingArguments<'_, B>,
- state: &mut WeaverNodeKvCache<B>,
- ) -> Result<Allocation<B>, DFlashTreeError<B>> {
- let context = &*self.context;
- let slots = params.options.budget + 1;
- let children_per_node = params.options.children_per_node;
- let capacity = slots * children_per_node;
- let width = params.options.frontier_width;
- let stride = params.max_depth;
- let pool_size = params.pool_size;
-
- let select =
- <B::Kernels as Kernels>::WeaverFrontierSelectKernel::new(context).map_err(DFlashTreeError::Backend)?;
- let scatter =
- <B::Kernels as Kernels>::WeaverFrontierScatterKernel::new(context).map_err(DFlashTreeError::Backend)?;
-
- let mut tree_values = vec![0u32; TreeIdx::COUNT * slots];
- for slot in 0..slots {
- tree_values[TreeIdx::ParentSlot as usize * slots + slot] = FRONTIER_NO_WINNER;
- }
- tree_values[TreeIdx::TokenId as usize * slots] = params.bonus_token;
- tree_values[TreeIdx::Valid as usize * slots] = 1;
-
- let mut tree = encoder.allocate_constant_from_slice(&tree_values).map_err(DFlashTreeError::Backend)?;
- let mut frontier = encoder
- .allocate_constant_from_slice(&vec![0u32; FrontierIdx::COUNT * capacity])
- .map_err(DFlashTreeError::Backend)?;
- let mut slot_ancestors =
- encoder.allocate_constant_from_slice(&vec![0u32; slots * stride]).map_err(DFlashTreeError::Backend)?;
-
- let mut round_token_id_values = vec![0u32; width];
- round_token_id_values[0] = params.bonus_token;
- let mut round_valid_values = vec![0u32; width];
- round_valid_values[0] = 1;
- let mut round_token_ids =
- encoder.allocate_constant_from_slice(&round_token_id_values).map_err(DFlashTreeError::Backend)?;
- let mut round_metadata = encoder
- .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * width])
- .map_err(DFlashTreeError::Backend)?;
- let mut round_ancestors =
- encoder.allocate_constant_from_slice(&vec![0u32; width * stride]).map_err(DFlashTreeError::Backend)?;
- let mut round_valid =
- encoder.allocate_constant_from_slice(&round_valid_values).map_err(DFlashTreeError::Backend)?;
- let mut round_candidate_ids =
- encoder.allocate_constant_from_slice(&vec![0u32; width * pool_size]).map_err(DFlashTreeError::Backend)?;
- let mut round_candidate_scores =
- encoder.allocate_constant_from_slice(&vec![0.0f32; width * pool_size]).map_err(DFlashTreeError::Backend)?;
-
- let mut slot_start = 0;
- while slot_start < slots {
- let rows = if slot_start == 0 {
- 1
- } else {
- width.min(slots - slot_start)
- };
- if slot_start > 0 {
- select.encode(
- &mut frontier,
- &mut tree,
- &mut slot_ancestors,
- &mut round_token_ids,
- &mut round_metadata,
- &mut round_ancestors,
- &mut round_valid,
- params.pool_ids,
- params.pool_scores,
- &mut round_candidate_ids,
- &mut round_candidate_scores,
- capacity as u32,
- slots as u32,
- rows as u32,
- slot_start as u32,
- stride as u32,
- params.max_depth as u32,
- params.lookahead_count as u32,
- params.pool_rows as u32,
- pool_size as u32,
- encoder,
+ nodes: Vec<ProposalNode>,
+ prng: &PRng,
+ root_position: usize,
+ #[cfg(grammar)] mut grammar: Option<&mut Grammar>,
+ ) -> TrieNode {
+ fn build(
+ nodes: &[ProposalNode],
+ index: usize,
+ prng: &PRng,
+ root_position: usize,
+ #[cfg(grammar)] grammar: &mut Option<&mut Grammar>,
+ ) -> TrieNode {
+ let node = &nodes[index];
+ let mut trie_node = TrieNode::new(node.token as u64, prng.derive((root_position + node.depth) as u64));
+ for &child_index in &node.children {
+ #[cfg(grammar)]
+ if let Some(grammar) = grammar.as_mut()
+ && grammar.accept_token(nodes[child_index].token as u64).is_err()
+ {
+ continue;
+ }
+ let child = build(
+ nodes,
+ child_index,
+ prng,
+ root_position,
+ #[cfg(grammar)]
+ grammar,
);
+ #[cfg(grammar)]
+ if let Some(grammar) = grammar.as_mut() {
+ grammar.rollback(1);
+ }
+ trie_node.add(child).expect("tree children are selected without replacement");
}
- let (candidate_ids, candidate_scores) = if slot_start == 0 {
- (params.pool_ids, params.pool_scores)
- } else {
- (&round_candidate_ids, &round_candidate_scores)
- };
- let input = WeaverStepBatch {
- node_count: rows,
- candidates_per_node: pool_size,
- ancestor_stride: stride,
- node_token_ids: &round_token_ids,
- candidate_ids,
- candidate_logits: candidate_scores,
- ancestor_indices: &round_ancestors,
- node_metadata: &round_metadata,
- };
- let children = params.weaver.encode_step_batch(
- params.prefix,
- &input,
- state,
- children_per_node,
- params.target_embedding,
- encoder,
- )?;
- scatter.encode(
- &tree,
- &round_metadata,
- &round_valid,
- &children.token_ids,
- &children.logprobs,
- &mut frontier,
- capacity as u32,
- slots as u32,
- rows as u32,
- children_per_node as u32,
- encoder,
- );
- drop(children);
- slot_start += rows;
+ trie_node
}
- Ok(tree)
- }
-}
-
-fn tree_from_slots(tree: &[u32]) -> Vec<HostTreeNode> {
- assert!(tree.len().is_multiple_of(TreeIdx::COUNT), "tree array must contain {} equal-length lanes", TreeIdx::COUNT);
- let slots = tree.len() / TreeIdx::COUNT;
- let field = |field: TreeIdx, slot: usize| tree[field as usize * slots + slot];
- let mut slot_to_node = vec![usize::MAX; slots];
- let mut nodes: Vec<HostTreeNode> = Vec::with_capacity(slots);
- for slot in 0..slots {
- if field(TreeIdx::Valid, slot) == 0 {
- continue;
- }
- let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;
- let parent = (parent_slot >= 0).then(|| {
- let parent = slot_to_node[parent_slot as usize];
- assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");
- parent
- });
- let index = nodes.len();
- slot_to_node[slot] = index;
- if let Some(parent) = parent {
- nodes[parent].children.push(index);
- }
- nodes.push(HostTreeNode {
- token: field(TreeIdx::TokenId, slot),
- parent,
- depth: field(TreeIdx::Depth, slot) as usize,
- logprob: f32::from_bits(field(TreeIdx::EdgeLogprobBits, slot)),
- children: Vec::new(),
- });
- }
- nodes
-}
-
-struct TreeEncodingArguments<'a, B: Backend> {
- weaver: &'a Weaver<B>,
- prefix: &'a WeaverPrefixKvCache<B>,
- target_embedding: &'a Embedding<B>,
- pool_ids: &'a Allocation<B>,
- pool_scores: &'a Allocation<B>,
- pool_rows: usize,
- pool_size: usize,
- options: DFlashTreeOptions,
- max_depth: usize,
- lookahead_count: usize,
- bonus_token: u32,
-}
-
-impl<'a, B: Backend> DFlashTfm<'a, B> {
- pub fn new(
- speculator: &'a DFlashSpeculator<B>,
- target: &'a LanguageModel<B>,
- ) -> Self {
- Self {
- speculator,
- target_embedding: target.embedding(),
- }
- }
-}
-
-impl<B: Backend> DFlashTfm<'_, B> {
- pub fn empty_state(
- &self,
- context_capacity: usize,
- ) -> Result<DFlashState<B>, B::Error> {
- self.speculator.model.empty_state(context_capacity, &self.speculator.context)
- }
-
- pub fn hidden_feature_layer_indices(&self) -> &[usize] {
- &self.speculator.config.draft_config.target_layer_ids
+ build(
+ &nodes,
+ 0,
+ prng,
+ root_position,
+ #[cfg(grammar)]
+ &mut grammar,
+ )
}
-
- pub fn append_state(
- &self,
- state: &mut DFlashState<B>,
- target_features: &[Allocation<B>],
- num_tokens: usize,
- target_output_norm: &Allocation<B>,
- encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- self.speculator.model.append_state(state, target_features, num_tokens, target_output_norm, encoder)
- }
-
- /// Proposes a tree rooted at `bonus_token`.
- /// The last append encoder must be complete.
- pub fn propose_tree(
- &self,
- state: &mut DFlashState<B>,
- bonus_token: u32,
- options: DFlashTreeOptions,
- ) -> Result<TreeProposal<B>, DFlashTreeError<B>> {
- self.speculator.propose_tree(state, bonus_token, self.target_embedding, options)
- }
-}
-
-fn log_softmax(logits: &[f32]) -> Vec<f32> {
- let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
- let log_sum = logits.iter().map(|value| (value - max).exp()).sum::<f32>().ln() + max;
- logits.iter().map(|value| value - log_sum).collect()
-}
-
-fn global_allocation<B: Backend, T: bytemuck::NoUninit + bytemuck::AnyBitPattern>(
- context: &B::Context,
- values: &[T],
-) -> Result<Allocation<B>, B::Error> {
- let mut allocation = context.create_allocation(std::mem::size_of_val(values), AllocationType::Global)?;
- allocation.copyin(values);
- Ok(allocation)
}
diff --git a/crates/backend-uzu/src/speculators/empty_speculator.rs b/crates/backend-uzu/src/speculators/empty_speculator.rs
deleted file mode 100644
index bc3d74d43..000000000
--- a/crates/backend-uzu/src/speculators/empty_speculator.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-use std::collections::HashMap;
-
-use super::speculator::Speculator;
-
-#[derive(Default, Debug, Clone, Copy)]
-pub struct EmptySpeculator;
-
-impl Speculator for EmptySpeculator {
- fn speculate(
- &self,
- _prefix: &[u64],
- ) -> HashMap<u64, f32> {
- HashMap::new()
- }
-}
diff --git a/crates/backend-uzu/src/speculators/fixed_token_speculator.rs b/crates/backend-uzu/src/speculators/fixed_token_speculator.rs
deleted file mode 100644
index 0bdd39ebc..000000000
--- a/crates/backend-uzu/src/speculators/fixed_token_speculator.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-use std::collections::HashMap;
-
-use itertools::Itertools;
-
-use crate::speculators::speculator::Speculator;
-
-pub struct FixedTokensSpeculator {
- tokens: Vec<Vec<u64>>,
-}
-
-impl FixedTokensSpeculator {
- pub fn new(tokens: Vec<Vec<u64>>) -> Self {
- Self {
- tokens,
- }
- }
-
- pub fn max_trie_nodes(&self) -> usize {
- self.tokens.iter().flat_map(|sequence| (0..=sequence.len()).map(|len| &sequence[..len])).unique().count()
- }
-}
-
-impl Speculator for FixedTokensSpeculator {
- fn speculate(
- &self,
- prefix: &[u64],
- ) -> HashMap<u64, f32> {
- let last_prefix_token = prefix.last().copied();
- let mut plausible_tokens = HashMap::new();
-
- for proposed_seq in &self.tokens {
- let mut proposal_index = 0;
-
- for (index, &token) in proposed_seq.iter().enumerate() {
- if last_prefix_token == Some(token) {
- proposal_index = index + 1;
- break;
- }
- }
-
- if let Some(&proposed_token) = proposed_seq.get(proposal_index).or(proposed_seq.first()) {
- *plausible_tokens.entry(proposed_token).or_insert(0.0) += 1.0
- }
- }
-
- let sum: f32 = plausible_tokens.values().sum();
- plausible_tokens.into_iter().map(|(k, v)| (k, v / sum)).collect()
- }
-}
-
-#[cfg(test)]
-#[path = "../../tests/unit/speculators/fixed_token_speculator.rs"]
-mod tests;
diff --git a/crates/backend-uzu/src/speculators/mod.rs b/crates/backend-uzu/src/speculators/mod.rs
index 0107e7b52..963522197 100644
--- a/crates/backend-uzu/src/speculators/mod.rs
+++ b/crates/backend-uzu/src/speculators/mod.rs
@@ -1,9 +1 @@
-pub mod chain_acceptance;
pub mod dflash_speculator;
-pub mod empty_speculator;
-pub mod fixed_token_speculator;
-pub mod ngram_speculator;
-pub mod prompt_lookup_speculator;
-pub mod repeat_speculator;
-pub mod speculator;
-pub mod token_finder;
diff --git a/crates/backend-uzu/src/speculators/ngram_speculator.rs b/crates/backend-uzu/src/speculators/ngram_speculator.rs
deleted file mode 100644
index 0cea21c3c..000000000
--- a/crates/backend-uzu/src/speculators/ngram_speculator.rs
+++ /dev/null
@@ -1,272 +0,0 @@
-use std::{collections::HashMap, ops::Deref};
-
-use bytemuck::{Pod, Zeroable, cast_slice, from_bytes};
-use xxhash_rust::xxh3::xxh3_64;
-
-use crate::speculators::speculator::Speculator;
-
-#[cfg(not(target_endian = "little"))]
-compile_error!("Only little endian is supported");
-
-const MAX_CTX: usize = 15;
-
-#[inline]
-fn seqhash(seq: &[u32]) -> u64 {
- xxh3_64(cast_slice(seq))
-}
-
-fn apply_temperature(
- probs: &mut HashMap<u64, f32>,
- inv_tau: f32,
-) {
- let mut sum = 0.0f32;
- for v in probs.values_mut() {
- *v = v.powf(inv_tau);
- sum += *v;
- }
- if sum > 0.0 {
- let inv_sum = 1.0 / sum;
- for v in probs.values_mut() {
- *v *= inv_sum;
- }
- }
-}
-
-#[repr(C)]
-#[derive(Clone, Copy, Pod, Zeroable)]
-struct TaggedTableHeader {
- hashtable_size: u32,
- top_k: u32,
- ngram_n: u32,
- ngram_pad: u32,
-}
-
-const HEADER_SIZE: usize = size_of::<TaggedTableHeader>();
-
-struct TaggedTableLayout {
- hashtable_size: u32,
- top_k: u32,
- ngram_n: u32,
- ngram_pad: u32,
- tags_start: usize,
- keys_start: usize,
- values_start: usize,
-}
-
-impl TaggedTableLayout {
- fn parse(
- bytes: &[u8],
- offset: usize,
- ) -> (Self, usize) {
- let header = from_bytes::<TaggedTableHeader>(&bytes[offset..offset + HEADER_SIZE]);
-
- assert!(header.hashtable_size > 0, "hashtable_size must be > 0");
- assert!(header.top_k > 0, "top_k must be > 0");
- assert!(header.ngram_n >= 1, "ngram_n must be >= 1, got {}", header.ngram_n);
- assert!(
- (header.ngram_n - 1) as usize <= MAX_CTX,
- "ngram_n {} exceeds max supported {}",
- header.ngram_n,
- MAX_CTX + 1
- );
-
- let hs = header.hashtable_size as usize;
- let k = header.top_k as usize;
- let mut off = offset + HEADER_SIZE;
-
- let tags_start = off;
- off += 8 * hs;
-
- let keys_start = off;
- off += 4 * k * hs;
-
- let values_start = off;
- off += 4 * k * hs;
-
- // Counts: used during training, skipped at inference time
- off += 4 * hs;
-
- (
- Self {
- hashtable_size: header.hashtable_size,
- top_k: header.top_k,
- ngram_n: header.ngram_n,
- ngram_pad: header.ngram_pad,
- tags_start,
- keys_start,
- values_start,
- },
- off - offset,
- )
- }
-
- #[inline]
- fn read_tag(
- &self,
- bytes: &[u8],
- idx: usize,
- ) -> u64 {
- let off = self.tags_start + idx * 8;
- u64::from_le_bytes(bytes[off..off + 8].try_into().unwrap())
- }
-
- #[inline]
- fn lookup(
- &self,
- bytes: &[u8],
- prefix: &[u64],
- ) -> Option<HashMap<u64, f32>> {
- let ngram_ctx = (self.ngram_n - 1) as usize;
-
- let mut ctx_buf = [0u32; MAX_CTX];
- if ngram_ctx > 0 {
- let prefix_len = prefix.len();
- if prefix_len >= ngram_ctx {
- for i in 0..ngram_ctx {
- ctx_buf[i] = prefix[prefix_len - ngram_ctx + i] as u32;
- }
- } else {
- let pad_count = ngram_ctx - prefix_len;
- for i in 0..pad_count {
- ctx_buf[i] = self.ngram_pad;
- }
- for i in 0..prefix_len {
- ctx_buf[pad_count + i] = prefix[i] as u32;
- }
- }
- }
-
- let hash = seqhash(&ctx_buf[..ngram_ctx]);
- let idx = (hash % self.hashtable_size as u64) as usize;
- let tag = hash / self.hashtable_size as u64;
-
- if self.read_tag(bytes, idx) != tag {
- return None;
- }
-
- let k = self.top_k as usize;
- let row_byte_off = idx * k * 4;
- let keys: &[u32] = cast_slice(&bytes[self.keys_start + row_byte_off..self.keys_start + row_byte_off + k * 4]);
- let values: &[f32] =
- cast_slice(&bytes[self.values_start + row_byte_off..self.values_start + row_byte_off + k * 4]);
-
- let mut result = HashMap::with_capacity(k);
- for i in 0..k {
- result.insert(keys[i] as u64, values[i]);
- }
- Some(result)
- }
-}
-
-/// Multi-table n-gram speculator with tag-based collision detection and cascading backoff.
-///
-/// Binary format: `[max_order: u32, discount: f32]` followed by per-level tagged tables.
-/// Each level stores a saturated hash table with tag-based collision detection:
-/// `tag = xxh3_64(context) / table_size`, stored alongside each bucket. On tag mismatch,
-/// the lookup backs off to the next lower-order table (no interpolation — probabilities
-/// are pre-computed with KN discounting at training time).
-///
-/// Optional temperature scaling sharpens (τ<1) or flattens (τ>1) the draft distribution.
-pub struct NGramSpeculator<B: Deref<Target = [u8]> + Send + Sync> {
- bytes: B,
- tables: Vec<TaggedTableLayout>,
- inv_tau: f32,
-}
-
-impl<B: Deref<Target = [u8]> + Send + Sync> NGramSpeculator<B> {
- pub fn new(
- bytes: B,
- temperature: Option<f32>,
- ) -> Self {
- let mut off = 0;
-
- let max_order = u32::from_le_bytes(bytes[off..off + 4].try_into().unwrap());
- // Bytes [4..8] store the discount parameter, but exported probabilities are already discounted.
- off += 8;
-
- let mut tables: Vec<TaggedTableLayout> = Vec::with_capacity(max_order as usize);
- for _ in 0..max_order {
- let table_len = u64::from_le_bytes(bytes[off..off + 8].try_into().unwrap()) as usize;
- off += 8;
-
- let (layout, parsed_size) = TaggedTableLayout::parse(&bytes, off);
- assert_eq!(
- parsed_size,
- table_len,
- "table {}: parsed size {parsed_size} != declared size {table_len}",
- tables.len()
- );
- if let Some(prev) = tables.last() {
- assert!(
- layout.ngram_n > prev.ngram_n,
- "tables must be in ascending ngram_n order: got {} after {}",
- layout.ngram_n,
- prev.ngram_n
- );
- }
- tables.push(layout);
- off += table_len;
- }
-
- assert_eq!(off, bytes.len(), "speculator file size mismatch: expected {off} bytes, got {}", bytes.len());
-
- let inv_tau = match temperature {
- Some(tau) => {
- assert!(tau > 0.0, "temperature must be positive, got {tau}");
- if tau != 1.0 {
- 1.0 / tau
- } else {
- 0.0
- }
- },
- None => 0.0,
- };
-
- Self {
- bytes,
- tables,
- inv_tau,
- }
- }
-
- fn lookup(
- &self,
- prefix: &[u64],
- ) -> Option<HashMap<u64, f32>> {
- for table in self.tables.iter().rev() {
- if let Some(result) = table.lookup(&self.bytes, prefix) {
- return Some(result);
- }
- }
- None
- }
-}
-
-impl NGramSpeculator<memmap2::Mmap> {
- pub fn load_with_temperature(
- path: &str,
- temperature: Option<f32>,
- ) -> Self {
- let file = std::fs::File::open(path).unwrap_or_else(|e| panic!("failed to open speculator file '{path}': {e}"));
- let mmap = unsafe { memmap2::MmapOptions::default().map(&file) }
- .unwrap_or_else(|e| panic!("failed to mmap speculator file '{path}': {e}"));
- Self::new(mmap, temperature)
- }
-}
-
-impl<B: Deref<Target = [u8]> + Send + Sync> Speculator for NGramSpeculator<B> {
- fn speculate(
- &self,
- prefix: &[u64],
- ) -> HashMap<u64, f32> {
- match self.lookup(prefix) {
- Some(mut probs) => {
- if self.inv_tau > 0.0 {
- apply_temperature(&mut probs, self.inv_tau);
- }
- probs
- },
- None => HashMap::new(),
- }
- }
-}
diff --git a/crates/backend-uzu/src/speculators/prompt_lookup_speculator.rs b/crates/backend-uzu/src/speculators/prompt_lookup_speculator.rs
deleted file mode 100644
index 3e8977f9e..000000000
--- a/crates/backend-uzu/src/speculators/prompt_lookup_speculator.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-use std::collections::HashMap;
-
-use crate::speculators::{speculator::Speculator, token_finder::TokenFinder};
-
-pub struct PromptLookupSpeculator {
- max_ngram_size: usize,
-}
-
-impl Default for PromptLookupSpeculator {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl PromptLookupSpeculator {
- pub fn new() -> Self {
- Self {
- max_ngram_size: 3,
- }
- }
-
- pub fn new_with_params(max_ngram_size: usize) -> Self {
- Self {
- max_ngram_size,
- }
- }
-}
-
-impl Speculator for PromptLookupSpeculator {
- fn speculate(
- &self,
- prefix: &[u64],
- ) -> HashMap<u64, f32> {
- let mut hm = HashMap::new();
-
- if let Some(tok) = TokenFinder::find_candidate_pred_token(prefix, self.max_ngram_size) {
- hm.insert(tok, 1.0);
- }
-
- hm
- }
-}
-
-#[cfg(test)]
-#[path = "../../tests/unit/speculators/prompt_lookup_speculator.rs"]
-mod tests;
diff --git a/crates/backend-uzu/src/speculators/repeat_speculator.rs b/crates/backend-uzu/src/speculators/repeat_speculator.rs
deleted file mode 100644
index 4ade882bc..000000000
--- a/crates/backend-uzu/src/speculators/repeat_speculator.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-use std::collections::HashMap;
-
-use crate::speculators::speculator::Speculator;
-
-pub struct RepeatSpeculator;
-
-impl Speculator for RepeatSpeculator {
- fn speculate(
- &self,
- prefix: &[u64],
- ) -> HashMap<u64, f32> {
- let mut hm = HashMap::new();
-
- for (pos, token) in prefix.iter().copied().enumerate() {
- *hm.entry(token).or_insert(0.0) += f32::sqrt((1 + pos) as f32);
- }
-
- let sum = hm.values().sum::<f32>();
-
- hm.into_iter().map(|(pos, weight)| (pos, weight / sum)).collect()
- }
-}
diff --git a/crates/backend-uzu/src/speculators/speculator.rs b/crates/backend-uzu/src/speculators/speculator.rs
deleted file mode 100644
index cdba2c610..000000000
--- a/crates/backend-uzu/src/speculators/speculator.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-use std::collections::HashMap;
-
-pub trait Speculator: Send + Sync {
- fn speculate(
- &self,
- prefix: &[u64],
- ) -> HashMap<u64, f32>;
-}
diff --git a/crates/backend-uzu/src/speculators/token_finder.rs b/crates/backend-uzu/src/speculators/token_finder.rs
deleted file mode 100644
index f01b4d5dc..000000000
--- a/crates/backend-uzu/src/speculators/token_finder.rs
+++ /dev/null
@@ -1,38 +0,0 @@
-pub struct TokenFinder;
-
-impl TokenFinder {
- pub fn find_candidate_pred_token(
- sequence: &[u64],
- max_ngram_size: usize,
- ) -> Option<u64> {
- let input_length = sequence.len();
-
- for ngram_size in (1..=max_ngram_size).rev() {
- if ngram_size >= input_length {
- continue;
- }
-
- let ngram_start = input_length - ngram_size;
- let ngram = &sequence[ngram_start..input_length];
-
- let max_start_index = input_length - ngram_size;
- for window_start in 0..max_start_index {
- let window = &sequence[window_start..(window_start + ngram_size)];
-
- if window == ngram {
- let start_idx = window_start + ngram_size;
-
- if start_idx < (input_length - ngram_size) {
- return Some(sequence[start_idx]);
- }
- }
- }
- }
-
- None
- }
-}
-
-#[cfg(test)]
-#[path = "../../tests/unit/speculators/token_finder_speculator.rs"]
-mod tests;
diff --git a/crates/backend-uzu/src/trie.rs b/crates/backend-uzu/src/trie.rs
index 490a5373e..f56c26b8e 100644
--- a/crates/backend-uzu/src/trie.rs
+++ b/crates/backend-uzu/src/trie.rs
@@ -2,11 +2,7 @@
use itertools::Itertools;
use thiserror::Error;
-use crate::{
- backends::common::gpu_types::trie::TrieNode as GpuTrieNode,
- encodable_block::sampling::{PRng, speculator_sample},
- speculators::speculator::Speculator,
-};
+use crate::{backends::common::gpu_types::trie::TrieNode as GpuTrieNode, encodable_block::sampling::PRng};
#[cfg(grammar)]
use crate::{
data_type::DataType,
@@ -26,18 +22,6 @@ pub enum TrieAcceptError {
Grammar(#[from] GrammarError),
}
-pub struct TrieCreationConfig {
- pub width: usize,
-}
-
-impl Default for TrieCreationConfig {
- fn default() -> Self {
- TrieCreationConfig {
- width: 4,
- }
- }
-}
-
#[derive(Debug)]
pub struct TrieNode {
token: u64,
@@ -88,108 +72,11 @@ impl TrieNode {
self.next.iter().find(|n| n.token == token)
}
- pub fn get_mut(
- &mut self,
- token: u64,
- ) -> Option<&mut TrieNode> {
- self.next.iter_mut().find(|n| n.token == token)
- }
-
#[cfg(test)]
pub fn token(&self) -> u64 {
self.token
}
- pub fn seed(&self) -> u64 {
- self.seed
- }
-
- pub fn from_speculator(
- prefix: &[u64],
- prng: &PRng,
- #[cfg(grammar)] mut grammar: Option<&mut Grammar>,
- speculator: &dyn Speculator,
- vocab_size: usize,
- creation_config: &TrieCreationConfig,
- max_length: usize,
- max_height: Option<usize>,
- ) -> Self {
- assert!(max_length >= 1, "can't have zero sized trie");
- assert!(max_height.is_none_or(|max_height| max_height > 0), "max height cannot be 0");
- assert!(!prefix.is_empty(), "need seed node");
-
- let prefix_length = prefix.len();
- let mut speculated_suffix = prefix.to_vec();
-
- let mut length = 1;
- let mut height = 0;
- let mut root = Self::new(*prefix.last().unwrap(), prng.derive((prefix_length - 1) as u64));
-
- let mut cur_node = &mut root;
- let mut cur_node_width = 0;
- let mut cur_node_speculator_weights = speculator.speculate(&speculated_suffix);
-
- let mut next_node = None;
-
- while length < max_length && max_height.is_none_or(|max_height| height < max_height) {
- // Guuumbel speculator trick: both speculator and llm sample via gumbel max trick using the same noise for increased acceptance rate
- if let Some(next_speculated_token) =
- speculator_sample(cur_node.seed(), vocab_size, &cur_node_speculator_weights)
- {
- // Add speculated token to the trie
- #[cfg(grammar)]
- if let Some(grammar) = grammar.as_mut() {
- if grammar.accept_token(next_speculated_token).is_err() {
- cur_node_speculator_weights.remove(&next_speculated_token);
- continue;
- }
- grammar.rollback(1);
- }
-
- let leaf_node = Self::new(next_speculated_token, prng.derive((prefix_length + height) as u64));
- cur_node.add(leaf_node).unwrap();
-
- // If this is the first node we sampled (most likely to be selected after gumbel noise application) - set it as the next one
- if next_node.is_none() {
- next_node = Some(next_speculated_token);
- }
-
- // Remove the token so that the next iteration samples the second most likely after gumbel noise application token
- cur_node_speculator_weights.remove(&next_speculated_token);
-
- length += 1;
- cur_node_width += 1;
- if cur_node_width >= creation_config.width {
- cur_node_speculator_weights.clear();
- }
- } else if let Some(next_node_token) = next_node.take() {
- // Out of speculated tokens for this node, move onto the likeliest next node
- speculated_suffix.push(next_node_token);
- #[cfg(grammar)]
- if let Some(grammar) = grammar.as_mut()
- && grammar.accept_token(next_node_token).is_err()
- {
- break;
- }
- height += 1;
- cur_node = cur_node.get_mut(next_node_token).unwrap();
- cur_node_width = 0;
- cur_node_speculator_weights = speculator.speculate(&speculated_suffix);
- continue;
- } else {
- // Dead end, exit
- break;
- };
- }
-
- #[cfg(grammar)]
- if let Some(grammar) = grammar {
- grammar.rollback(height);
- }
-
- root
- }
-
pub fn flat(
prefix_length: usize,
tokens: &[u64],
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
index 689b0522f..a4065ed74 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
@@ -623,3 +623,81 @@ fn a8w_mxu_output_bias_parity_bf16(
0.8,
);
}
+
+fn run_widened_f32<B: Backend>(
+ context: &B::Context,
+ input: &QuantInput<bf16>,
+) -> Vec<f32> {
+ use crate::{
+ backends::common::kernel::matmul::{MatmulA, MatmulB},
+ data_type::DataType,
+ tests::helpers::alloc_allocation,
+ };
+ let buffers = QuantBuffers::<B, bf16>::allocate(context, input);
+ let mut y = alloc_allocation::<B, f32>(context, (input.m as usize) * (input.n as usize));
+ let mut matmul =
+ <<B as Backend>::Kernels as Kernels>::MatmulKernel::new(context, DataType::BF16, DataType::BF16, DataType::F32)
+ .expect("MatmulKernel widened");
+ let b: MatmulB<'_, B> = match input.quant_method {
+ QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {
+ b: &buffers.w,
+ scales: &buffers.scales,
+ biases: buffers.bias.as_ref().expect("bias buffer"),
+ mode: input.mode,
+ group_size: input.group_size,
+ },
+ QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {
+ b: &buffers.w,
+ scales: &buffers.scales,
+ zero_points: buffers.zp.as_ref().expect("zp buffer"),
+ mode: input.mode,
+ group_size: input.group_size,
+ },
+ QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {
+ b: &buffers.w,
+ scales: &buffers.scales,
+ mode: input.mode,
+ group_size: input.group_size,
+ },
+ };
+ let mut encoder = Encoder::<B>::new(context).expect("encoder");
+ matmul
+ .encode(
+ crate::backends::common::kernel::matmul::MatmulArguments {
+ a: MatmulA::FullPrecision {
+ values: &buffers.x,
+ offset: 0,
+ },
+ b,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut y,
+ d_transform: MatmulDOps::none(),
+ gather_indices: None,
+ m: input.m,
+ n: input.n,
+ k: input.k,
+ },
+ &mut encoder,
+ )
+ .expect("widened encode failed");
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ allocation_to_vec::<B, f32>(&y)
+}
+
+#[rstest]
+#[test_attr(uzu_test)]
+#[case::gemv_m1(1)]
+#[case::gemv_m4(4)]
+#[case::gemm_m31(31)]
+fn parity_widened_f32_output(#[case] m: usize) {
+ // Mirrors the dflash draft-chain readout: quantized LM head (4-bit ZP,
+ // group 32), BF16 input, widened F32 output.
+ let (k, n, gs) = (5120usize, 32768usize, 32u32);
+ let context = MetalContext::new().expect("Metal context");
+ let input = QuantInput::<bf16>::new(m, k, n, gs, 4, QuantizationMethod::ScaleZeroPoint, 0);
+ let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");
+ let reference = run_widened_f32::<Cpu>(&cpu_context, &input);
+ let actual = run_widened_f32::<Metal>(&context, &input);
+ assert_parity::<f32>(&format!("widened f32 m={m}"), &reference, &actual, 0.05, 0.4);
+}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
index 773368b25..9cc098256 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
@@ -4,7 +4,7 @@ use test_runner::for_each_non_cpu_backend;
use crate::{
backends::{
- common::{Backend, Encoder, Kernels, kernel::WeaverTopChildrenKernel},
+ common::{Backend, Encoder, Kernels, gpu_types::weaver::MetadataIdx, kernel::WeaverTopChildrenKernel},
cpu::Cpu,
},
tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context},
@@ -12,6 +12,9 @@ use crate::{
const CANDIDATES: usize = 512;
const CHILDREN: usize = 8;
+const VOCAB_SIZE: u32 = 131_072;
+const DEPTHS: [u32; 3] = [0, 1, 2];
+const DEPTH_SEEDS: [u64; 3] = [0x9E3779B97F4A7C15, 0xD1B54A32D192ED03, 0x2545F4914F6CDD1D];
fn top_children<B: Backend>(
residual: &[bf16],
@@ -22,10 +25,17 @@ fn top_children<B: Backend>(
assert_eq!(residual.len(), rows * CANDIDATES);
assert_eq!(candidate_logits.len(), rows * CANDIDATES);
assert_eq!(ids.len(), rows * CANDIDATES);
+ assert!(rows <= DEPTHS.len());
let context = create_context::<B>();
let residual = alloc_allocation_with_data::<B, bf16>(&context, residual);
let candidate_logits = alloc_allocation_with_data::<B, f32>(&context, candidate_logits);
let ids = alloc_allocation_with_data::<B, u32>(&context, ids);
+ let depth_seeds = alloc_allocation_with_data::<B, u64>(&context, &DEPTH_SEEDS);
+ let mut metadata_values = vec![0u32; rows * MetadataIdx::COUNT];
+ for row in 0..rows {
+ metadata_values[MetadataIdx::Depth as usize * rows + row] = DEPTHS[row];
+ }
+ let node_metadata = alloc_allocation_with_data::<B, u32>(&context, &metadata_values);
let mut output_token_ids = alloc_allocation::<B, u32>(&context, rows * CHILDREN);
let mut output_model_logprobs = alloc_allocation::<B, f32>(&context, rows * CHILDREN);
let kernel = <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(&context).unwrap();
@@ -34,11 +44,14 @@ fn top_children<B: Backend>(
&residual,
&candidate_logits,
&ids,
+ &depth_seeds,
+ &node_metadata,
&mut output_token_ids,
&mut output_model_logprobs,
rows as u32,
CANDIDATES as u32,
CHILDREN as u32,
+ VOCAB_SIZE,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
diff --git a/crates/backend-uzu/tests/unit/speculators/chain_acceptance.rs b/crates/backend-uzu/tests/unit/speculators/chain_acceptance.rs
deleted file mode 100644
index 0c7a08ba8..000000000
--- a/crates/backend-uzu/tests/unit/speculators/chain_acceptance.rs
+++ /dev/null
@@ -1,122 +0,0 @@
-use proc_macros::uzu_test;
-
-use crate::speculators::chain_acceptance::{AcceptedProposal, ChainProposal};
-
-const NO_EOS: &[u64] = &[99999];
-
-fn make_proposal() -> ChainProposal {
- ChainProposal {
- token_ids: Box::new([10, 11, 12, 13]),
- token_positions: Box::new([5, 6, 7, 8]),
- }
-}
-
-fn accept(
- sampled_token_ids: &[u64],
- eos_token_ids: &[u64],
-) -> AcceptedProposal {
- make_proposal().accept(sampled_token_ids, 16, eos_token_ids)
-}
-
-#[uzu_test]
-fn test_full_chain_accepted_with_bonus() {
- let accepted = accept(&[11, 12, 13, 14], NO_EOS);
-
- assert_eq!(accepted.committed_token_ids, vec![11, 12, 13, 14]);
- assert_eq!(accepted.committed_token_positions, vec![6, 7, 8, 9]);
- assert_eq!(accepted.num_verified_nodes, 4);
- assert_eq!(accepted.next_context_nodes(), 0..4);
-}
-
-#[uzu_test]
-fn test_partial_chain_accepted() {
- let accepted = accept(&[11, 99, 13, 14], NO_EOS);
-
- assert_eq!(accepted.committed_token_ids, vec![11, 99]);
- assert_eq!(accepted.committed_token_positions, vec![6, 7]);
- assert_eq!(accepted.num_verified_nodes, 2);
-}
-
-#[uzu_test]
-fn test_all_drafts_rejected_emits_bonus_only() {
- let accepted = accept(&[99, 98, 97, 96], NO_EOS);
-
- assert_eq!(accepted.committed_token_ids, vec![99]);
- assert_eq!(accepted.committed_token_positions, vec![6]);
- assert_eq!(accepted.num_verified_nodes, 1);
-}
-
-#[uzu_test]
-fn test_eos_in_drafts_truncates_and_suppresses_bonus() {
- let accepted = accept(&[11, 12, 13, 14], &[12]);
-
- assert_eq!(accepted.committed_token_ids, vec![11, 12]);
- assert!(accepted.has_eos(&[12]));
- // EOS truncates the committed output only; all four nodes stay verified.
- assert_eq!(accepted.num_verified_nodes, 4);
-}
-
-#[uzu_test]
-fn test_eos_at_first_accepted_draft() {
- let accepted = accept(&[11, 12, 13, 14], &[11]);
-
- assert_eq!(accepted.committed_token_ids, vec![11]);
- assert!(accepted.has_eos(&[11]));
- assert_eq!(accepted.num_verified_nodes, 4);
-}
-
-#[uzu_test]
-fn test_eos_as_bonus_token() {
- let accepted = accept(&[11, 99, 13, 14], &[99]);
-
- assert_eq!(accepted.committed_token_ids, vec![11, 99]);
- assert!(accepted.has_eos(&[99]));
- assert_eq!(accepted.num_verified_nodes, 2);
-}
-
-#[uzu_test]
-fn test_remaining_length_caps_emission() {
- let accepted = make_proposal().accept(&[11, 12, 13, 14], 2, NO_EOS);
-
- assert_eq!(accepted.committed_token_ids, vec![11, 12]);
- // The output budget caps committed tokens only, not verified nodes.
- assert_eq!(accepted.num_verified_nodes, 4);
-}
-
-#[uzu_test]
-fn test_zero_remaining_length_commits_nothing() {
- let accepted = make_proposal().accept(&[11, 99, 13, 14], 0, NO_EOS);
-
- assert!(accepted.committed_token_ids.is_empty());
- assert_eq!(accepted.num_verified_nodes, 2);
- assert_eq!(accepted.last_token_id(), None);
- assert_eq!(accepted.last_covered_position(), None);
-}
-
-#[uzu_test]
-fn test_root_only_proposal_emits_bonus() {
- let proposal = ChainProposal {
- token_ids: Box::new([10]),
- token_positions: Box::new([5]),
- };
- let accepted = proposal.accept(&[42], 16, NO_EOS);
-
- assert_eq!(accepted.committed_token_ids, vec![42]);
- assert_eq!(accepted.committed_token_positions, vec![6]);
- assert_eq!(accepted.num_verified_nodes, 1);
-}
-
-#[uzu_test]
-fn test_last_token_id_and_index() {
- let accepted = accept(&[11, 99, 13, 14], NO_EOS);
-
- assert_eq!(accepted.last_token_id(), Some(99));
- assert_eq!(accepted.last_covered_position(), Some(6));
-}
-
-#[uzu_test]
-fn test_has_eos_negative() {
- let accepted = accept(&[11, 12, 13, 14], NO_EOS);
-
- assert!(!accepted.has_eos(&[12345]));
-}
diff --git a/crates/backend-uzu/tests/unit/speculators/fixed_token_speculator.rs b/crates/backend-uzu/tests/unit/speculators/fixed_token_speculator.rs
deleted file mode 100644
index f2b3d2025..000000000
--- a/crates/backend-uzu/tests/unit/speculators/fixed_token_speculator.rs
+++ /dev/null
@@ -1,36 +0,0 @@
-use std::collections::HashMap;
-
-use proc_macros::uzu_test;
-
-use crate::speculators::{fixed_token_speculator::FixedTokensSpeculator, speculator::Speculator};
-
-#[uzu_test]
-fn test_fixed_token_speculator_single() {
- let speculator = FixedTokensSpeculator::new(vec![vec![0, 1, 2, 3, 4]]);
-
- assert_eq!(speculator.speculate(&[]), HashMap::from([(0, 1.0)]));
- assert_eq!(speculator.speculate(&[1337]), HashMap::from([(0, 1.0)]));
-
- assert_eq!(speculator.speculate(&[0]), HashMap::from([(1, 1.0)]));
-
- assert_eq!(speculator.speculate(&[0, 1]), HashMap::from([(2, 1.0)]));
- assert_eq!(speculator.speculate(&[1]), HashMap::from([(2, 1.0)]));
-
- assert_eq!(speculator.speculate(&[4]), HashMap::from([(0, 1.0)]));
-}
-
-#[uzu_test]
-fn test_fixed_token_speculator_multi() {
- let speculator =
- FixedTokensSpeculator::new(vec![vec![0, 1, 2, 3, 4], vec![0, 1, 2, 4, 8], vec![0, 10, 20, 30, 40]]);
-
- assert_eq!(speculator.speculate(&[]), HashMap::from([(0, 1.0)]));
- assert_eq!(speculator.speculate(&[1337]), HashMap::from([(0, 1.0)]));
-
- assert_eq!(speculator.speculate(&[0]), HashMap::from([(1, 2.0 / 3.0), (10, 1.0 / 3.0)]));
-
- assert_eq!(speculator.speculate(&[0, 1]), HashMap::from([(0, 1.0 / 3.0), (2, 2.0 / 3.0)]));
- assert_eq!(speculator.speculate(&[1]), HashMap::from([(0, 1.0 / 3.0), (2, 2.0 / 3.0)]));
-
- assert_eq!(speculator.speculate(&[4]), HashMap::from([(8, 1.0 / 3.0), (0, 2.0 / 3.0)]));
-}
diff --git a/crates/backend-uzu/tests/unit/speculators/prompt_lookup_speculator.rs b/crates/backend-uzu/tests/unit/speculators/prompt_lookup_speculator.rs
deleted file mode 100644
index 28227a2d9..000000000
--- a/crates/backend-uzu/tests/unit/speculators/prompt_lookup_speculator.rs
+++ /dev/null
@@ -1,63 +0,0 @@
-use std::collections::HashMap;
-
-use proc_macros::uzu_test;
-use test_runner::path::get_test_model_path;
-use tokenizers::Tokenizer;
-
-use crate::speculators::{prompt_lookup_speculator::PromptLookupSpeculator, speculator::Speculator};
-
-fn load_tokenizer() -> Tokenizer {
- let tokenizer_path = get_test_model_path().join("tokenizer.json");
- match Tokenizer::from_file(tokenizer_path) {
- Ok(tokenizer) => tokenizer,
- Err(e) => {
- panic!("Failed to load tokenizer: {}", e);
- },
- }
-}
-
-fn tokenize_text(
- tokenizer: &Tokenizer,
- text: &str,
-) -> Vec<u64> {
- let encoding = tokenizer.encode(text, false).expect("Failed to encode text");
- encoding.get_ids().iter().map(|&id| id as u64).collect()
-}
-
-#[uzu_test]
-fn test_empty_input() {
- let speculator = PromptLookupSpeculator::new();
- let prefix: Vec<u64> = vec![];
-
- let proposals = speculator.speculate(&prefix);
-
- assert_eq!(proposals.len(), 0);
-}
-
-#[uzu_test]
-fn test_no_pattern_found() {
- let speculator = PromptLookupSpeculator::new();
- let prefix = vec![1, 2, 3, 4, 5, 6, 7, 8, 9];
-
- let proposals = speculator.speculate(&prefix);
-
- assert_eq!(proposals.len(), 0);
-}
-
-#[uzu_test]
-fn test_with_real_text() {
- let tokenizer = load_tokenizer();
- let speculator = PromptLookupSpeculator::new_with_params(3);
-
- let text = "The quick brown fox jumps over the lazy dog. The quick brown fox";
-
- let mut tokens = tokenize_text(&tokenizer, text);
-
- for expected in tokenize_text(&tokenizer, " jumps over the lazy dog") {
- let proposals = speculator.speculate(&tokens);
-
- assert_eq!(proposals, HashMap::from([(expected, 1.0)]));
-
- tokens.push(expected);
- }
-}
diff --git a/crates/backend-uzu/tests/unit/speculators/token_finder_speculator.rs b/crates/backend-uzu/tests/unit/speculators/token_finder_speculator.rs
deleted file mode 100644
index 4db3d4d41..000000000
--- a/crates/backend-uzu/tests/unit/speculators/token_finder_speculator.rs
+++ /dev/null
@@ -1,103 +0,0 @@
-use proc_macros::uzu_test;
-
-use crate::speculators::token_finder::TokenFinder;
-
-#[uzu_test]
-fn test_find_candidate_pred_token_empty_sequence() {
- let sequence: Vec<u64> = vec![];
-
- let result = TokenFinder::find_candidate_pred_token(&sequence, 3);
-
- assert!(result.is_none());
-}
-
-#[uzu_test]
-fn test_find_candidate_pred_token_sequence_too_short() {
- let sequence: Vec<u64> = vec![1, 2];
-
- let result = TokenFinder::find_candidate_pred_token(&sequence, 3);
-
- assert!(result.is_none());
-}
-
-#[uzu_test]
-fn test_find_candidate_pred_token_exact_pattern_match() {
- let sequence = vec![1, 2, 3, 4, 5, 1, 2, 3];
-
- let result = TokenFinder::find_candidate_pred_token(&sequence, 3);
-
- assert_eq!(result, Some(4));
-}
-
-#[uzu_test]
-fn test_find_candidate_pred_token_valid_match() {
- let mut sequence = vec![1, 2, 3, 4, 5, 1, 2, 3];
-
- let result1 = TokenFinder::find_candidate_pred_token(&sequence, 3);
-
- assert_eq!(result1, Some(4));
-
- sequence.push(4);
-
- let result2 = TokenFinder::find_candidate_pred_token(&sequence, 3);
-
- assert_eq!(result2, Some(5));
-}
-
-#[uzu_test]
-fn test_find_candidate_pred_token_multiple_matches() {
- let mut sequence = vec![1, 2, 3, 4, 5, 1, 2, 3, 6, 7, 1, 2, 3];
-
- let result1 = TokenFinder::find_candidate_pred_token(&sequence, 3);
-
- assert_eq!(result1, Some(4));
- assert_ne!(result1, Some(6));
-
- sequence.push(4);
-
- let result2 = TokenFinder::find_candidate_pred_token(&sequence, 3);
-
- assert_eq!(result2, Some(5));
- assert_ne!(result2, Some(7));
-}
-
-#[uzu_test]
-fn test_find_candidate_pred_token_different_ngram_sizes() {
- let mut sequence = vec![1, 2, 3, 4, 5, 1, 2, 3, 8, 9, 1, 2, 3, 4, 5];
-
- let result1_3 = TokenFinder::find_candidate_pred_token(&sequence, 3);
- let result1_2 = TokenFinder::find_candidate_pred_token(&sequence, 2);
- let result1_1 = TokenFinder::find_candidate_pred_token(&sequence, 1);
-
- assert_eq!(result1_3, Some(1));
- assert_eq!(result1_2, Some(1));
- assert_eq!(result1_1, Some(1));
-
- sequence.push(1);
-
- let result2_3 = TokenFinder::find_candidate_pred_token(&sequence, 3);
- let result2_2 = TokenFinder::find_candidate_pred_token(&sequence, 2);
- let result2_1 = TokenFinder::find_candidate_pred_token(&sequence, 1);
-
- assert_eq!(result2_3, Some(2));
- assert_eq!(result2_2, Some(2));
- assert_eq!(result2_1, Some(2));
-}
-
-#[uzu_test]
-fn test_find_candidate_pred_token_no_match() {
- let sequence = vec![1, 2, 3, 4, 5, 6, 7, 8, 9];
-
- let result = TokenFinder::find_candidate_pred_token(&sequence, 3);
-
- assert!(result.is_none());
-}
-
-#[uzu_test]
-fn test_find_candidate_pred_token_max_ngram_higher_than_input() {
- let sequence = vec![1, 2, 3, 4, 5];
-
- let result = TokenFinder::find_candidate_pred_token(&sequence, 8);
-
- assert!(result.is_none());
-}
diff --git a/crates/backend-uzu/tests/unit/trie_test.rs b/crates/backend-uzu/tests/unit/trie_test.rs
index 4b61666c3..335cfde35 100644
--- a/crates/backend-uzu/tests/unit/trie_test.rs
+++ b/crates/backend-uzu/tests/unit/trie_test.rs
@@ -1,35 +1,6 @@
-use std::collections::HashMap;
-
use proc_macros::uzu_test;
-use crate::{
- encodable_block::sampling::PRng,
- speculators::{empty_speculator::EmptySpeculator, speculator::Speculator},
- trie::{TrieCreationConfig, TrieNode},
-};
-
-pub struct StaticSpeculator {
- responses: HashMap<Vec<u64>, HashMap<u64, f32>>,
- default_response: Option<HashMap<u64, f32>>,
-}
-
-impl StaticSpeculator {
- pub fn new(responses: HashMap<Vec<u64>, HashMap<u64, f32>>) -> Self {
- Self {
- responses,
- default_response: None,
- }
- }
-}
-
-impl Speculator for StaticSpeculator {
- fn speculate(
- &self,
- prefix: &[u64],
- ) -> HashMap<u64, f32> {
- self.responses.get(prefix).cloned().or_else(|| self.default_response.clone()).unwrap_or_default()
- }
-}
+use crate::{encodable_block::sampling::PRng, trie::TrieNode};
fn verify_sprout(
trie_root: &TrieNode,
@@ -54,29 +25,6 @@ fn test_trie_manual_sprout() {
verify_sprout(&trie_root, 0);
}
-#[uzu_test]
-fn test_trie_from_speculator_sprout() {
- let rng = PRng::new(42);
-
- let speculator = EmptySpeculator;
-
- let trie_root = TrieNode::from_speculator(
- &[0],
- &rng,
- #[cfg(grammar)]
- None,
- &speculator,
- 1024,
- &TrieCreationConfig {
- width: 5,
- },
- 10,
- None,
- );
-
- verify_sprout(&trie_root, rng.derive(0));
-}
-
fn verify_stick(
trie_root: &TrieNode,
rng: &PRng,
@@ -101,7 +49,7 @@ fn verify_stick(
for i in 1..10 {
cur_node = cur_node.get(i).unwrap();
assert_eq!(cur_node.token(), i);
- assert_eq!(cur_node.seed(), rng.derive(i));
+ assert_eq!(cur_node.seed, rng.derive(i));
let position = flat_trie.index(cur_node).unwrap();
assert_eq!(token_ids[position], cur_node.token());
@@ -113,40 +61,15 @@ fn verify_stick(
#[uzu_test]
fn test_trie_manual_stick() {
let rng = PRng::new(0);
- let mut trie_root = TrieNode::new(0, rng.derive(0));
- let mut trie_leaf = &mut trie_root;
- for i in 1..10u64 {
- trie_leaf.add(TrieNode::new(i, rng.derive(i))).unwrap();
- trie_leaf = trie_leaf.get_mut(i).unwrap();
+ let mut trie_stick = TrieNode::new(9, rng.derive(9));
+ for i in (1..9u64).rev() {
+ let mut trie_parent = TrieNode::new(i, rng.derive(i));
+ trie_parent.add(trie_stick).unwrap();
+ trie_stick = trie_parent;
}
-
- verify_stick(&trie_root, &rng);
-}
-
-#[uzu_test]
-fn test_trie_from_speculator_stick() {
- let rng = PRng::new(42);
-
- let mut hs = HashMap::new();
- for i in 1..10 {
- hs.insert((0..i).collect(), vec![(i, 0.988), (888, 0.001), (999, 0.001)].into_iter().collect());
- }
- let speculator = StaticSpeculator::new(hs);
-
- let trie_root = TrieNode::from_speculator(
- &[0],
- &rng,
- #[cfg(grammar)]
- None,
- &speculator,
- 1024,
- &TrieCreationConfig {
- width: 1,
- },
- 10,
- None,
- );
+ let mut trie_root = TrieNode::new(0, rng.derive(0));
+ trie_root.add(trie_stick).unwrap();
verify_stick(&trie_root, &rng);
}
@@ -173,7 +96,7 @@ fn verify_bush(
for leaf_token in [1, 2, 3] {
let leaf = trie_root.get(leaf_token).unwrap();
assert_eq!(leaf.token(), leaf_token);
- assert_eq!(leaf.seed(), rng.derive(1));
+ assert_eq!(leaf.seed, rng.derive(1));
let position = flat_trie.index(leaf).unwrap();
assert_eq!(token_ids[position], leaf.token());
@@ -197,31 +120,6 @@ fn test_trie_manual_bush() {
verify_bush(&trie_root, &rng);
}
-#[uzu_test]
-fn test_trie_from_speculator_bush() {
- let rng = PRng::new(0);
-
- let mut hs = HashMap::new();
- hs.insert(vec![0], vec![(1, 0.33), (2, 0.33), (3, 0.33), (888, 0.005), (999, 0.005)].into_iter().collect());
- let speculator = StaticSpeculator::new(hs);
-
- let trie_root = TrieNode::from_speculator(
- &[0],
- &rng,
- #[cfg(grammar)]
- None,
- &speculator,
- 1024,
- &TrieCreationConfig {
- width: 3,
- },
- 10,
- None,
- );
-
- verify_bush(&trie_root, &rng);
-}
-
fn verify_tree(
trie_root: &TrieNode,
rng: &PRng,
@@ -267,15 +165,15 @@ fn test_trie_manual_tree() {
assert!(trie_root.add(TrieNode::new(1, rng.derive(1))).is_err());
assert!(trie_root.add(TrieNode::new(1, 10)).is_err());
- assert!(trie_root.add(TrieNode::new(2, rng.derive(1))).is_ok());
- assert!(trie_root.add(TrieNode::new(3, rng.derive(1))).is_ok());
-
- let mid_b = trie_root.get_mut(2).unwrap();
+ let mut mid_b = TrieNode::new(2, rng.derive(1));
assert!(mid_b.add(TrieNode::new(10, rng.derive(2))).is_ok());
- let mid_c = trie_root.get_mut(3).unwrap();
+ let mut mid_c = TrieNode::new(3, rng.derive(1));
assert!(mid_c.add(TrieNode::new(20, rng.derive(2))).is_ok());
assert!(mid_c.add(TrieNode::new(21, rng.derive(2))).is_ok());
+ assert!(trie_root.add(mid_b).is_ok());
+ assert!(trie_root.add(mid_c).is_ok());
+
verify_tree(&trie_root, &rng)
}
diff --git a/crates/shoji/src/types/session/chat/config.rs b/crates/shoji/src/types/session/chat/config.rs
index 884ab2063..169b3b3a8 100644
--- a/crates/shoji/src/types/session/chat/config.rs
+++ b/crates/shoji/src/types/session/chat/config.rs
@@ -1,16 +1,12 @@
use serde::{Deserialize, Serialize};
-use crate::types::{
- basic::{ContextLength, SamplingSeed},
- session::chat::ChatSpeculationPreset,
-};
+use crate::types::basic::{ContextLength, SamplingSeed};
#[bindings::export(Structure(Class))]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
pub struct ChatConfig {
pub context_length: ContextLength,
pub sampling_seed: SamplingSeed,
- pub speculation_preset: Option<ChatSpeculationPreset>,
}
#[bindings::export(Implementation)]
@@ -44,15 +40,4 @@ impl ChatConfig {
..self.clone()
}
}
-
- #[bindings::export(Method)]
- pub fn with_speculation_preset(
- &self,
- speculation_preset: Option<ChatSpeculationPreset>,
- ) -> Self {
- Self {
- speculation_preset,
- ..self.clone()
- }
- }
}
diff --git a/crates/shoji/src/types/session/chat/mod.rs b/crates/shoji/src/types/session/chat/mod.rs
index 9e4cd46b4..d86c0cd64 100644
--- a/crates/shoji/src/types/session/chat/mod.rs
+++ b/crates/shoji/src/types/session/chat/mod.rs
@@ -4,7 +4,6 @@ mod content_block;
mod message;
mod reply;
mod role;
-mod speculation_preset;
pub use capabilities::ChatModelCapabilities;
pub use config::ChatConfig;
@@ -14,4 +13,3 @@ pub use reply::{
ChatReply, ChatReplyConfig, ChatReplyFinishReason, ChatReplyPowerStats, ChatReplySpeculatorStats, ChatReplyStats,
};
pub use role::ChatRole;
-pub use speculation_preset::ChatSpeculationPreset;
diff --git a/crates/shoji/src/types/session/chat/speculation_preset.rs b/crates/shoji/src/types/session/chat/speculation_preset.rs
deleted file mode 100644
index af541d10a..000000000
--- a/crates/shoji/src/types/session/chat/speculation_preset.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use crate::types::basic::Feature;
-
-#[bindings::export(Enumeration)]
-#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
-pub enum ChatSpeculationPreset {
- GeneralChat {},
- Summarization {},
- Classification {
- feature: Feature,
- },
-}
diff --git a/crates/uzu/examples/chat_speculation_classification.rs b/crates/uzu/examples/chat_speculation_classification.rs
deleted file mode 100644
index a33337b30..000000000
--- a/crates/uzu/examples/chat_speculation_classification.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-use uzu::{
- engine::{Engine, EngineConfig},
- types::{
- basic::{Feature, ReasoningEffort, SamplingMethod},
- session::chat::{ChatConfig, ChatMessage, ChatReplyConfig, ChatSpeculationPreset},
- },
-};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
- let engine_config = EngineConfig::default();
- let engine = Engine::new(engine_config).await?;
-
- let model = engine.model("Qwen/Qwen3-0.6B".to_string()).await?.ok_or("Model not found")?;
- let downloader = engine.download(&model).await?;
- while let Some(update) = downloader.next().await {
- println!("Download progress: {}", update.progress());
- }
-
- let feature = Feature {
- name: "sentiment".to_string(),
- values: vec![
- "Happy".to_string(),
- "Sad".to_string(),
- "Angry".to_string(),
- "Fearful".to_string(),
- "Surprised".to_string(),
- "Disgusted".to_string(),
- ],
- };
- let chat_config = ChatConfig::default().with_speculation_preset(Some(ChatSpeculationPreset::Classification {
- feature: feature.clone(),
- }));
- let session = engine.chat(model, chat_config).await?;
-
- let text_to_detect_feature = "Today's been awesome! Everything just feels right, and I can't stop smiling.";
- let prompt = format!(
- "Text is: \"{text_to_detect_feature}\". Choose {} from the list: {}. Answer with one word. Don't add a dot at the end.",
- feature.name,
- feature.values.join(", ")
- );
- let messages = vec![
- ChatMessage::system().with_reasoning_effort(ReasoningEffort::Disabled),
- ChatMessage::user().with_text(prompt),
- ];
-
- let chat_reply_config =
- ChatReplyConfig::default().with_token_limit(Some(32)).with_sampling_method(SamplingMethod::Greedy {});
- let replies = session.reply(messages, chat_reply_config).await?;
- if let Some(reply) = replies.first() {
- println!("Prediction: {}", reply.message.text().unwrap_or_default());
- println!("Generated tokens: {}", reply.stats.tokens_count_output.unwrap_or_default());
- }
-
- Ok(())
-}
diff --git a/crates/uzu/examples/chat_speculation_summarization.rs b/crates/uzu/examples/chat_speculation_summarization.rs
deleted file mode 100644
index 68f6a4a96..000000000
--- a/crates/uzu/examples/chat_speculation_summarization.rs
+++ /dev/null
@@ -1,47 +0,0 @@
-use uzu::{
- engine::{Engine, EngineConfig},
- types::{
- basic::{ReasoningEffort, SamplingMethod},
- session::chat::{ChatConfig, ChatMessage, ChatReplyConfig, ChatSpeculationPreset},
- },
-};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
- let engine_config = EngineConfig::default();
- let engine = Engine::new(engine_config).await?;
-
- let model = engine.model("Qwen/Qwen3-0.6B".to_string()).await?.ok_or("Model not found")?;
- let downloader = engine.download(&model).await?;
- while let Some(update) = downloader.next().await {
- println!("Download progress: {}", update.progress());
- }
-
- let text_to_summarize = "A Large Language Model (LLM) is a type of artificial intelligence that processes and generates human-like text. \
- It is trained on vast datasets containing books, articles, and web content, allowing it to understand and predict language patterns. \
- LLMs use deep learning, particularly transformer-based architectures, to analyze text, recognize context, and generate coherent responses. \
- These models have a wide range of applications, including chatbots, content creation, translation, and code generation. \
- One of the key strengths of LLMs is their ability to generate contextually relevant text based on prompts. \
- They utilize self-attention mechanisms to weigh the importance of words within a sentence, improving accuracy and fluency. \
- Examples of popular LLMs include OpenAI's GPT series, Google's BERT, and Meta's LLaMA. \
- As these models grow in size and sophistication, they continue to enhance human-computer interactions, \
- making AI-powered communication more natural and effective.";
- let prompt = format!("Text is: \"{text_to_summarize}\". Write only summary itself.");
- let messages = vec![
- ChatMessage::system().with_reasoning_effort(ReasoningEffort::Disabled),
- ChatMessage::user().with_text(prompt),
- ];
-
- let chat_config = ChatConfig::default().with_speculation_preset(Some(ChatSpeculationPreset::Summarization {}));
- let session = engine.chat(model, chat_config).await?;
-
- let chat_reply_config =
- ChatReplyConfig::default().with_token_limit(Some(256)).with_sampling_method(SamplingMethod::Greedy {});
- let replies = session.reply(messages, chat_reply_config).await?;
- if let Some(reply) = replies.first() {
- println!("Summary: {}", reply.message.text().unwrap_or_default());
- println!("Generation t/s: {}", reply.stats.generate_tokens_per_second.unwrap_or_default());
- }
-
- Ok(())
-}
diff --git a/platforms.toml b/platforms.toml
index a410186de..65bed0afd 100644
--- a/platforms.toml
+++ b/platforms.toml
@@ -175,8 +175,6 @@ swift_version = '[![Swift](https://img.shields.io/badge/Swift-5.9-blue)](https:/
quick-start = { title = "Quick Start", description = "Minimal snippet that downloads a model and generates a single reply" }
chat = { title = "Chat", description = "In this example, we will download a model and get a reply to a specific list of messages:", explanation = "Once loaded, the same `ChatSession` can be reused for multiple requests until you drop it. Each model may consume a significant amount of RAM, so it's important to keep only one session loaded at a time. For iOS apps, we recommend adding the [Increased Memory Capability](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.developer.kernel.increased-memory-limit) entitlement to ensure your app can allocate the required memory." }
chat-cloud = { title = "Chat with the cloud model", description = "In this example, we will get a reply to a specific list of messages from a cloud model:" }
-chat-speculation-summarization = { title = "Chat using speculation preset for summarization", description = "In this example, we will use the `summarization` speculation preset to generate a summary of the input text:", explanation = "You will notice that the model’s run count is lower than the actual number of generated tokens due to speculative decoding, which significantly improves generation speed." }
-chat-speculation-classification = { title = "Chat using speculation preset for classification", description = "In this example, we will use the `classification` speculation preset to determine the sentiment of the user's input:", explanation = "You can view the stats to see that the answer will be ready immediately after the prefill step, and actual generation won’t even start due to speculative decoding, which significantly improves generation speed." }
chat-structured-output = { title = "Chat with structured output", description = "Sometimes you want the generated output to be valid JSON with predefined fields. You can use `Grammar` to manually specify a JSON schema for the response you want to receive:" }
classification = { title = "Classification", description = "In this example, we will use a classification model to determine whether the user's input is safe from a moderation perspective:" }
text-to-speech = { title = "Text to Speech", description = "In this example, we will generate audio from text:" }