Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Per-device speculative suffix size (#763)

uuuvnff43a2a17c2a
diff --git a/crates/uzu-engine/src/backends/common/context.rs b/crates/uzu-engine/src/backends/common/context.rs
index df10f9b74..bfdde01da 100644
--- a/crates/uzu-engine/src/backends/common/context.rs
+++ b/crates/uzu-engine/src/backends/common/context.rs
@@ -7,6 +7,8 @@ pub trait Context: Sized + Send + Sync {
fn new() -> Result<Arc<Self>, <Self::Backend as Backend>::Error>;
+ fn device_name(&self) -> Option<&str>;
+
fn create_command_buffer(
&self,
name: Option<&str>,
diff --git a/crates/uzu-engine/src/backends/cpu/context.rs b/crates/uzu-engine/src/backends/cpu/context.rs
index 9055f98d2..6519d6512 100644
--- a/crates/uzu-engine/src/backends/cpu/context.rs
+++ b/crates/uzu-engine/src/backends/cpu/context.rs
@@ -32,6 +32,10 @@ impl Context for CpuContext {
}))
}
+ fn device_name(&self) -> Option<&str> {
+ None
+ }
+
fn create_buffer(
&self,
size: usize,
diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rs
index 0670d313b..310f88b24 100644
--- a/crates/uzu-engine/src/backends/metal/context.rs
+++ b/crates/uzu-engine/src/backends/metal/context.rs
@@ -33,6 +33,7 @@ use crate::backends::{
pub struct MetalContext {
pub device: Retained<ProtocolObject<dyn MTLDevice>>,
+ device_name: String,
pub command_queue: Retained<ProtocolObject<dyn MTLCommandQueue>>,
pub command_queue4: Retained<ProtocolObject<dyn MTL4CommandQueue>>,
timeline_event: Retained<ProtocolObject<dyn MTLEvent>>,
@@ -148,6 +149,7 @@ impl Context for MetalContext {
fn new() -> Result<Arc<Self>, MetalError> {
let device: Retained<ProtocolObject<dyn MTLDevice>> =
<dyn MTLDevice>::system_default().ok_or(MetalError::CannotOpenDevice)?;
+ let device_name = device.name();
let command_queue =
device.new_command_queue_with_max_command_buffer_count(1024).ok_or(MetalError::CannotCreateCommandQueue)?;
@@ -170,6 +172,7 @@ impl Context for MetalContext {
Ok(Arc::new_cyclic(|weak_self| Self {
device,
+ device_name,
command_queue,
command_queue4,
timeline_event,
@@ -186,6 +189,10 @@ impl Context for MetalContext {
}))
}
+ fn device_name(&self) -> Option<&str> {
+ Some(&self.device_name)
+ }
+
fn create_buffer(
&self,
size: usize,
diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rs
index 94c5d315f..8e4c208bd 100644
--- a/crates/uzu-engine/src/engine/language_model/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/mod.rs
@@ -88,7 +88,8 @@ impl<B: Backend> Engine<B> {
let speculator = speculator_path
.as_deref()
.map(|speculator_path| DFlashTfmSpeculator::new(speculator_path, self.context.clone()))
- .transpose()?;
+ .transpose()?
+ .flatten();
let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index 1f85bf6db..85622c5d6 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -24,7 +24,6 @@ use crate::{
stream::{LanguageModelStreamError, LanguageModelStreamOptions},
},
},
- speculators::dflash_tfm::{DFlashTfmTreeConstructionMethod, DFlashTfmTreeShape},
trie::TrieNode,
};
@@ -547,15 +546,11 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
None
};
- let full_batch_size = 16;
- let speculation_batch = self
- .model_state
- .max_context_length
- .map_or(full_batch_size, |max_context_length| full_batch_size.min(max_context_length - context_length));
-
let mut pending = Vec::new();
- let (input_trie, chain_copy, full_accept) = if speculation_batch > 1
- && let Some(speculator) = &self.model.speculator
+ let (input_trie, chain_copy, full_accept) = if let Some(speculator) = &self.model.speculator
+ && let Some(shape) = speculator.make_shape(
+ self.model_state.max_context_length.map(|max_context_length| max_context_length - context_length),
+ )
&& let (root_token, Some(output_norm)) = prev_output.resolve(
&mut self.model_state.tokens,
#[cfg(grammar)]
@@ -569,20 +564,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
output_norm,
root_token as u32,
self.model.decoder.embedding(),
- DFlashTfmTreeShape {
- tree_budget: speculation_batch,
- max_tree_depth: 16,
- dflash_depth_override: None,
- construction_method: if speculator.has_weaver() {
- DFlashTfmTreeConstructionMethod::Weaver {
- rounds: 16,
- expand_per_round: 4,
- expand_width: 4,
- }
- } else {
- DFlashTfmTreeConstructionMethod::Argmax
- },
- },
+ shape,
#[cfg(grammar)]
self.options.grammar.as_mut(),
&self.model_state.prng,
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index 9ee920a2a..f8316410b 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -1,17 +1,21 @@
use std::{
+ collections::HashMap,
fs::File,
io::{self, BufReader},
path::Path,
sync::Arc,
};
+use serde::{Deserialize, Serialize};
use thiserror::Error;
pub use crate::encodable_block::dflash::DFlashState;
#[cfg(grammar)]
use crate::engine::language_model::grammar::Grammar;
use crate::{
- backends::common::{Allocation, AllocationPool, Backend, Encoder, gpu_types::trie::TrieNode as GpuTrieNode},
+ backends::common::{
+ Allocation, AllocationPool, Backend, Context, Encoder, gpu_types::trie::TrieNode as GpuTrieNode,
+ },
config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},
data_type::DataType,
encodable_block::{
@@ -53,6 +57,8 @@ pub enum DFlashSpeculatorLoadError<B: Backend> {
Weaver(#[from] WeaverNewError<B>),
}
+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]
+#[serde(tag = "type")]
pub enum DFlashTfmTreeConstructionMethod {
Argmax,
Weaver {
@@ -62,6 +68,7 @@ pub enum DFlashTfmTreeConstructionMethod {
},
}
+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]
pub struct DFlashTfmTreeShape {
pub tree_budget: u32,
pub max_tree_depth: u32,
@@ -75,13 +82,26 @@ pub struct DFlashTfmSpeculator<B: Backend> {
weaver: Option<Weaver<B>>,
sampling: Sampling<B>,
config: DFlashSpeculatorConfig,
+ shape: DFlashTfmTreeShape,
}
impl<B: Backend> DFlashTfmSpeculator<B> {
pub fn new(
model_path: &Path,
context: Arc<B::Context>,
- ) -> Result<Self, DFlashSpeculatorLoadError<B>> {
+ ) -> Result<Option<Self>, DFlashSpeculatorLoadError<B>> {
+ let mut shapes = serde_json::from_reader::<_, HashMap<String, DFlashTfmTreeShape>>(BufReader::new(
+ File::open(model_path.join("shapes.json"))?,
+ ))?;
+
+ let Some(shape) = context
+ .device_name()
+ .and_then(|device_name| shapes.remove(device_name))
+ .or_else(|| shapes.remove("default"))
+ else {
+ return Ok(None);
+ };
+
let data_type = DataType::BF16;
let config: SpeculatorModelConfig =
@@ -110,17 +130,14 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let sampling = Sampling::new(DataType::F32, config.draft_config.vocab_size);
- Ok(Self {
+ Ok(Some(Self {
context,
dflash,
weaver,
sampling,
config,
- })
- }
-
- pub fn has_weaver(&self) -> bool {
- self.weaver.is_some()
+ shape,
+ }))
}
pub fn hidden_feature_layer_indices(&self) -> &[u32] {
@@ -144,6 +161,31 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
self.dflash.encode_accept(state, target_features, accepted_indices, encoder)
}
+ pub fn make_shape(
+ &self,
+ max_depth: Option<u32>,
+ ) -> Option<DFlashTfmTreeShape> {
+ if max_depth.is_some_and(|max_depth| max_depth < 2) {
+ return None;
+ }
+
+ let mut shape = self.shape.clone();
+
+ if let Some(max_depth) = max_depth {
+ shape.max_tree_depth = u32::min(shape.max_tree_depth, max_depth);
+
+ if let DFlashTfmTreeConstructionMethod::Weaver {
+ rounds,
+ ..
+ } = &mut shape.construction_method
+ {
+ *rounds = u32::min(*rounds, max_depth);
+ }
+ }
+
+ Some(shape)
+ }
+
pub fn propose_tree(
&self,
state: &mut DFlashState<B>,