A high-performance inference engine for AI models
Per-device speculative suffix size (#763)
uuuvnff43a2a17c2a
diff --git a/crates/uzu-engine/src/backends/common/context.rs b/crates/uzu-engine/src/backends/common/context.rsindex df10f9b74..bfdde01da 100644--- a/crates/uzu-engine/src/backends/common/context.rs+++ b/crates/uzu-engine/src/backends/common/context.rs@@ -7,6 +7,8 @@ pub trait Context: Sized + Send + Sync {fn new() -> Result<Arc<Self>, <Self::Backend as Backend>::Error>;+ fn device_name(&self) -> Option<&str>;+fn create_command_buffer(&self,name: Option<&str>,diff --git a/crates/uzu-engine/src/backends/cpu/context.rs b/crates/uzu-engine/src/backends/cpu/context.rsindex 9055f98d2..6519d6512 100644--- a/crates/uzu-engine/src/backends/cpu/context.rs+++ b/crates/uzu-engine/src/backends/cpu/context.rs@@ -32,6 +32,10 @@ impl Context for CpuContext {}))}+ fn device_name(&self) -> Option<&str> {+ None+ }+fn create_buffer(&self,size: usize,diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rsindex 0670d313b..310f88b24 100644--- a/crates/uzu-engine/src/backends/metal/context.rs+++ b/crates/uzu-engine/src/backends/metal/context.rs@@ -33,6 +33,7 @@ use crate::backends::{pub struct MetalContext {pub device: Retained<ProtocolObject<dyn MTLDevice>>,+ device_name: String,pub command_queue: Retained<ProtocolObject<dyn MTLCommandQueue>>,pub command_queue4: Retained<ProtocolObject<dyn MTL4CommandQueue>>,timeline_event: Retained<ProtocolObject<dyn MTLEvent>>,@@ -148,6 +149,7 @@ impl Context for MetalContext {fn new() -> Result<Arc<Self>, MetalError> {let device: Retained<ProtocolObject<dyn MTLDevice>> =<dyn MTLDevice>::system_default().ok_or(MetalError::CannotOpenDevice)?;+ let device_name = device.name();let command_queue =device.new_command_queue_with_max_command_buffer_count(1024).ok_or(MetalError::CannotCreateCommandQueue)?;@@ -170,6 +172,7 @@ impl Context for MetalContext {Ok(Arc::new_cyclic(|weak_self| Self {device,+ device_name,command_queue,command_queue4,timeline_event,@@ -186,6 +189,10 @@ impl Context for MetalContext {}))}+ fn device_name(&self) -> Option<&str> {+ Some(&self.device_name)+ }+fn create_buffer(&self,size: usize,diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rsindex 94c5d315f..8e4c208bd 100644--- a/crates/uzu-engine/src/engine/language_model/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/mod.rs@@ -88,7 +88,8 @@ impl<B: Backend> Engine<B> {let speculator = speculator_path.as_deref().map(|speculator_path| DFlashTfmSpeculator::new(speculator_path, self.context.clone()))- .transpose()?;+ .transpose()?+ .flatten();let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex 1f85bf6db..85622c5d6 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -24,7 +24,6 @@ use crate::{stream::{LanguageModelStreamError, LanguageModelStreamOptions},},},- speculators::dflash_tfm::{DFlashTfmTreeConstructionMethod, DFlashTfmTreeShape},trie::TrieNode,};@@ -547,15 +546,11 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {None};- let full_batch_size = 16;- let speculation_batch = self- .model_state- .max_context_length- .map_or(full_batch_size, |max_context_length| full_batch_size.min(max_context_length - context_length));-let mut pending = Vec::new();- let (input_trie, chain_copy, full_accept) = if speculation_batch > 1- && let Some(speculator) = &self.model.speculator+ let (input_trie, chain_copy, full_accept) = if let Some(speculator) = &self.model.speculator+ && let Some(shape) = speculator.make_shape(+ self.model_state.max_context_length.map(|max_context_length| max_context_length - context_length),+ )&& let (root_token, Some(output_norm)) = prev_output.resolve(&mut self.model_state.tokens,#[cfg(grammar)]@@ -569,20 +564,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {output_norm,root_token as u32,self.model.decoder.embedding(),- DFlashTfmTreeShape {- tree_budget: speculation_batch,- max_tree_depth: 16,- dflash_depth_override: None,- construction_method: if speculator.has_weaver() {- DFlashTfmTreeConstructionMethod::Weaver {- rounds: 16,- expand_per_round: 4,- expand_width: 4,- }- } else {- DFlashTfmTreeConstructionMethod::Argmax- },- },+ shape,#[cfg(grammar)]self.options.grammar.as_mut(),&self.model_state.prng,diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex 9ee920a2a..f8316410b 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -1,17 +1,21 @@use std::{+ collections::HashMap,fs::File,io::{self, BufReader},path::Path,sync::Arc,};+use serde::{Deserialize, Serialize};use thiserror::Error;pub use crate::encodable_block::dflash::DFlashState;#[cfg(grammar)]use crate::engine::language_model::grammar::Grammar;use crate::{- backends::common::{Allocation, AllocationPool, Backend, Encoder, gpu_types::trie::TrieNode as GpuTrieNode},+ backends::common::{+ Allocation, AllocationPool, Backend, Context, Encoder, gpu_types::trie::TrieNode as GpuTrieNode,+ },config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},data_type::DataType,encodable_block::{@@ -53,6 +57,8 @@ pub enum DFlashSpeculatorLoadError<B: Backend> {Weaver(#[from] WeaverNewError<B>),}+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]+#[serde(tag = "type")]pub enum DFlashTfmTreeConstructionMethod {Argmax,Weaver {@@ -62,6 +68,7 @@ pub enum DFlashTfmTreeConstructionMethod {},}+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]pub struct DFlashTfmTreeShape {pub tree_budget: u32,pub max_tree_depth: u32,@@ -75,13 +82,26 @@ pub struct DFlashTfmSpeculator<B: Backend> {weaver: Option<Weaver<B>>,sampling: Sampling<B>,config: DFlashSpeculatorConfig,+ shape: DFlashTfmTreeShape,}impl<B: Backend> DFlashTfmSpeculator<B> {pub fn new(model_path: &Path,context: Arc<B::Context>,- ) -> Result<Self, DFlashSpeculatorLoadError<B>> {+ ) -> Result<Option<Self>, DFlashSpeculatorLoadError<B>> {+ let mut shapes = serde_json::from_reader::<_, HashMap<String, DFlashTfmTreeShape>>(BufReader::new(+ File::open(model_path.join("shapes.json"))?,+ ))?;++ let Some(shape) = context+ .device_name()+ .and_then(|device_name| shapes.remove(device_name))+ .or_else(|| shapes.remove("default"))+ else {+ return Ok(None);+ };+let data_type = DataType::BF16;let config: SpeculatorModelConfig =@@ -110,17 +130,14 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let sampling = Sampling::new(DataType::F32, config.draft_config.vocab_size);- Ok(Self {+ Ok(Some(Self {context,dflash,weaver,sampling,config,- })- }-- pub fn has_weaver(&self) -> bool {- self.weaver.is_some()+ shape,+ }))}pub fn hidden_feature_layer_indices(&self) -> &[u32] {@@ -144,6 +161,31 @@ impl<B: Backend> DFlashTfmSpeculator<B> {self.dflash.encode_accept(state, target_features, accepted_indices, encoder)}+ pub fn make_shape(+ &self,+ max_depth: Option<u32>,+ ) -> Option<DFlashTfmTreeShape> {+ if max_depth.is_some_and(|max_depth| max_depth < 2) {+ return None;+ }++ let mut shape = self.shape.clone();++ if let Some(max_depth) = max_depth {+ shape.max_tree_depth = u32::min(shape.max_tree_depth, max_depth);++ if let DFlashTfmTreeConstructionMethod::Weaver {+ rounds,+ ..+ } = &mut shape.construction_method+ {+ *rounds = u32::min(*rounds, max_depth);+ }+ }++ Some(shape)+ }+pub fn propose_tree(&self,state: &mut DFlashState<B>,