A high-performance inference engine for AI models
DFlash + Weaver Configs (#611)
CCYeh2f4a6edc5406
diff --git a/crates/backend-uzu/src/bridge/model_metadata.rs b/crates/backend-uzu/src/bridge/model_metadata.rsindex 214e733bc..2430e715b 100644--- a/crates/backend-uzu/src/bridge/model_metadata.rs+++ b/crates/backend-uzu/src/bridge/model_metadata.rs@@ -19,6 +19,5 @@ pub fn resolve_model_specialization(model_path: &Path) -> Result<ModelSpecializaOk(match config {AnyModelConfig::LanguageModelConfig(_) => ModelSpecialization::Chat {},AnyModelConfig::ClassifierModelConfig(_) => ModelSpecialization::Classification {},- AnyModelConfig::DFlashSpeculatorConfig(_) => ModelSpecialization::Speculation {},})}diff --git a/crates/backend-uzu/src/config/dflash.rs b/crates/backend-uzu/src/config/dflash.rsindex 37f4f7aea..84de9a796 100644--- a/crates/backend-uzu/src/config/dflash.rs+++ b/crates/backend-uzu/src/config/dflash.rs@@ -2,28 +2,12 @@ use proc_macros::uzu_config;use crate::config::{linear::LinearConfig, mlp::dense_mlp::DenseMLPConfig, normalization::NormalizationConfig, rope::AnyRoPEConfig,+ token_mixer::attention::AttentionConfig,};-#[uzu_config]-pub struct DFlashAttentionConfig {- pub linear_config: LinearConfig,- pub query_norm_config: NormalizationConfig,- pub key_norm_config: NormalizationConfig,- pub rope_config: AnyRoPEConfig,- pub num_heads: usize,- pub num_key_value_heads: usize,- pub head_dim: usize,- pub has_attention_biases: bool,- pub has_output_biases: bool,- /// Symmetric window: keys within +/- `sliding_window_size / 2` of the query- /// position, not the causal sliding window used by uzu attention.- pub sliding_window_size: Option<usize>,- pub scale: f32,-}-#[uzu_config]pub struct DFlashDraftLayerConfig {- pub attention_config: DFlashAttentionConfig,+ pub attention_config: AttentionConfig,pub input_norm_config: NormalizationConfig,/// Sits in the pre-MLP-norm position: normalizes hidden + attention output/// before the MLP, unlike uzu's post-mixer norm.@@ -42,6 +26,7 @@ pub struct DFlashDraftConfig {pub vocab_size: usize,pub context_projection_config: LinearConfig,pub context_norm_config: NormalizationConfig,+ pub rope_config: AnyRoPEConfig,pub layer_configs: Box<[DFlashDraftLayerConfig]>,pub output_norm_config: NormalizationConfig,}diff --git a/crates/backend-uzu/src/config/mod.rs b/crates/backend-uzu/src/config/mod.rsindex eeb01ec99..964ec30c9 100644--- a/crates/backend-uzu/src/config/mod.rs+++ b/crates/backend-uzu/src/config/mod.rs@@ -9,8 +9,10 @@ pub mod model;pub mod normalization;pub mod per_layer_embedding;pub mod rope;+pub mod speculator;pub mod token_codec;pub mod token_mixer;pub mod transformer;pub mod transformer_layer;+pub mod weaver;pub mod weight_matrix;diff --git a/crates/backend-uzu/src/config/model/dflash_speculator.rs b/crates/backend-uzu/src/config/model/dflash_speculator.rsdeleted file mode 100644index 63d05aac4..000000000--- a/crates/backend-uzu/src/config/model/dflash_speculator.rs+++ /dev/null@@ -1,9 +0,0 @@-use proc_macros::uzu_config;--use crate::config::dflash::DFlashDraftConfig;--#[allow(unused)] // TODO-#[uzu_config(super::ModelConfig)]-pub struct DFlashSpeculatorConfig {- pub draft_config: DFlashDraftConfig,-}diff --git a/crates/backend-uzu/src/config/model/mod.rs b/crates/backend-uzu/src/config/model/mod.rsindex 8b066afd7..a9fc012c1 100644--- a/crates/backend-uzu/src/config/model/mod.rs+++ b/crates/backend-uzu/src/config/model/mod.rs@@ -3,15 +3,10 @@ use proc_macros::uzu_config_abstract;use crate::config::token_codec::AnyTokenCodecConfig;pub mod classifier_model;-pub mod dflash_speculator;pub mod generation;pub mod language_model;-#[uzu_config_abstract(- language_model::LanguageModelConfig,- classifier_model::ClassifierModelConfig,- dflash_speculator::DFlashSpeculatorConfig-)]+#[uzu_config_abstract(language_model::LanguageModelConfig, classifier_model::ClassifierModelConfig)]pub struct ModelConfig {pub token_codec_config: AnyTokenCodecConfig,}diff --git a/crates/backend-uzu/src/config/speculator/dflash.rs b/crates/backend-uzu/src/config/speculator/dflash.rsnew file mode 100644index 000000000..01d108c0c--- /dev/null+++ b/crates/backend-uzu/src/config/speculator/dflash.rs@@ -0,0 +1,9 @@+use proc_macros::uzu_config;++use crate::config::{dflash::DFlashDraftConfig, weaver::WeaverConfig};++#[uzu_config(super::SpeculatorConfig)]+pub struct DFlashSpeculatorConfig {+ pub draft_config: DFlashDraftConfig,+ pub weaver_config: Option<WeaverConfig>,+}diff --git a/crates/backend-uzu/src/config/speculator/mod.rs b/crates/backend-uzu/src/config/speculator/mod.rsnew file mode 100644index 000000000..62ebc7099--- /dev/null+++ b/crates/backend-uzu/src/config/speculator/mod.rs@@ -0,0 +1,7 @@+use proc_macros::uzu_config_abstract;++pub mod dflash;+pub mod model;++#[uzu_config_abstract(dflash::DFlashSpeculatorConfig)]+pub struct SpeculatorConfig;diff --git a/crates/backend-uzu/src/config/speculator/model.rs b/crates/backend-uzu/src/config/speculator/model.rsnew file mode 100644index 000000000..8ea57734b--- /dev/null+++ b/crates/backend-uzu/src/config/speculator/model.rs@@ -0,0 +1,8 @@+use proc_macros::uzu_config;++use crate::config::speculator::AnySpeculatorConfig;++#[uzu_config(super::SpeculatorConfig)]+pub struct SpeculatorModelConfig {+ pub speculator_config: AnySpeculatorConfig,+}diff --git a/crates/backend-uzu/src/config/weaver.rs b/crates/backend-uzu/src/config/weaver.rsnew file mode 100644index 000000000..48f38cb2f--- /dev/null+++ b/crates/backend-uzu/src/config/weaver.rs@@ -0,0 +1,17 @@+use proc_macros::uzu_config;++use crate::config::{linear::LinearConfig, normalization::NormalizationConfig};++#[uzu_config]+pub struct WeaverConfig {+ pub model_dim: usize,+ pub target_model_dim: usize,+ pub target_embedding_dim: usize,+ pub num_layers: usize,+ pub num_heads: usize,+ pub hidden_dim: usize,+ pub max_depth: usize,+ pub candidate_pool_size: usize,+ pub linear_config: LinearConfig,+ pub norm_config: NormalizationConfig,+}diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rsindex 9929fdc99..6cb423c1e 100644--- a/crates/backend-uzu/src/encodable_block/dflash.rs+++ b/crates/backend-uzu/src/encodable_block/dflash.rs@@ -3,9 +3,10 @@ use thiserror::Error;use crate::{backends::common::Backend,config::{- dflash::{DFlashAttentionConfig, DFlashDraftConfig, DFlashDraftLayerConfig},+ dflash::{DFlashDraftConfig, DFlashDraftLayerConfig},mlp::AnyMLPConfig,normalization::NormalizationConfig,+ token_mixer::attention::AttentionConfig,},data_type::DataType,encodable_block::{@@ -16,7 +17,6 @@ use crate::{parameters::{ParameterLoaderError, ParameterTree},};-// TODO: remove once traversal verification consumes the loaded DFlash model.#[allow(dead_code)]pub(crate) struct DFlashDraft<B: Backend> {context_projection: Box<dyn Linear<B>>,@@ -27,7 +27,6 @@ pub(crate) struct DFlashDraft<B: Backend> {max_context_length: Option<usize>,}-// TODO: remove once traversal verification consumes the loaded DFlash model.#[allow(dead_code)]struct DFlashDraftLayer<B: Backend> {attention: DFlashAttention<B>,@@ -36,12 +35,10 @@ struct DFlashDraftLayer<B: Backend> {mlp: Box<dyn Mlp<B>>,}-// TODO: remove once traversal verification consumes the loaded DFlash model.#[allow(dead_code)]struct DFlashAttention<B: Backend> {- query_projection: Box<dyn Linear<B>>,- key_value_projection: Box<dyn Linear<B>>,- output_projection: Box<dyn Linear<B>>,+ qkv_projection: Box<dyn Linear<B>>,+ out_projection: Box<dyn Linear<B>>,query_norm: Normalization<B>,key_norm: Normalization<B>,}@@ -56,6 +53,8 @@ pub enum DFlashDraftNewError<B: Backend> {Normalization(#[from] NormalizationNewError<B>),#[error("MLP error: {0}")]Mlp(#[from] MlpBlockError<B>),+ #[error("invalid DFlash attention config: {0}")]+ InvalidAttentionConfig(&'static str),}fn plain_norm<B: Backend>(@@ -113,11 +112,7 @@ impl<B: Backend> DFlashDraft<B> {¶meter_tree.subtree("output_norm")?,data_type,)?;- let max_context_length = config- .layer_configs- .iter()- .map(|layer_config| *layer_config.attention_config.rope_config.max_sequence_length())- .min();+ let max_context_length = Some(*config.rope_config.max_sequence_length());Ok(Self {context_projection,context_norm,@@ -176,55 +171,44 @@ impl<B: Backend> DFlashAttention<B> {fn new(context: &B::Context,model_dim: usize,- config: &DFlashAttentionConfig,+ config: &AttentionConfig,parameter_tree: &ParameterTree<B>,data_type: DataType,) -> Result<Self, DFlashDraftNewError<B>> {let query_dim = config.num_heads * config.head_dim;- let key_value_dim = config.num_key_value_heads * config.head_dim;- let query_projection = <dyn Linear<B>>::new(- model_dim,- [query_dim],- config.has_attention_biases,- context,- data_type,- ¶meter_tree.subtree("query_projection")?,- )?;- let key_value_projection = <dyn Linear<B>>::new(+ let key_value_dim = config.num_groups * config.head_dim;+ let qkv_projection = <dyn Linear<B>>::new(model_dim,- [2 * key_value_dim],- config.has_attention_biases,+ [query_dim, key_value_dim, key_value_dim],+ config.has_qkv_biases,context,data_type,- ¶meter_tree.subtree("key_value_projection")?,+ ¶meter_tree.subtree("qkv_projection")?,)?;- let output_projection = <dyn Linear<B>>::new(+ let out_projection = <dyn Linear<B>>::new(query_dim,[model_dim],- config.has_output_biases,- context,- data_type,- ¶meter_tree.subtree("output_projection")?,- )?;- let query_norm = plain_norm(- context,- config.head_dim,- &config.query_norm_config,- ¶meter_tree.subtree("query_norm")?,- data_type,- )?;- let key_norm = plain_norm(+ config.has_out_biases,context,- config.head_dim,- &config.key_norm_config,- ¶meter_tree.subtree("key_norm")?,data_type,+ ¶meter_tree.subtree("out_projection")?,)?;+ let query_norm_config = config+ .query_norm_config+ .as_ref()+ .ok_or(DFlashDraftNewError::InvalidAttentionConfig("query_norm_config is required"))?;+ let query_norm =+ plain_norm(context, config.head_dim, query_norm_config, ¶meter_tree.subtree("query_norm")?, data_type)?;+ let key_norm_config = config+ .key_norm_config+ .as_ref()+ .ok_or(DFlashDraftNewError::InvalidAttentionConfig("key_norm_config is required"))?;+ let key_norm =+ plain_norm(context, config.head_dim, key_norm_config, ¶meter_tree.subtree("key_norm")?, data_type)?;Ok(Self {- query_projection,- key_value_projection,- output_projection,+ qkv_projection,+ out_projection,query_norm,key_norm,})diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rsindex 839f36fa4..db7d59ec6 100644--- a/crates/backend-uzu/src/encodable_block/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mod.rs@@ -12,3 +12,4 @@ pub mod prediction_head;pub mod sampling;pub mod transformer;pub mod transformer_layer;+pub mod weaver;diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rsnew file mode 100644index 000000000..c48b1a31e--- /dev/null+++ b/crates/backend-uzu/src/encodable_block/weaver.rs@@ -0,0 +1,189 @@+use thiserror::Error;++use crate::{+ backends::common::{Allocation, Backend},+ config::{normalization::NormalizationConfig, weaver::WeaverConfig},+ data_type::DataType,+ encodable_block::linear::{Linear, LinearBlockError},+ parameters::{ParameterLoaderError, ParameterTree},+};++#[allow(dead_code)]+pub struct Weaver<B: Backend> {+ embedding_norm: WeaverNorm<B>,+ hidden_state_norm: WeaverNorm<B>,+ embedding_projection: Box<dyn Linear<B>>,+ blocks: Box<[WeaverBlock<B>]>,+ output_norm: WeaverNorm<B>,+ hidden_state_projection: Box<dyn Linear<B>>,+ query_projection: Box<dyn Linear<B>>,+ position_embeddings: Allocation<B>,+}++#[allow(dead_code)]+struct WeaverNorm<B: Backend> {+ scales: Allocation<B>,+ biases: Option<Allocation<B>>,+}++#[allow(dead_code)]+struct WeaverBlock<B: Backend> {+ pre_attention_norm: WeaverNorm<B>,+ qkv_projection: Box<dyn Linear<B>>,+ out_projection: Box<dyn Linear<B>>,+ pre_mlp_norm: WeaverNorm<B>,+ up_projection: Box<dyn Linear<B>>,+ down_projection: Box<dyn Linear<B>>,+}++#[derive(Debug, Error)]+pub enum WeaverNewError<B: Backend> {+ #[error("parameter loader error: {0}")]+ ParameterLoader(#[from] ParameterLoaderError<B>),+ #[error("linear error: {0}")]+ Linear(#[from] LinearBlockError<B>),+ #[error("model_dim must be divisible by num_heads")]+ InvalidHeadConfig,+}++fn plain_norm<B: Backend>(+ dim: usize,+ config: &NormalizationConfig,+ parameter_tree: &ParameterTree<B>,+) -> Result<WeaverNorm<B>, ParameterLoaderError<B>> {+ let scales = parameter_tree.leaf("scales")?.validate(&[dim], DataType::F32)?.read_allocation()?;+ let biases = config+ .has_biases+ .then(|| parameter_tree.leaf("biases")?.validate(&[dim], DataType::F32)?.read_allocation())+ .transpose()?;+ Ok(WeaverNorm {+ scales,+ biases,+ })+}++impl<B: Backend> Weaver<B> {+ pub(crate) fn new(+ context: &B::Context,+ config: &WeaverConfig,+ parameter_tree: &ParameterTree<B>,+ data_type: DataType,+ ) -> Result<Self, WeaverNewError<B>> {+ if config.num_heads == 0 || !config.model_dim.is_multiple_of(config.num_heads) {+ return Err(WeaverNewError::InvalidHeadConfig);+ }++ let embedding_norm =+ plain_norm(config.target_embedding_dim, &config.norm_config, ¶meter_tree.subtree("embedding_norm")?)?;+ let hidden_state_norm =+ plain_norm(config.target_model_dim, &config.norm_config, ¶meter_tree.subtree("hidden_state_norm")?)?;+ let embedding_projection = <dyn Linear<B>>::new(+ config.target_embedding_dim,+ [config.model_dim],+ true,+ context,+ data_type,+ ¶meter_tree.subtree("embedding_projection")?,+ )?;+ let blocks_tree = parameter_tree.subtree("blocks")?;+ let blocks = (0..config.num_layers)+ .map(|index| {+ WeaverBlock::new(+ context,+ config.model_dim,+ config.hidden_dim,+ &config.norm_config,+ &blocks_tree.subtree(&index.to_string())?,+ data_type,+ )+ })+ .collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;+ let output_norm = plain_norm(config.model_dim, &config.norm_config, ¶meter_tree.subtree("output_norm")?)?;+ let hidden_state_projection = <dyn Linear<B>>::new(+ config.target_model_dim,+ [config.model_dim],+ true,+ context,+ data_type,+ ¶meter_tree.subtree("hidden_state_projection")?,+ )?;+ let query_projection = <dyn Linear<B>>::new(+ config.model_dim,+ [config.target_model_dim],+ false,+ context,+ data_type,+ ¶meter_tree.subtree("query_projection")?,+ )?;+ let position_embeddings = parameter_tree+ .leaf("position_embeddings")?+ .validate(&[config.max_depth, config.model_dim], DataType::F32)?+ .read_allocation()?;++ Ok(Self {+ embedding_norm,+ hidden_state_norm,+ embedding_projection,+ blocks,+ output_norm,+ hidden_state_projection,+ query_projection,+ position_embeddings,+ })+ }+}++impl<B: Backend> WeaverBlock<B> {+ fn new(+ context: &B::Context,+ model_dim: usize,+ hidden_dim: usize,+ norm_config: &NormalizationConfig,+ parameter_tree: &ParameterTree<B>,+ data_type: DataType,+ ) -> Result<Self, WeaverNewError<B>> {+ let pre_attention_norm = plain_norm(model_dim, norm_config, ¶meter_tree.subtree("pre_attention_norm")?)?;+ let qkv_projection = <dyn Linear<B>>::new(+ model_dim,+ [model_dim, model_dim, model_dim],+ false,+ context,+ data_type,+ ¶meter_tree.subtree("qkv_projection")?,+ )?;+ let out_projection = <dyn Linear<B>>::new(+ model_dim,+ [model_dim],+ false,+ context,+ data_type,+ ¶meter_tree.subtree("out_projection")?,+ )?;+ let pre_mlp_norm = plain_norm(model_dim, norm_config, ¶meter_tree.subtree("pre_mlp_norm")?)?;+ let up_projection = <dyn Linear<B>>::new(+ model_dim,+ [hidden_dim],+ true,+ context,+ data_type,+ ¶meter_tree.subtree("up_projection")?,+ )?;+ let down_projection = <dyn Linear<B>>::new(+ hidden_dim,+ [model_dim],+ true,+ context,+ data_type,+ ¶meter_tree.subtree("down_projection")?,+ )?;++ Ok(Self {+ pre_attention_norm,+ qkv_projection,+ out_projection,+ pre_mlp_norm,+ up_projection,+ down_projection,+ })+ }+}diff --git a/crates/backend-uzu/src/engine/dflash_speculator.rs b/crates/backend-uzu/src/engine/dflash_speculator.rsindex 6386d59b2..15dcd47e0 100644--- a/crates/backend-uzu/src/engine/dflash_speculator.rs+++ b/crates/backend-uzu/src/engine/dflash_speculator.rs@@ -8,9 +8,12 @@ use thiserror::Error;use crate::{backends::common::Backend,- config::model::dflash_speculator::DFlashSpeculatorConfig,+ config::speculator::{AnySpeculatorConfig, model::SpeculatorModelConfig},data_type::DataType,- encodable_block::dflash::{DFlashDraft, DFlashDraftNewError},+ encodable_block::{+ dflash::{DFlashDraft, DFlashDraftNewError},+ weaver::{Weaver, WeaverNewError},+ },engine::Engine,parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},speculators::dflash_speculator::DFlashSpeculator,@@ -28,6 +31,8 @@ pub enum EngineLoadDFlashSpeculatorError<B: Backend> {ParameterLoader(#[from] ParameterLoaderError<B>),#[error("DFlash draft error: {0}")]Draft(#[from] DFlashDraftNewError<B>),+ #[error("Weaver error: {0}")]+ Weaver(#[from] WeaverNewError<B>),}impl<B: Backend> Engine<B> {@@ -35,18 +40,27 @@ impl<B: Backend> Engine<B> {&self,model_path: &Path,) -> Result<DFlashSpeculator<B>, EngineLoadDFlashSpeculatorError<B>> {- let config: DFlashSpeculatorConfig =+ let config: SpeculatorModelConfig =serde_json::from_reader(BufReader::new(File::open(model_path.join("config.json"))?))?;+ let AnySpeculatorConfig::DFlashSpeculatorConfig(speculator_config) = config.speculator_config;let data_type = DataType::BF16;let weights_file = File::open(model_path.join("model.safetensors"))?;let weight_loader = ParameterLoader::new(&weights_file, &*self.context)?;- let draft_tree = weight_loader.tree().subtree("draft_model")?;- let draft_model = DFlashDraft::new(&*self.context, &config.draft_config, &draft_tree, data_type)?;+ let speculator_tree = weight_loader.tree().subtree("speculator")?;+ let draft_tree = speculator_tree.subtree("draft_model")?;+ let draft_model = DFlashDraft::new(&*self.context, &speculator_config.draft_config, &draft_tree, data_type)?;+ let weaver = speculator_config+ .weaver_config+ .as_ref()+ .map(|weaver_config| {+ Weaver::new(&*self.context, weaver_config, &speculator_tree.subtree("weaver")?, data_type)+ })+ .transpose()?;weight_loader.tree().assert_all_tensors_validated()?;- Ok(DFlashSpeculator::new(draft_model, config))+ Ok(DFlashSpeculator::new(draft_model, weaver, speculator_config))}}diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rsindex 613f42b41..d950f30a5 100644--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs+++ b/crates/backend-uzu/src/speculators/dflash_speculator.rs@@ -1,22 +1,25 @@use crate::{- backends::common::Backend, config::model::dflash_speculator::DFlashSpeculatorConfig,- encodable_block::dflash::DFlashDraft,+ backends::common::Backend,+ config::speculator::dflash::DFlashSpeculatorConfig,+ encodable_block::{dflash::DFlashDraft, weaver::Weaver},};-// TODO: remove once traversal verification consumes the loaded DFlash model.#[allow(dead_code)]pub struct DFlashSpeculator<B: Backend> {pub(crate) model: DFlashDraft<B>,+ pub(crate) weaver: Option<Weaver<B>>,pub(crate) config: DFlashSpeculatorConfig,}impl<B: Backend> DFlashSpeculator<B> {- pub(crate) fn new(+ pub(crate) const fn new(model: DFlashDraft<B>,+ weaver: Option<Weaver<B>>,config: DFlashSpeculatorConfig,) -> Self {Self {model,+ weaver,config,}}