A high-performance inference engine for AI models
F32 delta net (#452)
uuuvnfbf3396a6296
diff --git a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rsindex 497d78629..aa8028ddc 100644--- a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs@@ -55,7 +55,7 @@ pub(crate) struct DeltaNetMixer<B: Backend> {a_log: Allocation<B>,dt_bias: Allocation<B>,norm_weight: Allocation<B>,- data_type: DataType,+ inner_data_type: DataType,}pub(crate) struct DeltaNetArguments<'a, B: Backend> {@@ -69,8 +69,10 @@ impl<B: Backend> DeltaNetMixer<B> {config: &DeltaNetConfig,model_dim: usize,parameter_tree: &ParameterTree<B>,- data_type: DataType,+ outer_data_type: DataType,) -> Result<(Self, Option<Allocation<B>>), DeltaNetMixerError<B>> {+ let inner_data_type = DataType::F32;+if config.kernel_size < 2 {return Err(DeltaNetMixerError::UnsupportedConfiguration(format!("kernel_size must be >= 2, got {}",@@ -99,54 +101,62 @@ impl<B: Backend> DeltaNetMixer<B> {// Load weightslet conv_tree = parameter_tree.subtree("conv")?;- let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(- model_dim,- [total_proj_dim],- false,- context,- data_type,- ¶meter_tree.subtree("in_proj")?,- )?;+ let (in_projection, in_projection_input_hadamard_factors) =+ <dyn Linear<B>>::new_extracting_input_hadamard_mixed_precision(+ model_dim,+ [total_proj_dim],+ false,+ context,+ outer_data_type,+ outer_data_type,+ inner_data_type,+ ¶meter_tree.subtree("in_proj")?,+ )?;- let out_projection = <dyn Linear<B>>::new(+ let out_projection = <dyn Linear<B>>::new_mixed_precision(value_dim,[model_dim],false,context,- data_type,+ outer_data_type,+ inner_data_type,+ outer_data_type,¶meter_tree.subtree("out_proj")?,)?;let conv_weight =- conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], data_type)?.read_allocation()?;+ conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], inner_data_type)?.read_allocation()?;let conv_bias = if has_bias {- Some(conv_tree.leaf("biases")?.validate(&[conv_dim], data_type)?.read_allocation()?)+ Some(conv_tree.leaf("biases")?.validate(&[conv_dim], inner_data_type)?.read_allocation()?)} else {None};- let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], data_type)?.read_allocation()?;- let dt_bias = parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], data_type)?.read_allocation()?;- let norm_weight =- parameter_tree.leaf("norm.scales")?.validate(&[config.value_head_dim], data_type)?.read_allocation()?;+ let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], inner_data_type)?.read_allocation()?;+ let dt_bias =+ parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], inner_data_type)?.read_allocation()?;+ let norm_weight = parameter_tree+ .leaf("norm.scales")?+ .validate(&[config.value_head_dim], inner_data_type)?+ .read_allocation()?;// Create kernels- let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, data_type, has_bias)+ let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, inner_data_type, has_bias).map_err(DeltaNetMixerError::BackendError)?;let delta_net_update =- <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, data_type, config.head_dim as u32)+ <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, inner_data_type, config.head_dim as u32).map_err(DeltaNetMixerError::BackendError)?;- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type).map_err(DeltaNetMixerError::BackendError)?;- let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, data_type, has_bias)+ let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, inner_data_type, has_bias).map_err(DeltaNetMixerError::BackendError)?;let prefill_prep =- <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, data_type, config.head_dim as u32)+ <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, inner_data_type, config.head_dim as u32).map_err(DeltaNetMixerError::BackendError)?;let delta_net_prefill =- <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, data_type, config.head_dim as u32)+ <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, inner_data_type, config.head_dim as u32).map_err(DeltaNetMixerError::BackendError)?;- let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, data_type)+ let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, inner_data_type).map_err(DeltaNetMixerError::BackendError)?;Ok((@@ -174,7 +184,7 @@ impl<B: Backend> DeltaNetMixer<B> {a_log,dt_bias,norm_weight,- data_type,+ inner_data_type,},in_projection_input_hadamard_factors,))@@ -211,7 +221,7 @@ impl<B: Backend> DeltaNetMixer<B> {let conv_dim = self.conv_dim;let total_proj_dim = self.total_proj_dim;let mut padded = encoder- .allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], self.data_type))?;+ .allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], self.inner_data_type))?;self.conv_pack.encode(&layer.conv_state,@@ -248,7 +258,8 @@ impl<B: Backend> DeltaNetMixer<B> {active_row_count: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let mut out = encoder.allocate_scratch(size_for_shape(&[active_row_count, self.value_dim], self.data_type))?;+ let mut out =+ encoder.allocate_scratch(size_for_shape(&[active_row_count, self.value_dim], self.inner_data_type))?;self.delta_net_update.encode(in_proj,&self.a_log,@@ -300,7 +311,8 @@ impl<B: Backend> DeltaNetMixer<B> {encoder,);- let mut out = encoder.allocate_scratch(size_for_shape(&[suffix_length, self.value_dim], self.data_type))?;+ let mut out =+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.value_dim], self.inner_data_type))?;self.delta_net_prefill.encode(&prep_q_norm,&prep_k_norm,diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rsindex 8c63187dc..527e18c7a 100644--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs@@ -1,6 +1,7 @@use std::cell::Cell;use crate::{+ DataType,array::size_for_shape,backends::common::{AllocationType, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},config::token_mixer::AnyTokenMixerConfig,@@ -219,10 +220,11 @@ impl<B: Backend> CacheLayers<B> {})},AnyTokenMixerConfig::DeltaNetConfig(c) => {+ let data_type = DataType::F32;let conv_shape = [c.conv_dim(), c.kernel_size.saturating_sub(1)];let ssm_shape = [c.num_heads, c.value_head_dim, c.head_dim];- let conv_bytes = size_for_shape(&conv_shape, model_shape.data_type);- let ssm_bytes = size_for_shape(&ssm_shape, model_shape.data_type);+ let conv_bytes = size_for_shape(&conv_shape, data_type);+ let ssm_bytes = size_for_shape(&ssm_shape, data_type);CacheLayer::DeltaNet(DeltaNetLayer {conv_state: context@@ -233,7 +235,7 @@ impl<B: Backend> CacheLayers<B> {.create_allocation(ssm_bytes, AllocationType::Global).expect("Failed to create delta net ssm allocation"),ssm_shape,- data_type: model_shape.data_type,+ data_type,})},};