Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

F32 delta net (#452)

uuuvnfbf3396a6296
diff --git a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
index 497d78629..aa8028ddc 100644
--- a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
@@ -55,7 +55,7 @@ pub(crate) struct DeltaNetMixer<B: Backend> {
a_log: Allocation<B>,
dt_bias: Allocation<B>,
norm_weight: Allocation<B>,
- data_type: DataType,
+ inner_data_type: DataType,
}
pub(crate) struct DeltaNetArguments<'a, B: Backend> {
@@ -69,8 +69,10 @@ impl<B: Backend> DeltaNetMixer<B> {
config: &DeltaNetConfig,
model_dim: usize,
parameter_tree: &ParameterTree<B>,
- data_type: DataType,
+ outer_data_type: DataType,
) -> Result<(Self, Option<Allocation<B>>), DeltaNetMixerError<B>> {
+ let inner_data_type = DataType::F32;
+
if config.kernel_size < 2 {
return Err(DeltaNetMixerError::UnsupportedConfiguration(format!(
"kernel_size must be >= 2, got {}",
@@ -99,54 +101,62 @@ impl<B: Backend> DeltaNetMixer<B> {
// Load weights
let conv_tree = parameter_tree.subtree("conv")?;
- let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
- model_dim,
- [total_proj_dim],
- false,
- context,
- data_type,
- &parameter_tree.subtree("in_proj")?,
- )?;
+ let (in_projection, in_projection_input_hadamard_factors) =
+ <dyn Linear<B>>::new_extracting_input_hadamard_mixed_precision(
+ model_dim,
+ [total_proj_dim],
+ false,
+ context,
+ outer_data_type,
+ outer_data_type,
+ inner_data_type,
+ &parameter_tree.subtree("in_proj")?,
+ )?;
- let out_projection = <dyn Linear<B>>::new(
+ let out_projection = <dyn Linear<B>>::new_mixed_precision(
value_dim,
[model_dim],
false,
context,
- data_type,
+ outer_data_type,
+ inner_data_type,
+ outer_data_type,
&parameter_tree.subtree("out_proj")?,
)?;
let conv_weight =
- conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], data_type)?.read_allocation()?;
+ conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], inner_data_type)?.read_allocation()?;
let conv_bias = if has_bias {
- Some(conv_tree.leaf("biases")?.validate(&[conv_dim], data_type)?.read_allocation()?)
+ Some(conv_tree.leaf("biases")?.validate(&[conv_dim], inner_data_type)?.read_allocation()?)
} else {
None
};
- let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], data_type)?.read_allocation()?;
- let dt_bias = parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], data_type)?.read_allocation()?;
- let norm_weight =
- parameter_tree.leaf("norm.scales")?.validate(&[config.value_head_dim], data_type)?.read_allocation()?;
+ let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], inner_data_type)?.read_allocation()?;
+ let dt_bias =
+ parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], inner_data_type)?.read_allocation()?;
+ let norm_weight = parameter_tree
+ .leaf("norm.scales")?
+ .validate(&[config.value_head_dim], inner_data_type)?
+ .read_allocation()?;
// Create kernels
- let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, data_type, has_bias)
+ let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, inner_data_type, has_bias)
.map_err(DeltaNetMixerError::BackendError)?;
let delta_net_update =
- <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, data_type, config.head_dim as u32)
+ <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, inner_data_type, config.head_dim as u32)
.map_err(DeltaNetMixerError::BackendError)?;
- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)
+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type)
.map_err(DeltaNetMixerError::BackendError)?;
- let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, data_type, has_bias)
+ let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, inner_data_type, has_bias)
.map_err(DeltaNetMixerError::BackendError)?;
let prefill_prep =
- <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, data_type, config.head_dim as u32)
+ <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, inner_data_type, config.head_dim as u32)
.map_err(DeltaNetMixerError::BackendError)?;
let delta_net_prefill =
- <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, data_type, config.head_dim as u32)
+ <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, inner_data_type, config.head_dim as u32)
.map_err(DeltaNetMixerError::BackendError)?;
- let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, data_type)
+ let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, inner_data_type)
.map_err(DeltaNetMixerError::BackendError)?;
Ok((
@@ -174,7 +184,7 @@ impl<B: Backend> DeltaNetMixer<B> {
a_log,
dt_bias,
norm_weight,
- data_type,
+ inner_data_type,
},
in_projection_input_hadamard_factors,
))
@@ -211,7 +221,7 @@ impl<B: Backend> DeltaNetMixer<B> {
let conv_dim = self.conv_dim;
let total_proj_dim = self.total_proj_dim;
let mut padded = encoder
- .allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], self.data_type))?;
+ .allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], self.inner_data_type))?;
self.conv_pack.encode(
&layer.conv_state,
@@ -248,7 +258,8 @@ impl<B: Backend> DeltaNetMixer<B> {
active_row_count: usize,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut out = encoder.allocate_scratch(size_for_shape(&[active_row_count, self.value_dim], self.data_type))?;
+ let mut out =
+ encoder.allocate_scratch(size_for_shape(&[active_row_count, self.value_dim], self.inner_data_type))?;
self.delta_net_update.encode(
in_proj,
&self.a_log,
@@ -300,7 +311,8 @@ impl<B: Backend> DeltaNetMixer<B> {
encoder,
);
- let mut out = encoder.allocate_scratch(size_for_shape(&[suffix_length, self.value_dim], self.data_type))?;
+ let mut out =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.value_dim], self.inner_data_type))?;
self.delta_net_prefill.encode(
&prep_q_norm,
&prep_k_norm,
diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rs
index 8c63187dc..527e18c7a 100644
--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs
+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs
@@ -1,6 +1,7 @@
use std::cell::Cell;
use crate::{
+ DataType,
array::size_for_shape,
backends::common::{AllocationType, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
config::token_mixer::AnyTokenMixerConfig,
@@ -219,10 +220,11 @@ impl<B: Backend> CacheLayers<B> {
})
},
AnyTokenMixerConfig::DeltaNetConfig(c) => {
+ let data_type = DataType::F32;
let conv_shape = [c.conv_dim(), c.kernel_size.saturating_sub(1)];
let ssm_shape = [c.num_heads, c.value_head_dim, c.head_dim];
- let conv_bytes = size_for_shape(&conv_shape, model_shape.data_type);
- let ssm_bytes = size_for_shape(&ssm_shape, model_shape.data_type);
+ let conv_bytes = size_for_shape(&conv_shape, data_type);
+ let ssm_bytes = size_for_shape(&ssm_shape, data_type);
CacheLayer::DeltaNet(DeltaNetLayer {
conv_state: context
@@ -233,7 +235,7 @@ impl<B: Backend> CacheLayers<B> {
.create_allocation(ssm_bytes, AllocationType::Global)
.expect("Failed to create delta net ssm allocation"),
ssm_shape,
- data_type: model_shape.data_type,
+ data_type,
})
},
};