A high-performance inference engine for AI models
F32 short conv (#493)
uuuvnaa508869b685
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/short_conv/short_conv.rs b/crates/backend-uzu/src/backends/cpu/kernel/short_conv/short_conv.rsindex 1e0453f10..684f7a94a 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/short_conv/short_conv.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/short_conv/short_conv.rs@@ -43,11 +43,12 @@ pub fn short_conv_pack<T: ArrayElement + Float>(#[kernel(ShortConvPrefill)]#[variants(T, f32, f16, bf16)]-pub fn short_conv_prefill<T: ArrayElement + Float>(+#[variants(WeightT, f32)]+pub fn short_conv_prefill<T: ArrayElement + Float, WeightT: ArrayElement + Float>(padded: *const T,in_proj: *const T,- w: *const T,- #[optional(has_bias)] b: Option<*const T>,+ w: *const WeightT,+ #[optional(has_bias)] b: Option<*const WeightT>,out: *mut T,state_out: *mut T,suffix_len: u32,@@ -112,10 +113,11 @@ pub fn short_conv_prefill<T: ArrayElement + Float>(#[kernel(ShortConvDecode)]#[variants(T, f32, f16, bf16)]-pub fn short_conv_decode<T: ArrayElement + Float>(+#[variants(WeightT, f32)]+pub fn short_conv_decode<T: ArrayElement + Float, WeightT: ArrayElement + Float>(in_proj: *const T,- w: *const T,- #[optional(has_bias)] b: Option<*const T>,+ w: *const WeightT,+ #[optional(has_bias)] b: Option<*const WeightT>,#[optional(!state_in_place)] state: Option<*const T>,out: *mut T,next_state: *mut T,@@ -182,10 +184,11 @@ pub fn short_conv_decode<T: ArrayElement + Float>(#[kernel(ShortConvTrie)]#[variants(T, f32, f16, bf16)]-pub fn short_conv_trie<T: ArrayElement + Float>(+#[variants(WeightT, f32)]+pub fn short_conv_trie<T: ArrayElement + Float, WeightT: ArrayElement + Float>(in_proj: *const T,- w: *const T,- #[optional(has_bias)] b: Option<*const T>,+ w: *const WeightT,+ #[optional(has_bias)] b: Option<*const WeightT>,base_state: *const T,parents: *const i32,out: *mut T,diff --git a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metalindex d4f1eb964..b8b61860a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -32,13 +32,14 @@ PUBLIC KERNEL(ShortConvPack)(}}-template <typename T>+template <typename T, typename WeightT>VARIANTS(T, float, half, bfloat)+VARIANTS(WeightT, float)PUBLIC KERNEL(ShortConvPrefill)(device const T* padded,device const T* in_proj,- device const T* w,- device const T* b OPTIONAL(has_bias),+ device const WeightT* w,+ device const WeightT* b OPTIONAL(has_bias),device T* out,device T* state_out,constant const uint& suffix_len,@@ -51,7 +52,7 @@ PUBLIC KERNEL(ShortConvPrefill)(const uint channel_idx AXIS(model_dim, 256)) {const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;- const device T* w_row = w + channel_idx * kernel_size;+ const device WeightT* w_row = w + channel_idx * kernel_size;// Threads [0..suffix_len-1]: Compute outputsif (token_idx < suffix_len) {@@ -93,12 +94,13 @@ PUBLIC KERNEL(ShortConvPrefill)(}}-template <typename T>+template <typename T, typename WeightT>VARIANTS(T, float, half, bfloat)+VARIANTS(WeightT, float)PUBLIC KERNEL(ShortConvDecode)(device const T* in_proj,- device const T* w,- device const T* b OPTIONAL(has_bias),+ device const WeightT* w,+ device const WeightT* b OPTIONAL(has_bias),device const T* state OPTIONAL(!state_in_place),device T* out,device T* next_state,@@ -118,7 +120,7 @@ PUBLIC KERNEL(ShortConvDecode)(const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;const uint state_offset = channel_idx * state_stride;- const device T* w_row = w + channel_idx * kernel_size;+ const device WeightT* w_row = w + channel_idx * kernel_size;uint in_proj_idx = token_idx * in_proj_stride + channel_idx;float pre_conv_gate = float(in_proj[in_proj_idx]);@@ -152,12 +154,13 @@ PUBLIC KERNEL(ShortConvDecode)(}}-template <typename T>+template <typename T, typename WeightT>VARIANTS(T, float, half, bfloat)+VARIANTS(WeightT, float)PUBLIC KERNEL(ShortConvTrie)(device const T* in_proj,- device const T* w,- device const T* b OPTIONAL(has_bias),+ device const WeightT* w,+ device const WeightT* b OPTIONAL(has_bias),device const T* base_state,device const int* parents,device T* out,@@ -171,7 +174,7 @@ PUBLIC KERNEL(ShortConvTrie)(const uint channel_idx AXIS(model_dim, 256)) {const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;- const device T* w_row = w + channel_idx * kernel_size;+ const device WeightT* w_row = w + channel_idx * kernel_size;const uint base_state_offset = channel_idx * state_stride;for (uint node = 0; node < suffix_len; ++node) {diff --git a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rsindex eb93ea55d..46ac7f108 100644--- a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs@@ -82,12 +82,13 @@ impl<B: Backend> ShortConvMixer<B> {¶meter_tree.subtree("out_projection")?,)?;+ let conv_data_type = DataType::F32;let conv_weight = conv_tree.leaf("weights")?- .validate(&[model_dim, short_conv_config.kernel_size], data_type)?+ .validate(&[model_dim, short_conv_config.kernel_size], conv_data_type)?.read_allocation()?;let conv_bias = if short_conv_config.conv_config.has_biases {- Some(conv_tree.leaf("biases")?.validate(&[model_dim], data_type)?.read_allocation()?)+ Some(conv_tree.leaf("biases")?.validate(&[model_dim], conv_data_type)?.read_allocation()?)} else {None};@@ -95,12 +96,15 @@ impl<B: Backend> ShortConvMixer<B> {let has_bias = short_conv_config.conv_config.has_biases;let short_conv_pack = <B::Kernels as Kernels>::ShortConvPackKernel::new(context, data_type).map_err(ShortConvMixerError::BackendError)?;- let short_conv_prefill = <B::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, has_bias)- .map_err(ShortConvMixerError::BackendError)?;- let short_conv_decode = <B::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, has_bias, true)- .map_err(ShortConvMixerError::BackendError)?;- let short_conv_trie = <B::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)- .map_err(ShortConvMixerError::BackendError)?;+ let short_conv_prefill =+ <B::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, conv_data_type, has_bias)+ .map_err(ShortConvMixerError::BackendError)?;+ let short_conv_decode =+ <B::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, conv_data_type, has_bias, true)+ .map_err(ShortConvMixerError::BackendError)?;+ let short_conv_trie =+ <B::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, conv_data_type, has_bias)+ .map_err(ShortConvMixerError::BackendError)?;Ok((Self {diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rsindex 544e56cc7..11240d29a 100644--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs@@ -15,8 +15,8 @@ use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {in_proj: Box<[T]>,- w: Box<[T]>,- b: Option<Box<[T]>>,+ w: Box<[f32]>,+ b: Option<Box<[f32]>>,state: Box<[T]>,suffix_len: u32,kernel_size: u32,@@ -35,6 +35,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvDecodeKernel::new(&context,T::data_type(),+ DataType::F32,has_bias,state_in_place,)@@ -104,11 +105,11 @@ fn get_test_data_basic<T: ArrayElement + Float>(}// w[channel * kernel_size + tap]- let mut w = vec![T::zero(); model_dim * kernel_size];+ let mut w = vec![0.0f32; model_dim * kernel_size];for ch in 0..model_dim {for tap in 0..kernel_size {let val = 0.1 * (tap as f32) - 0.01 * (ch as f32) + 0.5;- w[ch * kernel_size + tap] = T::from(val).unwrap();+ w[ch * kernel_size + tap] = val;}}@@ -122,9 +123,9 @@ fn get_test_data_basic<T: ArrayElement + Float>(}let b = if has_bias {- let mut bias = vec![T::zero(); model_dim];+ let mut bias = vec![0.0f32; model_dim];for ch in 0..model_dim {- bias[ch] = T::from(0.01 * (ch as f32) + 0.1).unwrap();+ bias[ch] = 0.01 * (ch as f32) + 0.1;}Some(bias.into_boxed_slice())} else {@@ -166,11 +167,11 @@ fn get_test_data_edge<T: ArrayElement + Float>(in_proj[2 * model_dim + ch] = T::from(x_in).unwrap();}- let mut w = vec![T::zero(); model_dim * kernel_size];+ let mut w = vec![0.0f32; model_dim * kernel_size];for ch in 0..model_dim {for tap in 0..kernel_size {let val = 0.25 * (tap as f32) + 0.1;- w[ch * kernel_size + tap] = T::from(val).unwrap();+ w[ch * kernel_size + tap] = val;}}@@ -183,9 +184,9 @@ fn get_test_data_edge<T: ArrayElement + Float>(}let b = if has_bias {- let mut bias = vec![T::zero(); model_dim];+ let mut bias = vec![0.0f32; model_dim];for ch in 0..model_dim {- bias[ch] = T::from(0.005 * (ch as f32) + 0.01).unwrap();+ bias[ch] = 0.005 * (ch as f32) + 0.01;}Some(bias.into_boxed_slice())} else {diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rsindex 955e46039..0cd00ad97 100644--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs@@ -16,8 +16,8 @@ use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {padded: Box<[T]>,in_proj: Box<[T]>,- w: Box<[T]>,- b: Option<Box<[T]>>,+ w: Box<[f32]>,+ b: Option<Box<[f32]>>,suffix_len: u32,kernel_size: u32,in_proj_stride: u32,@@ -29,8 +29,13 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let context = B::Context::new().expect("Failed to create Context");let has_bias = input.b.is_some();- let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvPrefillKernel::new(&context, T::data_type(), has_bias)- .expect("Failed to create ShortConvPrefillKernel");+ let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvPrefillKernel::new(+ &context,+ T::data_type(),+ DataType::F32,+ has_bias,+ )+ .expect("Failed to create ShortConvPrefillKernel");let padded_array = context.create_array_from(&[input.padded.len()], &input.padded);let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);@@ -93,18 +98,18 @@ fn get_test_data_basic<T: ArrayElement + Float>(}// w[channel * kernel_size + tap]- let mut w = vec![T::zero(); model_dim * kernel_size];+ let mut w = vec![0.0f32; model_dim * kernel_size];for ch in 0..model_dim {for tap in 0..kernel_size {let val = 0.1 * (tap as f32) - 0.01 * (ch as f32) + 0.5;- w[ch * kernel_size + tap] = T::from(val).unwrap();+ w[ch * kernel_size + tap] = val;}}let b = if has_bias {- let mut bias = vec![T::zero(); model_dim];+ let mut bias = vec![0.0f32; model_dim];for ch in 0..model_dim {- bias[ch] = T::from(0.01 * (ch as f32) + 0.1).unwrap();+ bias[ch] = 0.01 * (ch as f32) + 0.1;}Some(bias.into_boxed_slice())} else {@@ -153,18 +158,18 @@ fn get_test_data_edge<T: ArrayElement + Float>(}}- let mut w = vec![T::zero(); model_dim * kernel_size];+ let mut w = vec![0.0f32; model_dim * kernel_size];for ch in 0..model_dim {for tap in 0..kernel_size {let val = 0.25 * (tap as f32) + 0.1;- w[ch * kernel_size + tap] = T::from(val).unwrap();+ w[ch * kernel_size + tap] = val;}}let b = if has_bias {- let mut bias = vec![T::zero(); model_dim];+ let mut bias = vec![0.0f32; model_dim];for ch in 0..model_dim {- bias[ch] = T::from(0.005 * (ch as f32) + 0.01).unwrap();+ bias[ch] = 0.005 * (ch as f32) + 0.01;}Some(bias.into_boxed_slice())} else {diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rsindex 8aca41000..ab3f600f4 100644--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs@@ -15,8 +15,8 @@ use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {in_proj: Box<[T]>,- w: Box<[T]>,- b: Option<Box<[T]>>,+ w: Box<[f32]>,+ b: Option<Box<[f32]>>,base_state: Box<[T]>,parents: Box<[i32]>,suffix_len: u32,@@ -30,8 +30,13 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let context = B::Context::new().expect("Failed to create Context");let has_bias = input.b.is_some();- let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(&context, T::data_type(), has_bias)- .expect("Failed to create ShortConvTrieKernel");+ let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(+ &context,+ T::data_type(),+ DataType::F32,+ has_bias,+ )+ .expect("Failed to create ShortConvTrieKernel");let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);let w_array = context.create_array_from(&[input.w.len()], &input.w);@@ -99,11 +104,11 @@ fn get_test_data_basic<T: ArrayElement + Float>(}}- let mut w = vec![T::zero(); model_dim * kernel_size];+ let mut w = vec![0.0f32; model_dim * kernel_size];for ch in 0..model_dim {for tap in 0..kernel_size {let val = 0.1 * (tap as f32) - 0.01 * (ch as f32) + 0.5;- w[ch * kernel_size + tap] = T::from(val).unwrap();+ w[ch * kernel_size + tap] = val;}}@@ -123,9 +128,9 @@ fn get_test_data_basic<T: ArrayElement + Float>(}let b = if has_bias {- let mut bias = vec![T::zero(); model_dim];+ let mut bias = vec![0.0f32; model_dim];for ch in 0..model_dim {- bias[ch] = T::from(0.01 * (ch as f32) + 0.1).unwrap();+ bias[ch] = 0.01 * (ch as f32) + 0.1;}Some(bias.into_boxed_slice())} else {@@ -171,11 +176,11 @@ fn get_test_data_branching<T: ArrayElement + Float>(}}- let mut w = vec![T::zero(); model_dim * kernel_size];+ let mut w = vec![0.0f32; model_dim * kernel_size];for ch in 0..model_dim {for tap in 0..kernel_size {let val = 0.15 * (tap as f32) - 0.02 * (ch as f32) + 0.4;- w[ch * kernel_size + tap] = T::from(val).unwrap();+ w[ch * kernel_size + tap] = val;}}@@ -191,9 +196,9 @@ fn get_test_data_branching<T: ArrayElement + Float>(let parents = vec![-1i32, 0, 0];let b = if has_bias {- let mut bias = vec![T::zero(); model_dim];+ let mut bias = vec![0.0f32; model_dim];for ch in 0..model_dim {- bias[ch] = T::from(0.02 * (ch as f32) + 0.05).unwrap();+ bias[ch] = 0.02 * (ch as f32) + 0.05;}Some(bias.into_boxed_slice())} else {@@ -237,11 +242,11 @@ fn get_test_data_edge<T: ArrayElement + Float>(in_proj[2 * model_dim + ch] = T::from(x_in).unwrap();}- let mut w = vec![T::zero(); model_dim * kernel_size];+ let mut w = vec![0.0f32; model_dim * kernel_size];for ch in 0..model_dim {for tap in 0..kernel_size {let val = 0.25 * (tap as f32) + 0.1;- w[ch * kernel_size + tap] = T::from(val).unwrap();+ w[ch * kernel_size + tap] = val;}}@@ -256,9 +261,9 @@ fn get_test_data_edge<T: ArrayElement + Float>(let parents = vec![-1i32];let b = if has_bias {- let mut bias = vec![T::zero(); model_dim];+ let mut bias = vec![0.0f32; model_dim];for ch in 0..model_dim {- bias[ch] = T::from(0.005 * (ch as f32) + 0.01).unwrap();+ bias[ch] = 0.005 * (ch as f32) + 0.01;}Some(bias.into_boxed_slice())} else {