Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

F32 short conv (#493)

uuuvnaa508869b685
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/short_conv/short_conv.rs b/crates/backend-uzu/src/backends/cpu/kernel/short_conv/short_conv.rs
index 1e0453f10..684f7a94a 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/short_conv/short_conv.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/short_conv/short_conv.rs
@@ -43,11 +43,12 @@ pub fn short_conv_pack<T: ArrayElement + Float>(
#[kernel(ShortConvPrefill)]
#[variants(T, f32, f16, bf16)]
-pub fn short_conv_prefill<T: ArrayElement + Float>(
+#[variants(WeightT, f32)]
+pub fn short_conv_prefill<T: ArrayElement + Float, WeightT: ArrayElement + Float>(
padded: *const T,
in_proj: *const T,
- w: *const T,
- #[optional(has_bias)] b: Option<*const T>,
+ w: *const WeightT,
+ #[optional(has_bias)] b: Option<*const WeightT>,
out: *mut T,
state_out: *mut T,
suffix_len: u32,
@@ -112,10 +113,11 @@ pub fn short_conv_prefill<T: ArrayElement + Float>(
#[kernel(ShortConvDecode)]
#[variants(T, f32, f16, bf16)]
-pub fn short_conv_decode<T: ArrayElement + Float>(
+#[variants(WeightT, f32)]
+pub fn short_conv_decode<T: ArrayElement + Float, WeightT: ArrayElement + Float>(
in_proj: *const T,
- w: *const T,
- #[optional(has_bias)] b: Option<*const T>,
+ w: *const WeightT,
+ #[optional(has_bias)] b: Option<*const WeightT>,
#[optional(!state_in_place)] state: Option<*const T>,
out: *mut T,
next_state: *mut T,
@@ -182,10 +184,11 @@ pub fn short_conv_decode<T: ArrayElement + Float>(
#[kernel(ShortConvTrie)]
#[variants(T, f32, f16, bf16)]
-pub fn short_conv_trie<T: ArrayElement + Float>(
+#[variants(WeightT, f32)]
+pub fn short_conv_trie<T: ArrayElement + Float, WeightT: ArrayElement + Float>(
in_proj: *const T,
- w: *const T,
- #[optional(has_bias)] b: Option<*const T>,
+ w: *const WeightT,
+ #[optional(has_bias)] b: Option<*const WeightT>,
base_state: *const T,
parents: *const i32,
out: *mut T,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal
index d4f1eb964..b8b61860a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -32,13 +32,14 @@ PUBLIC KERNEL(ShortConvPack)(
}
}
-template <typename T>
+template <typename T, typename WeightT>
VARIANTS(T, float, half, bfloat)
+VARIANTS(WeightT, float)
PUBLIC KERNEL(ShortConvPrefill)(
device const T* padded,
device const T* in_proj,
- device const T* w,
- device const T* b OPTIONAL(has_bias),
+ device const WeightT* w,
+ device const WeightT* b OPTIONAL(has_bias),
device T* out,
device T* state_out,
constant const uint& suffix_len,
@@ -51,7 +52,7 @@ PUBLIC KERNEL(ShortConvPrefill)(
const uint channel_idx AXIS(model_dim, 256)
) {
const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;
- const device T* w_row = w + channel_idx * kernel_size;
+ const device WeightT* w_row = w + channel_idx * kernel_size;
// Threads [0..suffix_len-1]: Compute outputs
if (token_idx < suffix_len) {
@@ -93,12 +94,13 @@ PUBLIC KERNEL(ShortConvPrefill)(
}
}
-template <typename T>
+template <typename T, typename WeightT>
VARIANTS(T, float, half, bfloat)
+VARIANTS(WeightT, float)
PUBLIC KERNEL(ShortConvDecode)(
device const T* in_proj,
- device const T* w,
- device const T* b OPTIONAL(has_bias),
+ device const WeightT* w,
+ device const WeightT* b OPTIONAL(has_bias),
device const T* state OPTIONAL(!state_in_place),
device T* out,
device T* next_state,
@@ -118,7 +120,7 @@ PUBLIC KERNEL(ShortConvDecode)(
const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;
const uint state_offset = channel_idx * state_stride;
- const device T* w_row = w + channel_idx * kernel_size;
+ const device WeightT* w_row = w + channel_idx * kernel_size;
uint in_proj_idx = token_idx * in_proj_stride + channel_idx;
float pre_conv_gate = float(in_proj[in_proj_idx]);
@@ -152,12 +154,13 @@ PUBLIC KERNEL(ShortConvDecode)(
}
}
-template <typename T>
+template <typename T, typename WeightT>
VARIANTS(T, float, half, bfloat)
+VARIANTS(WeightT, float)
PUBLIC KERNEL(ShortConvTrie)(
device const T* in_proj,
- device const T* w,
- device const T* b OPTIONAL(has_bias),
+ device const WeightT* w,
+ device const WeightT* b OPTIONAL(has_bias),
device const T* base_state,
device const int* parents,
device T* out,
@@ -171,7 +174,7 @@ PUBLIC KERNEL(ShortConvTrie)(
const uint channel_idx AXIS(model_dim, 256)
) {
const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;
- const device T* w_row = w + channel_idx * kernel_size;
+ const device WeightT* w_row = w + channel_idx * kernel_size;
const uint base_state_offset = channel_idx * state_stride;
for (uint node = 0; node < suffix_len; ++node) {
diff --git a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs
index eb93ea55d..46ac7f108 100644
--- a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs
@@ -82,12 +82,13 @@ impl<B: Backend> ShortConvMixer<B> {
&parameter_tree.subtree("out_projection")?,
)?;
+ let conv_data_type = DataType::F32;
let conv_weight = conv_tree
.leaf("weights")?
- .validate(&[model_dim, short_conv_config.kernel_size], data_type)?
+ .validate(&[model_dim, short_conv_config.kernel_size], conv_data_type)?
.read_allocation()?;
let conv_bias = if short_conv_config.conv_config.has_biases {
- Some(conv_tree.leaf("biases")?.validate(&[model_dim], data_type)?.read_allocation()?)
+ Some(conv_tree.leaf("biases")?.validate(&[model_dim], conv_data_type)?.read_allocation()?)
} else {
None
};
@@ -95,12 +96,15 @@ impl<B: Backend> ShortConvMixer<B> {
let has_bias = short_conv_config.conv_config.has_biases;
let short_conv_pack = <B::Kernels as Kernels>::ShortConvPackKernel::new(context, data_type)
.map_err(ShortConvMixerError::BackendError)?;
- let short_conv_prefill = <B::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, has_bias)
- .map_err(ShortConvMixerError::BackendError)?;
- let short_conv_decode = <B::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, has_bias, true)
- .map_err(ShortConvMixerError::BackendError)?;
- let short_conv_trie = <B::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)
- .map_err(ShortConvMixerError::BackendError)?;
+ let short_conv_prefill =
+ <B::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, conv_data_type, has_bias)
+ .map_err(ShortConvMixerError::BackendError)?;
+ let short_conv_decode =
+ <B::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, conv_data_type, has_bias, true)
+ .map_err(ShortConvMixerError::BackendError)?;
+ let short_conv_trie =
+ <B::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, conv_data_type, has_bias)
+ .map_err(ShortConvMixerError::BackendError)?;
Ok((
Self {
diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs
index 544e56cc7..11240d29a 100644
--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs
@@ -15,8 +15,8 @@ use crate::{common::assert::assert_eq_float, uzu_test};
struct Input<T: ArrayElement + Float> {
in_proj: Box<[T]>,
- w: Box<[T]>,
- b: Option<Box<[T]>>,
+ w: Box<[f32]>,
+ b: Option<Box<[f32]>>,
state: Box<[T]>,
suffix_len: u32,
kernel_size: u32,
@@ -35,6 +35,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvDecodeKernel::new(
&context,
T::data_type(),
+ DataType::F32,
has_bias,
state_in_place,
)
@@ -104,11 +105,11 @@ fn get_test_data_basic<T: ArrayElement + Float>(
}
// w[channel * kernel_size + tap]
- let mut w = vec![T::zero(); model_dim * kernel_size];
+ let mut w = vec![0.0f32; model_dim * kernel_size];
for ch in 0..model_dim {
for tap in 0..kernel_size {
let val = 0.1 * (tap as f32) - 0.01 * (ch as f32) + 0.5;
- w[ch * kernel_size + tap] = T::from(val).unwrap();
+ w[ch * kernel_size + tap] = val;
}
}
@@ -122,9 +123,9 @@ fn get_test_data_basic<T: ArrayElement + Float>(
}
let b = if has_bias {
- let mut bias = vec![T::zero(); model_dim];
+ let mut bias = vec![0.0f32; model_dim];
for ch in 0..model_dim {
- bias[ch] = T::from(0.01 * (ch as f32) + 0.1).unwrap();
+ bias[ch] = 0.01 * (ch as f32) + 0.1;
}
Some(bias.into_boxed_slice())
} else {
@@ -166,11 +167,11 @@ fn get_test_data_edge<T: ArrayElement + Float>(
in_proj[2 * model_dim + ch] = T::from(x_in).unwrap();
}
- let mut w = vec![T::zero(); model_dim * kernel_size];
+ let mut w = vec![0.0f32; model_dim * kernel_size];
for ch in 0..model_dim {
for tap in 0..kernel_size {
let val = 0.25 * (tap as f32) + 0.1;
- w[ch * kernel_size + tap] = T::from(val).unwrap();
+ w[ch * kernel_size + tap] = val;
}
}
@@ -183,9 +184,9 @@ fn get_test_data_edge<T: ArrayElement + Float>(
}
let b = if has_bias {
- let mut bias = vec![T::zero(); model_dim];
+ let mut bias = vec![0.0f32; model_dim];
for ch in 0..model_dim {
- bias[ch] = T::from(0.005 * (ch as f32) + 0.01).unwrap();
+ bias[ch] = 0.005 * (ch as f32) + 0.01;
}
Some(bias.into_boxed_slice())
} else {
diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs
index 955e46039..0cd00ad97 100644
--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs
@@ -16,8 +16,8 @@ use crate::{common::assert::assert_eq_float, uzu_test};
struct Input<T: ArrayElement + Float> {
padded: Box<[T]>,
in_proj: Box<[T]>,
- w: Box<[T]>,
- b: Option<Box<[T]>>,
+ w: Box<[f32]>,
+ b: Option<Box<[f32]>>,
suffix_len: u32,
kernel_size: u32,
in_proj_stride: u32,
@@ -29,8 +29,13 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let context = B::Context::new().expect("Failed to create Context");
let has_bias = input.b.is_some();
- let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvPrefillKernel::new(&context, T::data_type(), has_bias)
- .expect("Failed to create ShortConvPrefillKernel");
+ let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvPrefillKernel::new(
+ &context,
+ T::data_type(),
+ DataType::F32,
+ has_bias,
+ )
+ .expect("Failed to create ShortConvPrefillKernel");
let padded_array = context.create_array_from(&[input.padded.len()], &input.padded);
let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);
@@ -93,18 +98,18 @@ fn get_test_data_basic<T: ArrayElement + Float>(
}
// w[channel * kernel_size + tap]
- let mut w = vec![T::zero(); model_dim * kernel_size];
+ let mut w = vec![0.0f32; model_dim * kernel_size];
for ch in 0..model_dim {
for tap in 0..kernel_size {
let val = 0.1 * (tap as f32) - 0.01 * (ch as f32) + 0.5;
- w[ch * kernel_size + tap] = T::from(val).unwrap();
+ w[ch * kernel_size + tap] = val;
}
}
let b = if has_bias {
- let mut bias = vec![T::zero(); model_dim];
+ let mut bias = vec![0.0f32; model_dim];
for ch in 0..model_dim {
- bias[ch] = T::from(0.01 * (ch as f32) + 0.1).unwrap();
+ bias[ch] = 0.01 * (ch as f32) + 0.1;
}
Some(bias.into_boxed_slice())
} else {
@@ -153,18 +158,18 @@ fn get_test_data_edge<T: ArrayElement + Float>(
}
}
- let mut w = vec![T::zero(); model_dim * kernel_size];
+ let mut w = vec![0.0f32; model_dim * kernel_size];
for ch in 0..model_dim {
for tap in 0..kernel_size {
let val = 0.25 * (tap as f32) + 0.1;
- w[ch * kernel_size + tap] = T::from(val).unwrap();
+ w[ch * kernel_size + tap] = val;
}
}
let b = if has_bias {
- let mut bias = vec![T::zero(); model_dim];
+ let mut bias = vec![0.0f32; model_dim];
for ch in 0..model_dim {
- bias[ch] = T::from(0.005 * (ch as f32) + 0.01).unwrap();
+ bias[ch] = 0.005 * (ch as f32) + 0.01;
}
Some(bias.into_boxed_slice())
} else {
diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs
index 8aca41000..ab3f600f4 100644
--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs
@@ -15,8 +15,8 @@ use crate::{common::assert::assert_eq_float, uzu_test};
struct Input<T: ArrayElement + Float> {
in_proj: Box<[T]>,
- w: Box<[T]>,
- b: Option<Box<[T]>>,
+ w: Box<[f32]>,
+ b: Option<Box<[f32]>>,
base_state: Box<[T]>,
parents: Box<[i32]>,
suffix_len: u32,
@@ -30,8 +30,13 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let context = B::Context::new().expect("Failed to create Context");
let has_bias = input.b.is_some();
- let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(&context, T::data_type(), has_bias)
- .expect("Failed to create ShortConvTrieKernel");
+ let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(
+ &context,
+ T::data_type(),
+ DataType::F32,
+ has_bias,
+ )
+ .expect("Failed to create ShortConvTrieKernel");
let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);
let w_array = context.create_array_from(&[input.w.len()], &input.w);
@@ -99,11 +104,11 @@ fn get_test_data_basic<T: ArrayElement + Float>(
}
}
- let mut w = vec![T::zero(); model_dim * kernel_size];
+ let mut w = vec![0.0f32; model_dim * kernel_size];
for ch in 0..model_dim {
for tap in 0..kernel_size {
let val = 0.1 * (tap as f32) - 0.01 * (ch as f32) + 0.5;
- w[ch * kernel_size + tap] = T::from(val).unwrap();
+ w[ch * kernel_size + tap] = val;
}
}
@@ -123,9 +128,9 @@ fn get_test_data_basic<T: ArrayElement + Float>(
}
let b = if has_bias {
- let mut bias = vec![T::zero(); model_dim];
+ let mut bias = vec![0.0f32; model_dim];
for ch in 0..model_dim {
- bias[ch] = T::from(0.01 * (ch as f32) + 0.1).unwrap();
+ bias[ch] = 0.01 * (ch as f32) + 0.1;
}
Some(bias.into_boxed_slice())
} else {
@@ -171,11 +176,11 @@ fn get_test_data_branching<T: ArrayElement + Float>(
}
}
- let mut w = vec![T::zero(); model_dim * kernel_size];
+ let mut w = vec![0.0f32; model_dim * kernel_size];
for ch in 0..model_dim {
for tap in 0..kernel_size {
let val = 0.15 * (tap as f32) - 0.02 * (ch as f32) + 0.4;
- w[ch * kernel_size + tap] = T::from(val).unwrap();
+ w[ch * kernel_size + tap] = val;
}
}
@@ -191,9 +196,9 @@ fn get_test_data_branching<T: ArrayElement + Float>(
let parents = vec![-1i32, 0, 0];
let b = if has_bias {
- let mut bias = vec![T::zero(); model_dim];
+ let mut bias = vec![0.0f32; model_dim];
for ch in 0..model_dim {
- bias[ch] = T::from(0.02 * (ch as f32) + 0.05).unwrap();
+ bias[ch] = 0.02 * (ch as f32) + 0.05;
}
Some(bias.into_boxed_slice())
} else {
@@ -237,11 +242,11 @@ fn get_test_data_edge<T: ArrayElement + Float>(
in_proj[2 * model_dim + ch] = T::from(x_in).unwrap();
}
- let mut w = vec![T::zero(); model_dim * kernel_size];
+ let mut w = vec![0.0f32; model_dim * kernel_size];
for ch in 0..model_dim {
for tap in 0..kernel_size {
let val = 0.25 * (tap as f32) + 0.1;
- w[ch * kernel_size + tap] = T::from(val).unwrap();
+ w[ch * kernel_size + tap] = val;
}
}
@@ -256,9 +261,9 @@ fn get_test_data_edge<T: ArrayElement + Float>(
let parents = vec![-1i32];
let b = if has_bias {
- let mut bias = vec![T::zero(); model_dim];
+ let mut bias = vec![0.0f32; model_dim];
for ch in 0..model_dim {
- bias[ch] = T::from(0.005 * (ch as f32) + 0.01).unwrap();
+ bias[ch] = 0.005 * (ch as f32) + 0.01;
}
Some(bias.into_boxed_slice())
} else {