Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Ssm cpu kernels (#308)

Aleksandr Golokoz971cd98d192e
diff --git a/crates/uzu/src/backends/common/gpu_types/activation_type.rs b/crates/uzu/src/backends/common/gpu_types/activation_type.rs
index 4a902e9d4..7524bff9b 100644
--- a/crates/uzu/src/backends/common/gpu_types/activation_type.rs
+++ b/crates/uzu/src/backends/common/gpu_types/activation_type.rs
@@ -10,6 +10,7 @@ pub enum ActivationType {
GELU,
TANH,
IDENTITY,
+ SOFTPLUS,
}
impl ActivationType {
@@ -24,6 +25,7 @@ impl ActivationType {
ActivationType::GELU => gelu(x),
ActivationType::TANH => tanh_activation(x),
ActivationType::IDENTITY => x,
+ ActivationType::SOFTPLUS => softplus(x),
}
}
}
@@ -52,3 +54,13 @@ fn tanh_activation<T: Float>(x: T) -> T {
let x_float = x.to_f32().unwrap();
T::from(x_float.tanh()).unwrap()
}
+
+fn softplus<T: Float>(x: T) -> T {
+ let x_float = x.to_f32().unwrap();
+ if x_float > 20f32 {
+ return x;
+ }
+
+ let result = (1f32 + x_float.exp()).ln();
+ T::from(result).unwrap()
+}
diff --git a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
index 7d944cf18..b14dfdb03 100644
--- a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
+++ b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
@@ -22,8 +22,8 @@ pub struct SSDPrefillArguments<'a, B: Backend> {
pub state: &'a mut B::Buffer,
pub y: &'a mut B::Buffer,
pub suffix_len: usize,
- pub group_size: i32,
- pub state_size: i32,
+ pub group_size: u32,
+ pub state_size: u32,
pub x_strides: [usize; 3],
pub dt_strides: [usize; 2],
pub cb_strides: [usize; 3],
diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs
index d00fedb10..1fb06b66d 100644
--- a/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs
+++ b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs
@@ -7,73 +7,194 @@ use crate::ArrayElement;
#[kernel(Conv1dPack)]
#[variants(T, f32, f16, bf16)]
pub fn conv1d_pack<T: ArrayElement + Float>(
- #[allow(unused)] state_in: *const T,
- #[allow(unused)] x: *const T,
- #[allow(unused)] padded: *mut T,
- #[allow(unused)] state_stride: u32,
- #[allow(unused)] row_stride: u32,
- #[allow(unused)] suffix_len: u32,
- #[allow(unused)] num_channels: u32,
+ state_in: *const T,
+ x: *const T,
+ padded: *mut T,
+ state_stride: u32,
+ row_stride: u32,
+ suffix_len: u32,
+ num_channels: u32,
) {
- todo!()
+ let state_stride = state_stride as usize;
+ let row_stride = row_stride as usize;
+ let suffix_len = suffix_len as usize;
+ let num_channels = num_channels as usize;
+
+ unsafe {
+ for channel_idx in 0..num_channels {
+ for row_idx in 0..state_stride + suffix_len {
+ let padded_index = row_idx * row_stride + channel_idx;
+ if row_idx < state_stride {
+ let state_index = channel_idx * state_stride + row_idx;
+ *padded.add(padded_index) = *state_in.add(state_index);
+ } else {
+ let token = row_idx - state_stride;
+ let x_index = token * row_stride + channel_idx;
+ *padded.add(padded_index) = *x.add(x_index);
+ }
+ }
+ }
+ }
}
#[kernel(Conv1dDecode)]
#[variants(T, f32, f16, bf16)]
pub fn conv1d_decode<T: ArrayElement + Float>(
- #[allow(unused)] x: *const T,
- #[allow(unused)] w: *const T,
- #[allow(unused)]
- #[optional(has_bias)]
- b: Option<*const T>,
- #[allow(unused)]
- #[optional(!state_in_place)]
- state: Option<*const T>,
- #[allow(unused)] x_out: *mut T,
- #[allow(unused)] b_out: *mut T,
- #[allow(unused)] c_out: *mut T,
- #[allow(unused)] next_state: *mut T,
- #[allow(unused)] kernel_size: u32,
- #[allow(unused)] row_stride: u32,
- #[allow(unused)] state_stride: u32,
- #[allow(unused)] num_channels: u32,
- #[allow(unused)] suffix_len: u32,
- #[allow(unused)] inner_dim: u32,
- #[allow(unused)] proj_dim: u32,
- #[allow(unused)] activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,
- #[allow(unused)]
- #[specialize]
- has_bias: bool,
- #[allow(unused)]
- #[specialize]
- state_in_place: bool,
+ x: *const T,
+ w: *const T,
+ #[optional(has_bias)] b: Option<*const T>,
+ #[optional(!state_in_place)] state: Option<*const T>,
+ x_out: *mut T,
+ b_out: *mut T,
+ c_out: *mut T,
+ next_state: *mut T,
+ kernel_size: u32,
+ row_stride: u32,
+ state_stride: u32,
+ num_channels: u32,
+ suffix_len: u32,
+ inner_dim: u32,
+ proj_dim: u32,
+ activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,
+ #[specialize] has_bias: bool,
+ #[specialize] state_in_place: bool,
) {
- todo!()
+ let kernel_size = kernel_size as usize;
+ let row_stride = row_stride as usize;
+ let state_stride = state_stride as usize;
+ let num_channels = num_channels as usize;
+ let suffix_len = suffix_len as usize;
+ let inner_dim = inner_dim as usize;
+ let proj_dim = proj_dim as usize;
+
+ let state = match state_in_place {
+ true => next_state,
+ false => state.unwrap(),
+ };
+
+ unsafe {
+ for token_idx in 0..suffix_len {
+ for channel_idx in 0..num_channels {
+ let x_idx = token_idx * row_stride + channel_idx;
+ let state_offset = channel_idx * state_stride;
+ let w_row = w.add(channel_idx * kernel_size);
+
+ let mut acc = 0.0f32;
+ if has_bias {
+ acc = (*b.unwrap().add(channel_idx)).to_f32().unwrap();
+ }
+
+ let state_taps = kernel_size.saturating_sub(1);
+ for tap in 0..state_taps {
+ let state_index = state_offset + tap;
+ let sample = (*state.add(state_index)).to_f32().unwrap();
+ acc += (*w_row.add(tap)).to_f32().unwrap() * sample;
+ }
+
+ let current = (*x.add(x_idx)).to_f32().unwrap();
+ let current_tap_index = state_taps;
+ if current_tap_index < kernel_size {
+ acc += (*w_row.add(current_tap_index)).to_f32().unwrap() * current;
+ }
+
+ let activated = activation_type.activate(T::from(acc).unwrap());
+ if channel_idx < inner_dim {
+ let dst = token_idx * inner_dim + channel_idx;
+ *x_out.add(dst) = activated;
+ } else if channel_idx < inner_dim + proj_dim {
+ let dst = token_idx * proj_dim + (channel_idx - inner_dim);
+ *b_out.add(dst) = activated;
+ } else if channel_idx < inner_dim + 2 * proj_dim {
+ let dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
+ *c_out.add(dst) = activated;
+ }
+ if state_taps == 0 {
+ continue;
+ }
+
+ for tap in 0..state_taps - 1 {
+ let src_index = state_offset + tap + 1;
+ let dst_index = state_offset + tap;
+ *next_state.add(dst_index) = *state.add(src_index);
+ }
+
+ let tail_index = state_offset + state_taps - 1;
+ *next_state.add(tail_index) = T::from(current).unwrap();
+ }
+ }
+ }
}
#[kernel(Conv1dScan)]
#[variants(T, f32, f16, bf16)]
pub fn conv1d_scan<T: ArrayElement + Float>(
- #[allow(unused)] padded: *const T,
- #[allow(unused)] w: *const T,
- #[allow(unused)]
- #[optional(has_bias)]
- b: Option<*const T>,
- #[allow(unused)] x_out: *mut T,
- #[allow(unused)] b_out: *mut T,
- #[allow(unused)] c_out: *mut T,
- #[allow(unused)] state_out: *mut T,
- #[allow(unused)] suffix_len: u32,
- #[allow(unused)] kernel_size: u32,
- #[allow(unused)] row_stride: u32,
- #[allow(unused)] state_stride: u32,
- #[allow(unused)] num_channels: u32,
- #[allow(unused)] inner_dim: u32,
- #[allow(unused)] proj_dim: u32,
- #[allow(unused)] activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,
- #[allow(unused)]
- #[specialize]
- has_bias: bool,
+ padded: *const T,
+ w: *const T,
+ #[optional(has_bias)] b: Option<*const T>,
+ x_out: *mut T,
+ b_out: *mut T,
+ c_out: *mut T,
+ state_out: *mut T,
+ suffix_len: u32,
+ kernel_size: u32,
+ row_stride: u32,
+ state_stride: u32,
+ num_channels: u32,
+ inner_dim: u32,
+ proj_dim: u32,
+ activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,
+ #[specialize] has_bias: bool,
) {
- todo!()
+ let suffix_len = suffix_len as usize;
+ let kernel_size = kernel_size as usize;
+ let row_stride = row_stride as usize;
+ let state_stride = state_stride as usize;
+ let num_channels = num_channels as usize;
+ let inner_dim = inner_dim as usize;
+ let proj_dim = proj_dim as usize;
+
+ unsafe {
+ for token_idx in 0..suffix_len + kernel_size.saturating_sub(1) {
+ for channel_idx in 0..num_channels {
+ let tap_count = kernel_size.saturating_sub(1);
+ let state_offset = channel_idx * state_stride;
+ let weight_offset = channel_idx * kernel_size;
+ let w_row = w.add(weight_offset);
+
+ if token_idx < suffix_len {
+ let mut acc = 0.0f32;
+ if has_bias {
+ acc = (*b.unwrap().add(channel_idx)).to_f32().unwrap();
+ }
+
+ for tap in 0..kernel_size {
+ let padded_index = (token_idx + tap) * row_stride + channel_idx;
+ let sample = (*padded.add(padded_index)).to_f32().unwrap();
+ acc += (*w_row.add(tap)).to_f32().unwrap() * sample;
+ }
+
+ let activated = activation_type.activate(T::from(acc).unwrap());
+ if channel_idx < inner_dim {
+ let dst = token_idx * inner_dim + channel_idx;
+ *x_out.add(dst) = activated;
+ } else if channel_idx < inner_dim + proj_dim {
+ let dst = token_idx * proj_dim + (channel_idx - inner_dim);
+ *b_out.add(dst) = activated;
+ } else if channel_idx < inner_dim + 2 * proj_dim {
+ let dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
+ *c_out.add(dst) = activated;
+ }
+ } else if tap_count > 0 {
+ let tap = token_idx - suffix_len;
+ if tap >= tap_count {
+ return;
+ }
+
+ let padded_index = token_idx * row_stride + channel_idx;
+ let sample = (*padded.add(padded_index)).to_f32().unwrap();
+ *state_out.add(state_offset + tap) = T::from(sample).unwrap();
+ }
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs b/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs
index 873e4908f..8df79081c 100644
--- a/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs
+++ b/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs
@@ -7,16 +7,39 @@ use crate::ArrayElement;
#[kernel(SplitInProj)]
#[variants(T, f32, f16, bf16)]
pub fn split_in_proj<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] conv_out: *mut T,
- #[allow(unused)] z_out: *mut T,
- #[allow(unused)] dt_out: *mut T,
- #[allow(unused)] z_bias: *const T,
- #[allow(unused)] suffix_length: u32,
- #[allow(unused)] total_dim: u32,
- #[allow(unused)] conv_dim: u32,
- #[allow(unused)] inner_dim: u32,
- #[allow(unused)] num_heads: u32,
+ input: *const T,
+ conv_out: *mut T,
+ z_out: *mut T,
+ dt_out: *mut T,
+ z_bias: *const T,
+ suffix_length: u32,
+ total_dim: u32,
+ conv_dim: u32,
+ inner_dim: u32,
+ num_heads: u32,
) {
- todo!()
+ let suffix_length = suffix_length as usize;
+ let total_dim = total_dim as usize;
+ let conv_dim = conv_dim as usize;
+ let inner_dim = inner_dim as usize;
+ let num_heads = num_heads as usize;
+
+ unsafe {
+ for row in 0..suffix_length {
+ for col in 0..total_dim {
+ let input_idx = row * total_dim + col;
+ if col < conv_dim {
+ let dst = row * conv_dim + col;
+ *conv_out.add(dst) = *input.add(input_idx);
+ } else if col < conv_dim + inner_dim {
+ let dst = row * inner_dim + (col - conv_dim);
+ let bias_idx = col - conv_dim;
+ *z_out.add(dst) = *input.add(input_idx) + *z_bias.add(bias_idx);
+ } else if col < conv_dim + inner_dim + num_heads {
+ let dst = row * num_heads + (col - conv_dim - inner_dim);
+ *dt_out.add(dst) = *input.add(input_idx);
+ }
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs
index 0ca159e13..3a673f923 100644
--- a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs
+++ b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs
@@ -2,7 +2,7 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{ArrayElement, backends::common::gpu_types::ActivationType};
#[kernel(SSDPrefill64)]
#[variants(T, f32, f16, bf16)]
@@ -16,8 +16,8 @@ pub fn ssd_prefill64<T: ArrayElement + Float>(
#[allow(unused)] state: *mut T,
#[allow(unused)] y: *mut T,
#[allow(unused)] suffix_len: u32,
- #[allow(unused)] group_size: i32,
- #[allow(unused)] state_size: i32,
+ #[allow(unused)] group_size: u32,
+ #[allow(unused)] state_size: u32,
#[allow(unused)] x_strides: &[u32],
#[allow(unused)] dt_strides: &[u32],
#[allow(unused)] cb_strides: &[u32],
@@ -40,8 +40,8 @@ pub fn ssd_prefill<T: ArrayElement + Float>(
#[allow(unused)] state: *mut T,
#[allow(unused)] y: *mut T,
#[allow(unused)] suffix_len: u32,
- #[allow(unused)] group_size: i32,
- #[allow(unused)] state_size: i32,
+ #[allow(unused)] group_size: u32,
+ #[allow(unused)] state_size: u32,
#[allow(unused)] x_strides: &[u32],
#[allow(unused)] dt_strides: &[u32],
#[allow(unused)] cb_strides: &[u32],
@@ -55,23 +55,64 @@ pub fn ssd_prefill<T: ArrayElement + Float>(
#[kernel(SSDPrefillSequential)]
#[variants(T, f32, f16, bf16)]
pub fn ssd_prefill_sequential<T: ArrayElement + Float>(
- #[allow(unused)] x: *const T,
- #[allow(unused)] dt_raw: *const T,
- #[allow(unused)] b: *const T,
- #[allow(unused)] c: *const T,
- #[allow(unused)] d: *const T,
- #[allow(unused)] z: *const T,
- #[allow(unused)] state: *mut T,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] suffix_len: u32,
- #[allow(unused)] group_size: i32,
- #[allow(unused)] state_size: i32,
- #[allow(unused)] x_strides: &[u32],
- #[allow(unused)] dt_strides: &[u32],
- #[allow(unused)] cb_strides: &[u32],
- #[allow(unused)] state_strides: &[u32],
- #[allow(unused)] channels: u32,
- #[allow(unused)] head_dim: u32,
+ x: *const T,
+ dt_raw: *const T,
+ b: *const T,
+ c: *const T,
+ d: *const T,
+ z: *const T,
+ state: *mut T,
+ y: *mut T,
+ suffix_len: u32,
+ group_size: u32,
+ state_size: u32,
+ x_strides: &[u32],
+ dt_strides: &[u32],
+ cb_strides: &[u32],
+ state_strides: &[u32],
+ channels: u32,
+ head_dim: u32,
) {
- todo!()
+ let suffix_len = suffix_len as usize;
+ let group_size = group_size as usize;
+ let state_size = state_size as usize;
+ let channels = channels as usize;
+ let head_dim = head_dim as usize;
+
+ let total_pairs = suffix_len * channels * head_dim;
+ let safe_group = group_size.max(1) as usize;
+
+ unsafe {
+ for h in 0..channels {
+ let group_idx = h / safe_group;
+ for dh in 0..head_dim {
+ let state_base = h * state_strides[0] as usize + dh * state_strides[1] as usize;
+ for token in 0..suffix_len {
+ let x_idx = token * x_strides[0] as usize + h * x_strides[1] as usize + dh * x_strides[2] as usize;
+ let dt_idx = token * dt_strides[0] as usize + h * dt_strides[1] as usize;
+ let cb_base = token * cb_strides[0] as usize + group_idx * cb_strides[1] as usize;
+
+ let x_val = *x.add(x_idx);
+ let dt_raw = *dt_raw.add(dt_idx);
+ let dt_val = ActivationType::SOFTPLUS.activate(dt_raw);
+ let decay_val = (-dt_val).exp();
+ let dt_scaled_input = x_val;
+
+ let mut acc = *d.add(h) * x_val;
+ for s in 0..state_size {
+ let state_idx = state_base + s * state_strides[2] as usize;
+ let cb_idx = cb_base + s * cb_strides[2] as usize;
+ let b_coeff = *b.add(cb_idx);
+ let c_coeff = *c.add(cb_idx);
+ let new_state = decay_val * *state.add(state_idx) + dt_scaled_input * b_coeff;
+ *state.add(state_idx) = new_state;
+ acc = acc + new_state * c_coeff;
+ }
+
+ let gate = ActivationType::SILU.activate(*z.add(x_idx));
+ *y.add(x_idx) = acc * gate;
+ }
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs
index 59d069521..2de6ecea9 100644
--- a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs
+++ b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs
@@ -2,34 +2,74 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{ArrayElement, backends::common::gpu_types::ActivationType};
#[kernel(SSDUpdate)]
#[variants(T, f32, f16, bf16)]
pub fn ssd_update<T: ArrayElement + Float>(
- #[allow(unused)] x: *const T,
- #[allow(unused)] dt_raw: *const T,
- #[allow(unused)] b: *const T,
- #[allow(unused)] c: *const T,
- #[allow(unused)] d: *const T,
- #[allow(unused)] z: *const T,
- #[allow(unused)]
- #[optional(!state_in_place)]
- state: Option<*const T>,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] next_state: *mut T,
- #[allow(unused)] group_size: u32,
- #[allow(unused)] state_size: u32,
- #[allow(unused)] x_strides: &[u32],
- #[allow(unused)] dt_strides: &[u32],
- #[allow(unused)] cb_strides: &[u32],
- #[allow(unused)] state_strides: &[u32],
- #[allow(unused)] b_size: u32,
- #[allow(unused)] h_size: u32,
- #[allow(unused)] dh_size: u32,
- #[allow(unused)]
- #[specialize]
- state_in_place: bool,
+ x: *const T,
+ dt_raw: *const T,
+ b: *const T,
+ c: *const T,
+ d: *const T,
+ z: *const T,
+ #[optional(!state_in_place)] state: Option<*const T>,
+ y: *mut T,
+ next_state: *mut T,
+ group_size: u32,
+ state_size: u32,
+ x_strides: &[u32],
+ dt_strides: &[u32],
+ cb_strides: &[u32],
+ state_strides: &[u32],
+ b_size: u32,
+ h_size: u32,
+ dh_size: u32,
+ #[specialize] state_in_place: bool,
) {
- todo!()
+ let group_size = group_size as usize;
+ let state_size = state_size as usize;
+ let b_size = b_size as usize;
+ let h_size = h_size as usize;
+ let dh_size = dh_size as usize;
+
+ let state = match state_in_place {
+ true => next_state,
+ false => state.unwrap(),
+ };
+
+ unsafe {
+ for b_idx in 0..b_size {
+ for h_idx in 0..h_size {
+ for dh_idx in 0..dh_size {
+ let cb_start_idx = b_idx * cb_strides[0] as usize + (h_idx / group_size) * cb_strides[1] as usize;
+ let x_idx =
+ b_idx * x_strides[0] as usize + h_idx * x_strides[1] as usize + dh_idx * x_strides[2] as usize;
+ let dt_idx = b_idx * dt_strides[0] as usize + h_idx * dt_strides[1] as usize;
+ let state_start_idx = b_idx * state_strides[0] as usize
+ + h_idx * state_strides[1] as usize
+ + dh_idx * state_strides[2] as usize;
+
+ let this_x = *x.add(x_idx);
+ let dt_raw_val = *dt_raw.add(dt_idx);
+ let this_dt = ActivationType::SOFTPLUS.activate(dt_raw_val);
+ let this_decay = T::from((-this_dt.to_f32().unwrap()).exp()).unwrap();
+ let this_d = *d.add(h_idx);
+ let this_z = ActivationType::SILU.activate(*z.add(x_idx));
+
+ let mut temp = T::zero();
+ for i in 0..state_size {
+ let cb_idx = cb_start_idx + i;
+ let state_idx = state_start_idx + i;
+ let this_new_state = *state.add(state_idx) * this_decay + *b.add(cb_idx) * this_x;
+ *next_state.add(state_idx) = this_new_state;
+ temp = temp + this_new_state * *c.add(cb_idx);
+ }
+ temp = temp + this_d * this_x;
+ temp = temp * this_z;
+ *y.add(x_idx) = temp;
+ }
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/metal/generated/activation_type.h b/crates/uzu/src/backends/metal/generated/activation_type.h
index 290f091ea..2cf6beef1 100644
--- a/crates/uzu/src/backends/metal/generated/activation_type.h
+++ b/crates/uzu/src/backends/metal/generated/activation_type.h
@@ -19,6 +19,7 @@ enum ActivationType : uint32_t {
GELU,
TANH,
IDENTITY,
+ SOFTPLUS,
};
#ifdef __METAL_VERSION__
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activations.h b/crates/uzu/src/backends/metal/kernel/activation/activations.h
index b09b0b45d..6df1548ae 100644
--- a/crates/uzu/src/backends/metal/kernel/activation/activations.h
+++ b/crates/uzu/src/backends/metal/kernel/activation/activations.h
@@ -35,6 +35,17 @@ inline T activate_leaky_relu(T x, float alpha) {
return static_cast<T>(yf);
}
+template <typename T>
+inline T activate_softplus(T x) {
+ float xf = static_cast<float>(x);
+ if (xf > 20.0f) {
+ return x;
+ }
+
+ float result = fast::log(1.0f + fast::exp(xf));
+ return static_cast<T>(result);
+}
+
template <typename T>
inline T activate_tanh(T x) {
return static_cast<T>(metal::tanh(float(x)));
@@ -51,5 +62,7 @@ inline T activate(T x, ActivationType type) {
return activate_tanh(x);
case IDENTITY:
return x;
+ case SOFTPLUS:
+ return activate_softplus(x);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal b/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal
index e0474b957..685fce193 100644
--- a/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal
@@ -1,7 +1,6 @@
#include <metal_stdlib>
#include "../activation/activations.h"
#include "../common/dsl.h"
-#include "../ssm/ssm_common.h"
using namespace metal;
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal b/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal
index 7132561aa..8b9119f41 100644
--- a/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal
@@ -1,7 +1,6 @@
#include <metal_stdlib>
#include "../activation/activations.h"
#include "../common/dsl.h"
-#include "../ssm/ssm_common.h"
using namespace metal;
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
index 2a7020b36..75e19ddf6 100644
--- a/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
@@ -3,7 +3,6 @@
#include "../common/dsl.h"
#include "../common/thread_context.h"
#include "../common/threadgroup_reduce.h"
-#include "../ssm/ssm_common.h"
using namespace metal;
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal
index 3e5f78703..b26376bb2 100644
--- a/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal
@@ -1,7 +1,6 @@
#include <metal_stdlib>
#include "../common/defines.h"
#include "../common/dsl.h"
-#include "../ssm/ssm_common.h"
using namespace metal;
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal b/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
index 65256e6b1..6365c2092 100644
--- a/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
@@ -1,7 +1,7 @@
#include <metal_stdlib>
+#include "../activation/activations.h"
#include "../common/defines.h"
#include "../common/dsl.h"
-#include "../ssm/ssm_common.h"
using namespace metal;
@@ -88,7 +88,7 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(
float a_raw =
float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);
- float sp = softplus(a_raw + float(dt_bias[hv]));
+ float sp = activate_softplus(a_raw + float(dt_bias[hv]));
float decay = fast::exp(-fast::exp(float(a_log[hv])) * sp);
beta_out[token_idx * num_v_heads + hv] = beta;
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/uzu/src/backends/metal/kernel/delta_net/update.metal
index a12e283a4..094cfe0c3 100644
--- a/crates/uzu/src/backends/metal/kernel/delta_net/update.metal
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/update.metal
@@ -4,7 +4,6 @@
#include "../common/dsl.h"
#include "../common/thread_context.h"
#include "../common/threadgroup_reduce.h"
-#include "../ssm/ssm_common.h"
using namespace metal;
@@ -97,7 +96,7 @@ PUBLIC KERNEL(DeltaNetUpdate)(
float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);
float beta = 1.0f / (1.0f + fast::exp(-beta_raw));
float a_raw = float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);
- float sp = softplus(a_raw + float(dt_bias[hv_idx]));
+ float sp = activate_softplus(a_raw + float(dt_bias[hv_idx]));
float decay = fast::exp(-fast::exp(float(a_log[hv_idx])) * sp);
float v_i =
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
index 2fc6e8114..e6100b52e 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
@@ -137,33 +137,31 @@ PUBLIC KERNEL(Conv1dScan)(
if (has_bias) {
acc = float(b[channel_idx]);
}
+
for (uint tap = 0; tap < kernel_size; ++tap) {
- const uint padded_idx = size_t(token_idx) + size_t(tap);
- float sample = 0.0f;
- const uint padded_index = size_t(padded_idx) * row_stride + channel_idx;
- sample = float(padded[padded_index]);
+ const uint padded_index = (token_idx + tap) * row_stride + channel_idx;
+ float sample = float(padded[padded_index]);
acc += float(w_row[tap]) * sample;
}
const T activated = activate(static_cast<T>(acc), activation_type);
if (channel_idx < inner_dim) {
- const size_t dst = size_t(token_idx) * inner_dim + channel_idx;
+ const uint dst = token_idx * inner_dim + channel_idx;
x_out[dst] = activated;
} else if (channel_idx < inner_dim + proj_dim) {
- const size_t dst =
- size_t(token_idx) * proj_dim + (channel_idx - inner_dim);
+ const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);
b_out[dst] = activated;
} else if (channel_idx < inner_dim + 2 * proj_dim) {
- const size_t dst =
- size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);
+ const uint dst =
+ token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
c_out[dst] = activated;
}
} else if (tap_count > 0) {
- const size_t tap = size_t(token_idx - suffix_len);
- if (tap >= size_t(tap_count)) {
+ const uint tap = token_idx - suffix_len;
+ if (tap >= tap_count) {
return;
}
- const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;
+ const uint padded_index = token_idx * row_stride + channel_idx;
const float sample = float(padded[padded_index]);
state_out[state_offset + tap] = static_cast<T>(sample);
}
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
index fb1b85ebd..18bd8805b 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
@@ -17,16 +17,16 @@ PUBLIC KERNEL(SplitInProj)(
const uint row AXIS(suffix_length, 16),
const uint col AXIS(total_dim, 16)
) {
- const int input_idx = row * total_dim + col;
+ const uint input_idx = row * total_dim + col;
if (col < conv_dim) {
- const int dst = row * conv_dim + col;
+ const uint dst = row * conv_dim + col;
conv_out[dst] = input[input_idx];
} else if (col < conv_dim + inner_dim) {
- const int dst = row * inner_dim + (col - conv_dim);
- const int bias_idx = col - conv_dim;
+ const uint dst = row * inner_dim + (col - conv_dim);
+ const uint bias_idx = col - conv_dim;
z_out[dst] = input[input_idx] + z_bias[bias_idx];
} else if (col < conv_dim + inner_dim + num_heads) {
- const int dst = row * num_heads + (col - conv_dim - inner_dim);
+ const uint dst = row * num_heads + (col - conv_dim - inner_dim);
dt_out[dst] = input[input_idx];
}
}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index 6330408ee..d795c70a4 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -3,7 +3,6 @@
#include "../activation/activations.h"
#include "../common/dsl.h"
#include "../common/thread_context.h"
-#include "ssm_common.h"
using namespace metal;
@@ -21,8 +20,8 @@ PUBLIC KERNEL(SSDPrefill64)(
device T* state, // (h, dh, n)
device T* y, // (suffix, h, dh)
constant const uint& suffix_len,
- constant const int& group_size,
- constant const int& state_size,
+ constant const uint& group_size,
+ constant const uint& state_size,
constant const uint* x_strides,
constant const uint* dt_strides,
constant const uint* cb_strides,
@@ -35,7 +34,7 @@ PUBLIC KERNEL(SSDPrefill64)(
) {
const uint h_idx = pair_idx / head_dim;
const uint dh_idx = pair_idx % head_dim;
- const uint safe_group = uint(max(group_size, 1));
+ const uint safe_group = max(group_size, 1u);
const uint group_idx = h_idx / safe_group;
const uint x_token_stride = x_strides[0];
@@ -59,27 +58,21 @@ PUBLIC KERNEL(SSDPrefill64)(
const uint idx0 = lane_idx;
const uint idx1 = lane_idx + thread_context.simdgroup_size;
- float state0 = 0.0f;
- float state1 = 0.0f;
- uint state_idx0 = 0;
- uint state_idx1 = 0;
- state_idx0 = state_base + idx0 * state_inner_stride;
- state0 = float(state[state_idx0]);
+ const uint state_idx0 = state_base + idx0 * state_inner_stride;
+ const uint state_idx1 = state_base + idx1 * state_inner_stride;
+ float state0 = float(state[state_idx0]);
+ float state1 = float(state[state_idx1]);
- state_idx1 = state_base + idx1 * state_inner_stride;
- state1 = float(state[state_idx1]);
-
- uint cb_idx0 = 0;
- uint cb_idx1 = 0;
- cb_idx0 = cb_group_base + idx0 * cb_state_stride;
- cb_idx1 = cb_group_base + idx1 * cb_state_stride;
+ uint cb_idx0 = cb_group_base + idx0 * cb_state_stride;
+ uint cb_idx1 = cb_group_base + idx1 * cb_state_stride;
for (uint token = 0; token < suffix_len; ++token) {
const uint x_idx = token * x_token_stride + x_base;
const uint dt_idx = token * dt_token_stride + dt_base;
const float x_val = float(x[x_idx]);
- const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));
+ const float decay_val =
+ fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));
const float gate = float(activate_silu(z[x_idx]));
const float skip = d_scalar * x_val;
const float dt_scaled_input = x_val;
@@ -119,8 +112,8 @@ PUBLIC KERNEL(SSDPrefill)(
device T* state, // (h, dh, n)
device T* y, // (suffix, h, dh)
constant const uint& suffix_len,
- constant const int& group_size,
- constant const int& state_size,
+ constant const uint& group_size,
+ constant const uint& state_size,
constant const uint* x_strides,
constant const uint* dt_strides,
constant const uint* cb_strides,
@@ -134,7 +127,7 @@ PUBLIC KERNEL(SSDPrefill)(
const int state_dim = state_size;
const uint h_idx = pair_idx / head_dim;
const uint dh_idx = pair_idx % head_dim;
- const uint safe_group = uint(max(group_size, 1));
+ const uint safe_group = max(group_size, 1u);
const uint group_idx = h_idx / safe_group;
const uint x_token_stride = x_strides[0];
@@ -167,8 +160,12 @@ PUBLIC KERNEL(SSDPrefill)(
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);
- const uint state_idx = state_base + idx * state_inner_stride;
- lane_states[chunk] = float(state[state_idx]);
+ if (idx < state_dim) {
+ const uint state_idx = state_base + idx * state_inner_stride;
+ lane_states[chunk] = float(state[state_idx]);
+ } else {
+ lane_states[chunk] = 0.0f;
+ }
}
for (uint token = 0; token < suffix_len; ++token) {
@@ -176,7 +173,8 @@ PUBLIC KERNEL(SSDPrefill)(
const uint dt_idx = token * dt_token_stride + dt_base;
const float x_val = float(x[x_idx]);
- const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));
+ const float decay_val =
+ fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));
const float gate = float(activate_silu(z[x_idx]));
const float skip = d_scalar * x_val;
const float dt_scaled_input = x_val;
@@ -186,12 +184,14 @@ PUBLIC KERNEL(SSDPrefill)(
for (int chunk = 0; chunk < chunk_count; ++chunk) {
const int idx =
chunk * int(thread_context.simdgroup_size) + int(lane_idx);
- const uint cb_idx =
- cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;
- const float new_state =
- decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);
- lane_states[chunk] = new_state;
- contrib_sum += new_state * float(c[cb_idx]);
+ if (idx < state_dim) {
+ const uint cb_idx = cb_group_base + uint(idx) * cb_state_stride +
+ token * cb_token_stride;
+ const float new_state =
+ decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);
+ lane_states[chunk] = new_state;
+ contrib_sum += new_state * float(c[cb_idx]);
+ }
}
float dot = simd_sum(contrib_sum);
@@ -203,8 +203,10 @@ PUBLIC KERNEL(SSDPrefill)(
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);
- const uint state_idx = state_base + uint(idx) * state_inner_stride;
- state[state_idx] = static_cast<T>(lane_states[chunk]);
+ if (idx < state_dim) {
+ const uint state_idx = state_base + uint(idx) * state_inner_stride;
+ state[state_idx] = static_cast<T>(lane_states[chunk]);
+ }
}
}
@@ -220,8 +222,8 @@ PUBLIC KERNEL(SSDPrefillSequential)(
device T* state, // (h, dh, n)
device T* y, // (suffix, h, dh)
constant const uint& suffix_len,
- constant const int& group_size,
- constant const int& state_size,
+ constant const uint& group_size,
+ constant const uint& state_size,
constant const uint* x_strides,
constant const uint* dt_strides,
constant const uint* cb_strides,
@@ -231,7 +233,7 @@ PUBLIC KERNEL(SSDPrefillSequential)(
const uint h_idx AXIS(channels, 32),
const uint dh_idx AXIS(head_dim, 32)
) {
- const uint safe_group = uint(max(group_size, 1));
+ const uint safe_group = max(group_size, 1u);
const uint group_idx = h_idx / safe_group;
device T* state_row = state + size_t(h_idx) * state_strides[0] +
size_t(dh_idx) * state_strides[1];
@@ -244,7 +246,7 @@ PUBLIC KERNEL(SSDPrefillSequential)(
const T this_x = x[x_idx];
const T dt_raw_val = dt_raw[dt_idx];
- const T this_dt = softplus(dt_raw_val);
+ const T this_dt = activate_softplus(dt_raw_val);
const T this_decay = static_cast<T>(fast::exp(-float(this_dt)));
const T this_D = d[h_idx];
const T this_z = activate_silu(z[x_idx]);
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
index 4afcc0302..c4564d601 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
@@ -1,7 +1,6 @@
#include <metal_stdlib>
#include "../activation/activations.h"
#include "../common/dsl.h"
-#include "ssm_common.h"
using namespace metal;
@@ -52,7 +51,7 @@ PUBLIC KERNEL(SSDUpdate)(
// load data
T this_x = x[x_idx];
T dt_raw_val = dt_raw[dt_idx];
- T this_dt = softplus(dt_raw_val);
+ T this_dt = activate_softplus(dt_raw_val);
T this_decay = static_cast<T>(fast::exp(-float(this_dt)));
T this_d = d[h_idx];
T this_z = z[x_idx];
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
deleted file mode 100644
index 1884fc210..000000000
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#pragma once
-
-#include <metal_stdlib>
-
-template <typename T>
-inline T softplus(T x) {
- float xf = float(x);
- if (xf > 20.0f) {
- return x;
- }
- return static_cast<T>(log(1.0f + fast::exp(xf)));
-}
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index 183ba1f7a..0863028d5 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -292,8 +292,8 @@ impl<B: Backend> MambaMixer<B> {
state: state_arr.buffer().borrow_mut().deref_mut(),
y: out.buffer().borrow_mut().deref_mut(),
suffix_len: suffix_length,
- group_size: (self.config.num_heads / self.config.num_groups) as i32,
- state_size: self.config.state_dim as i32,
+ group_size: (self.config.num_heads / self.config.num_groups) as u32,
+ state_size: self.config.state_dim as u32,
x_strides: [self.config.num_heads * self.config.head_dim, self.config.head_dim, 1],
dt_strides: [self.config.num_heads, 1],
cb_strides: [self.config.num_groups * self.config.state_dim, self.config.state_dim, 1],
diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rs
index 1dbd6a2ae..a14a00f7b 100644
--- a/crates/uzu/src/encodable_block/mlp/moe.rs
+++ b/crates/uzu/src/encodable_block/mlp/moe.rs
@@ -182,11 +182,8 @@ impl<B: Backend> MoeBlock<B> {
ActivationType::SILU {
..
} => 2,
- ActivationType::TANH => {
- panic!("Tanh activation is not supported for MoE kernels")
- },
- ActivationType::IDENTITY => {
- panic!("Identity activation is not supported for MoE kernels")
+ _ => {
+ panic!("{:?} is not supported for MoE kernels", activation)
},
}
}
diff --git a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
index 34f9dcc9e..2904de720 100644
--- a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
+++ b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
@@ -1,64 +1,46 @@
-#![cfg(metal_backend)]
-
-use std::mem::size_of;
+use std::{
+ mem::size_of,
+ ops::{Deref, DerefMut},
+};
-use bytemuck;
-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
-use objc2::runtime::ProtocolObject;
+use uzu::backends::common::Buffer;
use crate::{
- DataType,
- backends::{
- common::{
- Backend, Context, Encoder, Kernels,
- gpu_types::ActivationType,
- kernel::{
- Conv1dScanKernel,
- ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
- },
+ ArrayContextExt, DataType,
+ backends::common::{
+ Backend, Context, Encoder, Kernels,
+ gpu_types::ActivationType,
+ kernel::{
+ Conv1dScanKernel,
+ ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
},
- metal::Metal,
},
+ for_each_non_cpu_backend,
};
-const STORAGE_MODE: MTLResourceOptions = MTLResourceOptions::STORAGE_MODE_SHARED;
-
-fn silu_scalar(x: f32) -> f32 {
- let y = 1.0 / (1.0 + (-x).exp());
- x * y
-}
-
-fn softplus_f32(x: f32) -> f32 {
- if x > 20.0 {
- x
- } else {
- (1.0 + x.exp()).ln()
- }
-}
-
-fn write_buffer(
- buf: &ProtocolObject<dyn MTLBuffer>,
+fn write_buffer<B: Backend>(
+ buf: &B::Buffer,
data: &[f32],
) {
unsafe {
- std::ptr::copy_nonoverlapping(data.as_ptr(), buf.contents().as_ptr() as *mut f32, data.len());
+ std::ptr::copy_nonoverlapping(data.as_ptr(), buf.cpu_ptr().as_ptr() as *mut f32, data.len());
}
}
-fn read_buffer(
- buf: &ProtocolObject<dyn MTLBuffer>,
+fn read_buffer<B: Backend>(
+ buf: &B::Buffer,
len: usize,
) -> Vec<f32> {
let mut out = vec![0.0f32; len];
unsafe {
- std::ptr::copy_nonoverlapping(buf.contents().as_ptr() as *const f32, out.as_mut_ptr(), len);
+ std::ptr::copy_nonoverlapping(buf.cpu_ptr().as_ptr() as *const f32, out.as_mut_ptr(), len);
}
out
}
-fn zero_buffer(buf: &ProtocolObject<dyn MTLBuffer>) {
+fn zero_buffer<B: Backend>(buf: &B::Buffer) {
unsafe {
- std::ptr::write_bytes(buf.contents().as_ptr(), 0, buf.length() as usize);
+ std::ptr::write_bytes(buf.cpu_ptr().as_ptr(), 0, buf.length());
}
}
@@ -96,10 +78,10 @@ fn ssd_prefill_cpu_reference(
let x_val = x_data[x_idx];
let dt_raw = dt_raw_data[dt_idx];
- let dt_val = softplus_f32(dt_raw);
+ let dt_val = ActivationType::SOFTPLUS.activate(dt_raw);
let decay_val = (-dt_val).exp();
let dt_scaled_input = x_val;
- let gate = silu_scalar(z_data[x_idx]);
+ let gate = ActivationType::SILU.activate(z_data[x_idx]);
let mut acc = d_data[h] * x_val;
for s in 0..state_dim {
@@ -196,49 +178,65 @@ impl SSDPrefillFixture {
}
}
-fn run_prefill_kernel_mode(
- ctx: &<Metal as Backend>::Context,
- kernel: &SSDPrefillKernels<Metal>,
+fn run_prefill_kernel_mode<B: Backend>(
+ ctx: &B::Context,
+ kernel: &SSDPrefillKernels<B>,
fixture: &SSDPrefillFixture,
mode: SSDPrefillMode,
) -> (Vec<f32>, Vec<f32>, Option<(Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>)>) {
- let device = &ctx.device;
- let x_buf = device
- .new_buffer_with_data(bytemuck::cast_slice(&fixture.x_data), STORAGE_MODE)
- .expect("Failed to create buffer");
- let dt_buf = device
- .new_buffer_with_data(bytemuck::cast_slice(&fixture.dt_data), STORAGE_MODE)
- .expect("Failed to create buffer");
- let b_buf = device
- .new_buffer_with_data(bytemuck::cast_slice(&fixture.b_data), STORAGE_MODE)
- .expect("Failed to create buffer");
- let c_buf = device
- .new_buffer_with_data(bytemuck::cast_slice(&fixture.c_data), STORAGE_MODE)
- .expect("Failed to create buffer");
- let d_buf = device
- .new_buffer_with_data(bytemuck::cast_slice(&fixture.d_data), STORAGE_MODE)
- .expect("Failed to create buffer");
- let z_buf = device
- .new_buffer_with_data(bytemuck::cast_slice(&fixture.z_data), STORAGE_MODE)
- .expect("Failed to create buffer");
- let mut state_buf = device.new_buffer(fixture.total_state * 4, STORAGE_MODE).expect("Failed to create buffer");
- let mut y_buf = device.new_buffer(fixture.total_x * 4, STORAGE_MODE).expect("Failed to create buffer");
-
- write_buffer(&state_buf, &fixture.state_init);
- zero_buffer(&y_buf);
-
- let args = SSDPrefillArguments::<Metal> {
- x: &x_buf,
- dt: &dt_buf,
- b: &b_buf,
- c: &c_buf,
- d: &d_buf,
- z: &z_buf,
- state: &mut state_buf,
- y: &mut y_buf,
+ let x_array = ctx.create_array_from(&[fixture.x_data.len()], &fixture.x_data, "");
+ let x_array_buf = x_array.buffer();
+ let x_array_buf_borrow = x_array_buf.borrow();
+ let x_buf = x_array_buf_borrow.deref();
+
+ let dt_array = ctx.create_array_from(&[fixture.dt_data.len()], &fixture.dt_data, "");
+ let dt_array_buf = dt_array.buffer();
+ let dt_array_buf_borrow = dt_array_buf.borrow();
+ let dt_buf = dt_array_buf_borrow.deref();
+
+ let b_array = ctx.create_array_from(&[fixture.b_data.len()], &fixture.b_data, "");
+ let b_array_buf = b_array.buffer();
+ let b_array_buf_borrow = b_array_buf.borrow();
+ let b_buf = b_array_buf_borrow.deref();
+
+ let c_array = ctx.create_array_from(&[fixture.c_data.len()], &fixture.c_data, "");
+ let c_array_buf = c_array.buffer();
+ let c_array_buf_borrow = c_array_buf.borrow();
+ let c_buf = c_array_buf_borrow.deref();
+
+ let d_array = ctx.create_array_from(&[fixture.d_data.len()], &fixture.d_data, "");
+ let d_array_buf = d_array.buffer();
+ let d_array_buf_borrow = d_array_buf.borrow();
+ let d_buf = d_array_buf_borrow.deref();
+
+ let z_array = ctx.create_array_from(&[fixture.z_data.len()], &fixture.z_data, "");
+ let z_array_buf = z_array.buffer();
+ let z_array_buf_borrow = z_array_buf.borrow();
+ let z_buf = z_array_buf_borrow.deref();
+
+ let state_array = ctx.create_array_from(&[fixture.state_init.len()], &fixture.state_init, "");
+ let state_array_buf = state_array.buffer();
+ let mut state_array_buf_borrow = state_array_buf.borrow_mut();
+ let state_buf = state_array_buf_borrow.deref_mut();
+
+ let y = vec![0f32; fixture.total_x];
+ let y_array = ctx.create_array_from(&[y.len()], &y, "");
+ let y_array_buf = y_array.buffer();
+ let mut y_array_buf_borrow = y_array_buf.borrow_mut();
+ let y_buf = y_array_buf_borrow.deref_mut();
+
+ let args = SSDPrefillArguments::<B> {
+ x: x_buf,
+ dt: dt_buf,
+ b: b_buf,
+ c: c_buf,
+ d: d_buf,
+ z: z_buf,
+ state: state_buf,
+ y: y_buf,
suffix_len: fixture.suffix_len,
- group_size: fixture.group_size,
- state_size: fixture.state_dim as i32,
+ group_size: fixture.group_size as u32,
+ state_size: fixture.state_dim as u32,
x_strides: fixture.x_strides,
dt_strides: fixture.dt_strides,
cb_strides: fixture.cb_strides,
@@ -251,14 +249,14 @@ fn run_prefill_kernel_mode(
kernel.encode(&mut encoder, args, mode);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let y_vec = read_buffer(&y_buf, fixture.total_x);
- let state_vec = read_buffer(&state_buf, fixture.total_state);
+ let y_vec = read_buffer::<B>(y_buf, fixture.total_x);
+ let state_vec = read_buffer::<B>(state_buf, fixture.total_state);
(y_vec, state_vec, None)
}
-fn run_conv_scan_once(
- ctx: &<Metal as Backend>::Context,
- kernel: &<<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel,
+fn run_conv_scan_once<B: Backend>(
+ ctx: &<B as Backend>::Context,
+ kernel: &<<B as Backend>::Kernels as Kernels>::Conv1dScanKernel,
suffix_len: usize,
channels: usize,
kernel_size: i32,
@@ -270,39 +268,40 @@ fn run_conv_scan_once(
use_scratch: bool,
alias_io: bool,
) -> (Vec<f32>, Vec<f32>) {
- let device = &ctx.device;
let total_x = suffix_len * channels;
let _total_w = channels * kernel_size as usize;
let total_state = channels * tap_count;
- let mut y_buf = if alias_io {
- device.new_buffer_with_data(bytemuck::cast_slice(x_data), STORAGE_MODE).expect("Failed to create buffer")
- } else {
- let buf = device.new_buffer(total_x * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");
- zero_buffer(&buf);
- buf
- };
- let w_buf =
- device.new_buffer_with_data(bytemuck::cast_slice(w_data), STORAGE_MODE).expect("Failed to create buffer");
- let b_buf =
- device.new_buffer_with_data(bytemuck::cast_slice(b_data), STORAGE_MODE).expect("Failed to create buffer");
- let mut state_buf =
- device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");
- let scratch_buf = if use_scratch && tap_count > 0 {
- Some(device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer"))
+ let y_array = if alias_io {
+ ctx.create_array_from(&[x_data.len()], x_data, "")
} else {
- None
+ ctx.create_array_zeros(&[total_x], DataType::F32, "")
};
+ let b_out_array = ctx.create_array_zeros(&[total_x], DataType::F32, "");
+ let c_out_array = ctx.create_array_zeros(&[total_x], DataType::F32, "");
+
+ let w_array = ctx.create_array_from(&[w_data.len()], w_data, "");
+ let b_array = ctx.create_array_from(&[b_data.len()], b_data, "");
- write_buffer(&state_buf, state_init);
- if let Some(ref scratch) = scratch_buf {
- zero_buffer(scratch);
+ let state_array = ctx.create_array_uninitialized(&[total_state], DataType::F32, "");
+ let scratch_array = ctx.create_array_uninitialized(&[total_state], DataType::F32, "");
+
+ {
+ let state_buf = state_array.buffer();
+ let state_borrow = state_buf.borrow();
+ write_buffer::<B>(state_borrow.deref(), state_init);
+ if use_scratch && tap_count > 0 {
+ let scratch_buf = scratch_array.buffer();
+ let scratch_borrow = scratch_buf.borrow();
+ zero_buffer::<B>(scratch_borrow.deref());
+ }
}
let padded_len = tap_count + suffix_len;
- let padded_buf =
- device.new_buffer(padded_len * channels * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");
+ let padded_array = ctx.create_array_uninitialized(&[padded_len * channels], DataType::F32, "");
{
+ let padded_buf = padded_array.buffer();
+ let padded_borrow = padded_buf.borrow();
let mut host = vec![0.0f32; padded_len * channels];
for tap in 0..tap_count {
for ch in 0..channels {
@@ -314,67 +313,89 @@ fn run_conv_scan_once(
host[(tap_count + token) * channels + ch] = x_data[token * channels + ch];
}
}
- write_buffer(&padded_buf, &host);
+ write_buffer::<B>(padded_borrow.deref(), &host);
}
let mut encoder = Encoder::new(ctx).unwrap();
- let mut b_out = y_buf.clone();
- let mut c_out = y_buf.clone();
- let mut state_out = scratch_buf.as_ref().unwrap_or(&state_buf).clone();
- kernel.encode(
- &padded_buf,
- &w_buf,
- Some(&b_buf),
- &mut y_buf,
- &mut b_out,
- &mut c_out,
- &mut state_out,
- suffix_len as u32,
- kernel_size as u32,
- channels as u32,
- tap_count as u32,
- channels as u32,
- channels as u32,
- 0u32,
- ActivationType::SILU,
- &mut encoder,
- );
+ {
+ let padded_buf = padded_array.buffer();
+ let padded_borrow = padded_buf.borrow();
+ let w_buf = w_array.buffer();
+ let w_borrow = w_buf.borrow();
+ let b_buf = b_array.buffer();
+ let b_borrow = b_buf.borrow();
+ let y_buf = y_array.buffer();
+ let mut y_borrow = y_buf.borrow_mut();
+ let b_out_buf = b_out_array.buffer();
+ let mut b_out_borrow = b_out_buf.borrow_mut();
+ let c_out_buf = c_out_array.buffer();
+ let mut c_out_borrow = c_out_buf.borrow_mut();
+ let state_buf = state_array.buffer();
+ let mut state_borrow = state_buf.borrow_mut();
+
+ kernel.encode(
+ padded_borrow.deref(),
+ w_borrow.deref(),
+ Some(b_borrow.deref()),
+ y_borrow.deref_mut(),
+ b_out_borrow.deref_mut(),
+ c_out_borrow.deref_mut(),
+ state_borrow.deref_mut(),
+ suffix_len as u32,
+ kernel_size as u32,
+ channels as u32,
+ tap_count as u32,
+ channels as u32,
+ channels as u32,
+ 0u32,
+ ActivationType::SILU,
+ &mut encoder,
+ );
+ }
- if let Some(ref scratch) = scratch_buf {
+ if use_scratch && tap_count > 0 {
let bytes = channels * tap_count * size_of::<f32>();
if bytes > 0 {
- encoder.encode_copy(scratch, 0..bytes, &mut state_buf, 0..bytes);
+ let scratch_buf = scratch_array.buffer();
+ let scratch_borrow = scratch_buf.borrow();
+ let state_buf = state_array.buffer();
+ let mut state_borrow = state_buf.borrow_mut();
+ encoder.encode_copy(scratch_borrow.deref(), 0..bytes, state_borrow.deref_mut(), 0..bytes);
}
}
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let y_vec = read_buffer(&y_buf, total_x);
- let state_vec = read_buffer(&state_buf, total_state);
+ let y_buf = y_array.buffer();
+ let y_borrow = y_buf.borrow();
+ let state_buf = state_array.buffer();
+ let state_borrow = state_buf.borrow();
+ let y_vec = read_buffer::<B>(y_borrow.deref(), total_x);
+ let state_vec = read_buffer::<B>(state_borrow.deref(), total_state);
(y_vec, state_vec)
}
-fn assert_deterministic_for_mode(mode: SSDPrefillMode) {
- let Some(ctx) = <Metal as Backend>::Context::new().ok() else {
+fn assert_deterministic_for_mode<B: Backend>(mode: SSDPrefillMode) {
+ let Some(ctx) = <B as Backend>::Context::new().ok() else {
eprintln!("Skipping SSD prefill determinism test: no Metal device");
return;
};
- let kernel = SSDPrefillKernels::<Metal>::new(&ctx, DataType::F32).unwrap();
+ let kernel = SSDPrefillKernels::<B>::new(&ctx, DataType::F32).unwrap();
let fixture = SSDPrefillFixture::new();
- let (y_a, state_a, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
- let (y_b, state_b, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
+ let (y_a, state_a, _) = run_prefill_kernel_mode::<B>(&ctx, &kernel, &fixture, mode);
+ let (y_b, state_b, _) = run_prefill_kernel_mode::<B>(&ctx, &kernel, &fixture, mode);
assert_eq!(y_a, y_b, "Prefill outputs differ in {:?} mode", mode);
assert_eq!(state_a, state_b, "Prefill states differ in {:?} mode", mode);
}
-fn assert_matches_cpu_reference(mode: SSDPrefillMode) {
- let Some(ctx) = <Metal as Backend>::Context::new().ok() else {
+fn assert_matches_cpu_reference<B: Backend>(mode: SSDPrefillMode) {
+ let Some(ctx) = <B as Backend>::Context::new().ok() else {
eprintln!("Skipping SSD prefill reference test: no Metal device");
return;
};
- let kernel = SSDPrefillKernels::<Metal>::new(&ctx, DataType::F32).unwrap();
+ let kernel = SSDPrefillKernels::<B>::new(&ctx, DataType::F32).unwrap();
let fixture = SSDPrefillFixture::new();
let (y_ref, state_ref) = ssd_prefill_cpu_reference(
@@ -396,7 +417,7 @@ fn assert_matches_cpu_reference(mode: SSDPrefillMode) {
fixture.state_strides,
);
- let (y_gpu, state_gpu, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
+ let (y_gpu, state_gpu, _) = run_prefill_kernel_mode::<B>(&ctx, &kernel, &fixture, mode);
let tolerance = 5e-5f32;
let mut max_y_diff = 0.0f32;
@@ -434,33 +455,12 @@ fn assert_matches_cpu_reference(mode: SSDPrefillMode) {
);
}
-#[test]
-fn ssd_prefill_sequential_is_deterministic() {
- assert_deterministic_for_mode(SSDPrefillMode::Sequential);
-}
-
-#[test]
-fn ssd_prefill_single_pass_is_deterministic() {
- assert_deterministic_for_mode(SSDPrefillMode::SinglePass);
-}
-
-#[test]
-fn ssd_prefill_sequential_matches_cpu_reference() {
- assert_matches_cpu_reference(SSDPrefillMode::Sequential);
-}
-
-#[test]
-fn ssd_prefill_single_pass_matches_cpu_reference() {
- assert_matches_cpu_reference(SSDPrefillMode::SinglePass);
-}
-
-#[test]
-fn conv1d_scan_is_deterministic() {
- let Some(ctx) = <Metal as Backend>::Context::new().ok() else {
+fn conv1d_scan_deterministic_internal<B: Backend>() {
+ let Some(ctx) = <B as Backend>::Context::new().ok() else {
eprintln!("Skipping conv1d scan determinism test: no Metal device");
return;
};
- let kernel = <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, true).unwrap();
+ let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, true).unwrap();
let suffix_len = 192usize;
let channels = 8usize;
@@ -478,7 +478,7 @@ fn conv1d_scan_is_deterministic() {
let use_scratch = tap_count > 0 && suffix_len > 1;
for &alias_io in &[false, true] {
- let first = run_conv_scan_once(
+ let first = run_conv_scan_once::<B>(
&ctx,
&kernel,
suffix_len,
@@ -492,7 +492,7 @@ fn conv1d_scan_is_deterministic() {
use_scratch,
alias_io,
);
- let second = run_conv_scan_once(
+ let second = run_conv_scan_once::<B>(
&ctx,
&kernel,
suffix_len,
@@ -511,3 +511,38 @@ fn conv1d_scan_is_deterministic() {
assert_eq!(first.1, second.1, "Conv states differ (alias_io={alias_io})");
}
}
+
+#[test]
+fn ssd_prefill_sequential_is_deterministic() {
+ for_each_non_cpu_backend!(|B| {
+ assert_deterministic_for_mode::<B>(SSDPrefillMode::Sequential);
+ });
+}
+
+#[test]
+fn ssd_prefill_single_pass_is_deterministic() {
+ for_each_non_cpu_backend!(|B| {
+ assert_deterministic_for_mode::<B>(SSDPrefillMode::SinglePass);
+ });
+}
+
+#[test]
+fn ssd_prefill_sequential_matches_cpu_reference() {
+ for_each_non_cpu_backend!(|B| {
+ assert_matches_cpu_reference::<B>(SSDPrefillMode::Sequential);
+ });
+}
+
+#[test]
+fn ssd_prefill_single_pass_matches_cpu_reference() {
+ for_each_non_cpu_backend!(|B| {
+ assert_matches_cpu_reference::<B>(SSDPrefillMode::SinglePass);
+ });
+}
+
+#[test]
+fn conv1d_scan_is_deterministic() {
+ for_each_non_cpu_backend!(|B| {
+ conv1d_scan_deterministic_internal::<B>();
+ });
+}
diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rs
index 7e47c316a..e9290f87c 100644
--- a/crates/uzu/tests/unit/kernel/mod.rs
+++ b/crates/uzu/tests/unit/kernel/mod.rs
@@ -18,6 +18,7 @@ mod rms_norm_test;
mod rope_test;
mod sampling;
mod short_conv;
+mod ssm;
mod tensor_add_bias_test;
mod tensor_add_scale_test;
mod tensor_add_swap_test;
diff --git a/crates/uzu/tests/unit/kernel/ssd_update_test.rs b/crates/uzu/tests/unit/kernel/ssd_update_test.rs
deleted file mode 100644
index 84c885c75..000000000
--- a/crates/uzu/tests/unit/kernel/ssd_update_test.rs
+++ /dev/null
@@ -1,281 +0,0 @@
-#![cfg(metal_backend)]
-
-use bytemuck;
-use half::bf16;
-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
-use uzu::{
- DataType,
- backends::{
- common::{
- Backend, Context, Encoder, Kernels, kernel::SSDUpdateKernel,
- },
- metal::Metal,
- },
-};
-
-#[allow(dead_code)]
-fn ssm_update_ref_bf16(
- x: &[bf16],
- dt: &[bf16],
- a: &[bf16],
- b: &[bf16],
- c: &[bf16],
- d: &[bf16],
- z: &[bf16],
- state: &[bf16],
- bsz: usize,
- h: usize,
-) -> (Vec<bf16>, Vec<bf16>) {
- let n: usize = 16;
- let mut y = vec![bf16::from_f32(0.0); bsz * h];
- let mut next_state = vec![bf16::from_f32(0.0); bsz * h * n];
- for bb in 0..bsz {
- for ch in 0..h {
- let idx = bb * h + ch;
- let vx = x[idx].to_f32();
- let sig = 1.0 / (1.0 + (-vx.abs()).exp());
- let this_x = if vx < 0.0 {
- (1.0 - sig) * vx
- } else {
- sig * vx
- };
- let vz = z[idx].to_f32();
- let sigz = 1.0 / (1.0 + (-vz.abs()).exp());
- let this_z = if vz < 0.0 {
- (1.0 - sigz) * vz
- } else {
- sigz * vz
- };
- let delta = (1.0 + dt[idx].to_f32().exp()).ln();
- let cb_offset = bb * n;
- let state_offset = idx * n;
- let mut acc = 0.0f32;
- for i in 0..n {
- let new_s = state[state_offset + i].to_f32() * (a[i].to_f32() * delta).exp()
- + b[cb_offset + i].to_f32() * delta * this_x;
- next_state[state_offset + i] = bf16::from_f32(new_s);
- acc += new_s * c[cb_offset + i].to_f32();
- }
- acc += d[ch].to_f32() * this_x;
- acc *= this_z;
- y[idx] = bf16::from_f32(acc);
- }
- }
- (y, next_state)
-}
-
-fn softplus_f32(x: f32) -> f32 {
- if x > 20.0 {
- x
- } else {
- (1.0 + x.exp()).ln()
- }
-}
-
-#[allow(dead_code)]
-fn ssd_update_no_z_ref_bf16(
- x: &[bf16],
- dt_raw: &[bf16],
- b: &[bf16],
- c: &[bf16],
- d: &[bf16],
- state: &[bf16],
- bsz: usize,
- h: usize,
- dh: usize,
- g: usize,
- n: usize,
-) -> (Vec<bf16>, Vec<bf16>) {
- let group_size = h / g;
- let mut y = vec![bf16::from_f32(0.0); bsz * h * dh];
- let mut next_state = vec![bf16::from_f32(0.0); bsz * h * dh * n];
- for bb in 0..bsz {
- for hh in 0..h {
- for ddh in 0..dh {
- let x_idx = bb * h * dh + hh * dh + ddh;
- let dt_idx = bb * h + hh;
- let cb_idx0 = bb * g * n + (hh / group_size) * n;
- let state_idx0 = (bb * h * dh + hh * dh + ddh) * n;
- let this_x = x[x_idx].to_f32();
- let dt_raw_val = dt_raw[dt_idx].to_f32();
- let this_dt = softplus_f32(dt_raw_val);
- let this_decay = (-this_dt).exp();
- let this_d = d[hh].to_f32();
- let dt_safe = this_dt.max(1e-6);
- let normalized_x = this_x / dt_safe;
- let dt_scaled_input = normalized_x * this_dt;
- let mut acc = 0.0f32;
- for i in 0..n {
- let s_old = state[state_idx0 + i].to_f32();
- let new_s = s_old * this_decay + b[cb_idx0 + i].to_f32() * dt_scaled_input;
- next_state[state_idx0 + i] = bf16::from_f32(new_s);
- acc += new_s * c[cb_idx0 + i].to_f32();
- }
- acc += this_d * this_x;
- y[x_idx] = bf16::from_f32(acc);
- }
- }
- }
- (y, next_state)
-}
-
-fn ssd_update_ref_bf16(
- x: &[bf16],
- dt_raw: &[bf16],
- b: &[bf16],
- c: &[bf16],
- d: &[bf16],
- z: &[bf16],
- state: &[bf16],
- bsz: usize,
- h: usize,
- dh: usize,
- g: usize,
- n: usize,
-) -> (Vec<bf16>, Vec<bf16>) {
- let group_size = h / g;
- let mut y = vec![bf16::from_f32(0.0); bsz * h * dh];
- let mut next_state = vec![bf16::from_f32(0.0); bsz * h * dh * n];
- for bb in 0..bsz {
- for hh in 0..h {
- for ddh in 0..dh {
- let x_idx = bb * h * dh + hh * dh + ddh;
- let dt_idx = bb * h + hh;
- let cb_idx0 = bb * g * n + (hh / group_size) * n;
- let state_idx0 = (bb * h * dh + hh * dh + ddh) * n;
- let this_x = x[x_idx].to_f32();
- let dt_raw_val = dt_raw[dt_idx].to_f32();
- let this_dt = softplus_f32(dt_raw_val);
- let this_decay = (-this_dt).exp();
- let this_d = d[hh].to_f32();
- let dt_safe = this_dt.max(1e-6);
- let normalized_x = this_x / dt_safe;
- let dt_scaled_input = normalized_x * this_dt;
- let vz = z[x_idx].to_f32();
- let sigz = 1.0 / (1.0 + (-vz.abs()).exp());
- let this_z = if vz < 0.0 {
- (1.0 - sigz) * vz
- } else {
- sigz * vz
- };
- let mut acc = 0.0f32;
- for i in 0..n {
- let s_old = state[state_idx0 + i].to_f32();
- let new_s = s_old * this_decay + b[cb_idx0 + i].to_f32() * dt_scaled_input;
- next_state[state_idx0 + i] = bf16::from_f32(new_s);
- acc += new_s * c[cb_idx0 + i].to_f32();
- }
- acc += this_d * this_x;
- acc *= this_z;
- y[x_idx] = bf16::from_f32(acc);
- }
- }
- }
- (y, next_state)
-}
-
-#[test]
-fn ssd_update_with_z_bf16() {
- let Some(ctx) = <Metal as Backend>::Context::new().ok() else {
- eprintln!("Skipping: no Metal device");
- return;
- };
- let bsz = 1usize;
- let h = 4usize;
- let dh = 3usize;
- let g = 2usize;
- let n = 8usize;
-
- let x: Vec<bf16> = (0..bsz * h * dh).map(|i| bf16::from_f32(((i % 7) as f32) * 0.1 - 0.2)).collect();
- let z: Vec<bf16> = (0..bsz * h * dh).map(|i| bf16::from_f32(((i % 5) as f32) * 0.1 - 0.1)).collect();
- let dt: Vec<bf16> = (0..bsz * h).map(|i| bf16::from_f32(((i % 5) as f32) * 0.3 - 1.0)).collect();
- let b: Vec<bf16> = (0..bsz * g * n).map(|i| bf16::from_f32(((i % 11) as f32) * 0.02 - 0.05)).collect();
- let c: Vec<bf16> = (0..bsz * g * n).map(|i| bf16::from_f32(((i % 13) as f32) * 0.015)).collect();
- let d: Vec<bf16> = (0..h).map(|i| bf16::from_f32(((i % 3) as f32) * 0.05)).collect();
- let state: Vec<bf16> = (0..bsz * h * dh * n).map(|i| bf16::from_f32(((i % 23) as f32) * 0.01 - 0.05)).collect();
-
- let (y_exp, ns_exp) = ssd_update_ref_bf16(&x, &dt, &b, &c, &d, &z, &state, bsz, h, dh, g, n);
- let y_exp_f32: Vec<f32> = y_exp.iter().map(|&v| v.to_f32()).collect();
- let ns_exp_f32: Vec<f32> = ns_exp.iter().map(|&v| v.to_f32()).collect();
-
- let x_strides = [(h * dh) as u32, dh as u32, 1u32];
- let dt_strides = [h as u32, 1u32];
- let cb_strides = [(g * n) as u32, n as u32, 1u32];
- let state_strides = [(h * dh * n) as u32, (dh * n) as u32, n as u32, 1u32];
-
- let x_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&x), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let dt_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&dt), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let b_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&b), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let c_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&c), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let d_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&d), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let z_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&z), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let state_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&state), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let mut y_buf = ctx
- .device
- .new_buffer(bsz * h * dh * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let mut ns_buf = ctx
- .device
- .new_buffer(bsz * h * dh * n * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let kernel = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&ctx, DataType::BF16, false).unwrap();
- let mut encoder = Encoder::new(ctx.as_ref()).unwrap();
- kernel.encode(
- &x_buf,
- &dt_buf,
- &b_buf,
- &c_buf,
- &d_buf,
- &z_buf,
- Some(&state_buf),
- &mut y_buf,
- &mut ns_buf,
- (h / g) as u32,
- n as u32,
- x_strides.as_slice(),
- dt_strides.as_slice(),
- cb_strides.as_slice(),
- state_strides.as_slice(),
- bsz as u32,
- h as u32,
- dh as u32,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- let y_ptr = y_buf.contents().as_ptr() as *const bf16;
- let y_out = unsafe { std::slice::from_raw_parts(y_ptr, bsz * h * dh) };
- let y_out_f32: Vec<f32> = y_out.iter().map(|&v| v.to_f32()).collect();
- let ns_ptr = ns_buf.contents().as_ptr() as *const bf16;
- let ns_out = unsafe { std::slice::from_raw_parts(ns_ptr, bsz * h * dh * n) };
- let ns_out_f32: Vec<f32> = ns_out.iter().map(|&v| v.to_f32()).collect();
- let tol = 2e-2;
- for (i, (a, b)) in y_out_f32.iter().zip(y_exp_f32.iter()).enumerate() {
- assert!((a - b).abs() < tol, "y {} {} {}", i, a, b);
- }
- for (i, (a, b)) in ns_out_f32.iter().zip(ns_exp_f32.iter()).enumerate() {
- assert!((a - b).abs() < tol, "ns {} {} {}", i, a, b);
- }
-}
diff --git a/crates/uzu/tests/unit/kernel/ssm/mod.rs b/crates/uzu/tests/unit/kernel/ssm/mod.rs
new file mode 100644
index 000000000..f704533c4
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/ssm/mod.rs
@@ -0,0 +1,6 @@
+mod ssm_conv1d_decode_test;
+mod ssm_conv1d_pack_test;
+mod ssm_conv1d_scan_test;
+mod ssm_prefill_test;
+mod ssm_split_inproj_test;
+mod ssm_update_test;
diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs
new file mode 100644
index 000000000..40210ca72
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs
@@ -0,0 +1,348 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::Conv1dDecodeKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ x: Box<[T]>,
+ w: Box<[T]>,
+ b: Option<Box<[T]>>,
+ state: Box<[T]>,
+ kernel_size: u32,
+ row_stride: u32,
+ state_stride: u32,
+ num_channels: u32,
+ suffix_len: u32,
+ inner_dim: u32,
+ proj_dim: u32,
+ activation_type: ActivationType,
+ state_in_place: bool,
+}
+
+struct Output<T: ArrayElement + Float> {
+ x_out: Vec<T>,
+ b_out: Vec<T>,
+ c_out: Vec<T>,
+ next_state: Vec<T>,
+}
+
+fn get_input<T: ArrayElement + Float>(
+ kernel_size: u32,
+ num_channels: u32,
+ inner_dim: u32,
+ proj_dim: u32,
+ suffix_len: u32,
+ has_bias: bool,
+ state_in_place: bool,
+ activation_type: ActivationType,
+) -> Input<T> {
+ let row_stride = num_channels;
+ let state_stride = kernel_size.saturating_sub(1);
+ let state_size = num_channels as usize * state_stride as usize;
+ let x_size = suffix_len as usize * row_stride as usize;
+ let w_size = num_channels as usize * kernel_size as usize;
+
+ let x: Vec<T> = (0..x_size).map(|i| T::from(0.1 * ((i as f64 % 7.0) - 3.0)).unwrap()).collect();
+ let w: Vec<T> = (0..w_size).map(|i| T::from(0.05 * ((i as f64 % 5.0) - 2.0)).unwrap()).collect();
+ let b = if has_bias {
+ Some(
+ (0..num_channels as usize)
+ .map(|i| T::from(0.01 * (i as f64 + 1.0)).unwrap())
+ .collect::<Vec<T>>()
+ .into_boxed_slice(),
+ )
+ } else {
+ None
+ };
+ let state: Vec<T> = (0..state_size).map(|i| T::from(0.02 * ((i as f64 % 9.0) - 4.0)).unwrap()).collect();
+
+ Input {
+ x: x.into_boxed_slice(),
+ w: w.into_boxed_slice(),
+ b,
+ state: state.into_boxed_slice(),
+ kernel_size,
+ row_stride,
+ state_stride,
+ num_channels,
+ suffix_len,
+ inner_dim,
+ proj_dim,
+ activation_type,
+ state_in_place,
+ }
+}
+
+fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let has_bias = input.b.is_some();
+ let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dDecodeKernel::new(
+ &context,
+ T::data_type(),
+ has_bias,
+ input.state_in_place,
+ )
+ .expect("Failed to create Conv1dDecodeKernel");
+
+ let x_out_size = input.suffix_len as usize * input.inner_dim as usize;
+ let b_out_size = input.suffix_len as usize * input.proj_dim as usize;
+ let c_out_size = input.suffix_len as usize * input.proj_dim as usize;
+ let state_size = input.num_channels as usize * input.state_stride as usize;
+
+ let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");
+ let w_array = context.create_array_from(&[input.w.len()], &input.w, "w");
+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, "b"));
+
+ let x_out_array = context.create_array_uninitialized(&[x_out_size], T::data_type(), "x_out");
+ let b_out_array = context.create_array_uninitialized(&[b_out_size], T::data_type(), "b_out");
+ let c_out_array = context.create_array_uninitialized(&[c_out_size], T::data_type(), "c_out");
+
+ let b_buf = b_array.as_ref().map(|a| a.buffer());
+ let b_borrow = b_buf.as_ref().map(|rc| rc.borrow());
+ let b_deref: Option<&B::Buffer> = b_borrow.as_ref().map(|b| b.deref());
+
+ if input.state_in_place {
+ let next_state_array = context.create_array_from(&[state_size], &input.state, "next_state");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ x_array.buffer().borrow().deref(),
+ w_array.buffer().borrow().deref(),
+ b_deref,
+ None::<&B::Buffer>,
+ x_out_array.buffer().borrow_mut().deref_mut(),
+ b_out_array.buffer().borrow_mut().deref_mut(),
+ c_out_array.buffer().borrow_mut().deref_mut(),
+ next_state_array.buffer().borrow_mut().deref_mut(),
+ input.kernel_size,
+ input.row_stride,
+ input.state_stride,
+ input.num_channels,
+ input.suffix_len,
+ input.inner_dim,
+ input.proj_dim,
+ input.activation_type,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ Output {
+ x_out: x_out_array.as_slice().to_vec(),
+ b_out: b_out_array.as_slice().to_vec(),
+ c_out: c_out_array.as_slice().to_vec(),
+ next_state: next_state_array.as_slice().to_vec(),
+ }
+ } else {
+ let state_array = context.create_array_from(&[state_size], &input.state, "state");
+ let next_state_array = context.create_array_uninitialized(&[state_size], T::data_type(), "next_state");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ x_array.buffer().borrow().deref(),
+ w_array.buffer().borrow().deref(),
+ b_deref,
+ Some(state_array.buffer().borrow().deref()),
+ x_out_array.buffer().borrow_mut().deref_mut(),
+ b_out_array.buffer().borrow_mut().deref_mut(),
+ c_out_array.buffer().borrow_mut().deref_mut(),
+ next_state_array.buffer().borrow_mut().deref_mut(),
+ input.kernel_size,
+ input.row_stride,
+ input.state_stride,
+ input.num_channels,
+ input.suffix_len,
+ input.inner_dim,
+ input.proj_dim,
+ input.activation_type,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ Output {
+ x_out: x_out_array.as_slice().to_vec(),
+ b_out: b_out_array.as_slice().to_vec(),
+ c_out: c_out_array.as_slice().to_vec(),
+ next_state: next_state_array.as_slice().to_vec(),
+ }
+ }
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ kernel_size: u32,
+ num_channels: u32,
+ inner_dim: u32,
+ proj_dim: u32,
+ suffix_len: u32,
+ has_bias: bool,
+ state_in_place: bool,
+ activation_type: ActivationType,
+) -> (Input<T>, Output<T>) {
+ let input = get_input::<T>(
+ kernel_size,
+ num_channels,
+ inner_dim,
+ proj_dim,
+ suffix_len,
+ has_bias,
+ state_in_place,
+ activation_type,
+ );
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &Output<T>,
+ label: &str,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-2f32
+ } else {
+ 1e-5
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input);
+ let backend_name = std::any::type_name::<B>();
+ let type_name = std::any::type_name::<T>();
+
+ assert_eq_float::<T>(
+ &expected.x_out,
+ &output.x_out,
+ eps,
+ &format!("Conv1dDecode x_out {backend_name} {label} (type={type_name})"),
+ );
+ assert_eq_float::<T>(
+ &expected.b_out,
+ &output.b_out,
+ eps,
+ &format!("Conv1dDecode b_out {backend_name} {label} (type={type_name})"),
+ );
+ assert_eq_float::<T>(
+ &expected.c_out,
+ &output.c_out,
+ eps,
+ &format!("Conv1dDecode c_out {backend_name} {label} (type={type_name})"),
+ );
+ assert_eq_float::<T>(
+ &expected.next_state,
+ &output.next_state,
+ eps,
+ &format!("Conv1dDecode next_state {backend_name} {label} (type={type_name})"),
+ );
+ });
+}
+
+// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4
+fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+ for state_in_place in [true, false] {
+ for has_bias in [true, false] {
+ let label = format!("basic(bias={has_bias},in_place={state_in_place})");
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, has_bias, state_in_place, ActivationType::SILU);
+ test_internal(&input, &expected, &label);
+ }
+ }
+}
+
+// kernel_size=2 (state_stride=1, minimal state)
+fn test_small_kernel<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 1, true, true, ActivationType::SILU);
+ test_internal(&input, &expected, "small_kernel");
+}
+
+// Larger dimensions closer to real usage
+fn test_large<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 1, true, true, ActivationType::SILU);
+ test_internal(&input, &expected, "large");
+}
+
+fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, false, true, ActivationType::IDENTITY);
+ test_internal(&input, &expected, "identity_activation");
+}
+
+fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::GELU);
+ test_internal(&input, &expected, "gelu_activation");
+}
+
+fn test_tanh_activation<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::TANH);
+ test_internal(&input, &expected, "tanh_activation");
+}
+
+// f32
+#[test]
+fn test_basic_f32() {
+ test_basic::<f32>();
+}
+
+#[test]
+fn test_small_kernel_f32() {
+ test_small_kernel::<f32>();
+}
+
+#[test]
+fn test_large_f32() {
+ test_large::<f32>();
+}
+
+#[test]
+fn test_identity_activation_f32() {
+ test_identity_activation::<f32>();
+}
+
+#[test]
+fn test_gelu_activation_f32() {
+ test_gelu_activation::<f32>();
+}
+
+#[test]
+fn test_tanh_activation_f32() {
+ test_tanh_activation::<f32>();
+}
+
+// f16
+#[test]
+fn test_basic_f16() {
+ test_basic::<f16>();
+}
+
+#[test]
+fn test_small_kernel_f16() {
+ test_small_kernel::<f16>();
+}
+
+#[test]
+fn test_large_f16() {
+ test_large::<f16>();
+}
+
+// bf16
+#[test]
+fn test_basic_bf16() {
+ test_basic::<bf16>();
+}
+
+#[test]
+fn test_small_kernel_bf16() {
+ test_small_kernel::<bf16>();
+}
+
+#[test]
+fn test_large_bf16() {
+ test_large::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs
new file mode 100644
index 000000000..29d026dcd
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs
@@ -0,0 +1,204 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::Conv1dPackKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ state_in: Box<[T]>,
+ x: Box<[T]>,
+ state_stride: u32,
+ row_stride: u32,
+ suffix_len: u32,
+ num_channels: u32,
+}
+
+fn get_input<T: ArrayElement + Float>(
+ state_stride: u32,
+ row_stride: u32,
+ suffix_len: u32,
+ num_channels: u32,
+) -> Input<T> {
+ let state_size = num_channels as usize * state_stride as usize;
+ let x_size = suffix_len as usize * row_stride as usize;
+
+ let state_in: Vec<T> = (0..state_size).map(|i| T::from(0.1 * (i as f64 + 1.0)).unwrap()).collect();
+ let x: Vec<T> = (0..x_size).map(|i| T::from(-0.05 * (i as f64 + 1.0)).unwrap()).collect();
+
+ Input {
+ state_in: state_in.into_boxed_slice(),
+ x: x.into_boxed_slice(),
+ state_stride,
+ row_stride,
+ suffix_len,
+ num_channels,
+ }
+}
+
+fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, T::data_type())
+ .expect("Failed to create Conv1dPackKernel");
+
+ let state_size = input.state_in.len();
+ let x_size = input.x.len();
+ let total_rows = input.state_stride as usize + input.suffix_len as usize;
+ let padded_size = total_rows * input.row_stride as usize;
+
+ let state_array = context.create_array_from(&[state_size], &input.state_in, "state_in");
+ let x_array = context.create_array_from(&[x_size], &input.x, "x");
+ let padded_array = context.create_array_uninitialized(&[padded_size], T::data_type(), "padded");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ state_array.buffer().borrow().deref(),
+ x_array.buffer().borrow().deref(),
+ padded_array.buffer().borrow_mut().deref_mut(),
+ input.state_stride,
+ input.row_stride,
+ input.suffix_len,
+ input.num_channels,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ padded_array.as_slice().to_vec()
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ state_stride: u32,
+ row_stride: u32,
+ suffix_len: u32,
+ num_channels: u32,
+) -> (Input<T>, Vec<T>) {
+ let input = get_input::<T>(state_stride, row_stride, suffix_len, num_channels);
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &[T],
+ label: &str,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-3f32
+ } else {
+ 1e-6
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input);
+ let msg = format!("Conv1dPack {} {} (type={})", std::any::type_name::<B>(), label, std::any::type_name::<T>(),);
+ assert_eq_float::<T>(expected, &output, eps, &msg);
+ });
+}
+
+fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+ // 4 channels, state_stride=3 (kernel_size=4), 2 tokens
+ let (input, expected) = get_test_data::<T>(3, 4, 2, 4);
+ test_internal(&input, &expected, "basic");
+}
+
+fn test_single_token<T: ArrayElement + Float + Debug + Display>() {
+ // 8 channels, state_stride=1, 1 token
+ let (input, expected) = get_test_data::<T>(1, 8, 1, 8);
+ test_internal(&input, &expected, "single_token");
+}
+
+fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {
+ // Minimal state (kernel_size=2), many tokens
+ let (input, expected) = get_test_data::<T>(1, 4, 8, 4);
+ test_internal(&input, &expected, "many_tokens");
+}
+
+fn test_large<T: ArrayElement + Float + Debug + Display>() {
+ // Larger dimensions closer to real usage
+ let (input, expected) = get_test_data::<T>(3, 128, 16, 128);
+ test_internal(&input, &expected, "large");
+}
+
+fn test_channels_lt_row_stride<T: ArrayElement + Float + Debug + Display>() {
+ // num_channels < row_stride (padded channels that kernel doesn't write)
+ let (input, expected) = get_test_data::<T>(2, 8, 3, 6);
+ test_internal(&input, &expected, "channels_lt_row_stride");
+}
+
+// f32
+#[test]
+fn test_basic_f32() {
+ test_basic::<f32>();
+}
+
+#[test]
+fn test_single_token_f32() {
+ test_single_token::<f32>();
+}
+
+#[test]
+fn test_many_tokens_f32() {
+ test_many_tokens::<f32>();
+}
+
+#[test]
+fn test_large_f32() {
+ test_large::<f32>();
+}
+
+#[test]
+fn test_channels_lt_row_stride_f32() {
+ test_channels_lt_row_stride::<f32>();
+}
+
+// f16
+#[test]
+fn test_basic_f16() {
+ test_basic::<f16>();
+}
+
+#[test]
+fn test_single_token_f16() {
+ test_single_token::<f16>();
+}
+
+#[test]
+fn test_many_tokens_f16() {
+ test_many_tokens::<f16>();
+}
+
+#[test]
+fn test_large_f16() {
+ test_large::<f16>();
+}
+
+// bf16
+#[test]
+fn test_basic_bf16() {
+ test_basic::<bf16>();
+}
+
+#[test]
+fn test_single_token_bf16() {
+ test_single_token::<bf16>();
+}
+
+#[test]
+fn test_many_tokens_bf16() {
+ test_many_tokens::<bf16>();
+}
+
+#[test]
+fn test_large_bf16() {
+ test_large::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs
new file mode 100644
index 000000000..9d249dd38
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs
@@ -0,0 +1,331 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::Conv1dScanKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ padded: Box<[T]>,
+ w: Box<[T]>,
+ b: Option<Box<[T]>>,
+ suffix_len: u32,
+ kernel_size: u32,
+ row_stride: u32,
+ state_stride: u32,
+ num_channels: u32,
+ inner_dim: u32,
+ proj_dim: u32,
+ activation_type: ActivationType,
+}
+
+struct Output<T: ArrayElement + Float> {
+ x_out: Vec<T>,
+ b_out: Vec<T>,
+ c_out: Vec<T>,
+ state_out: Vec<T>,
+}
+
+fn get_input<T: ArrayElement + Float>(
+ kernel_size: u32,
+ num_channels: u32,
+ inner_dim: u32,
+ proj_dim: u32,
+ suffix_len: u32,
+ has_bias: bool,
+ activation_type: ActivationType,
+) -> Input<T> {
+ let row_stride = num_channels;
+ let state_stride = kernel_size.saturating_sub(1);
+ let total_rows = (suffix_len + state_stride) as usize;
+ let padded_size = total_rows * row_stride as usize;
+ let w_size = num_channels as usize * kernel_size as usize;
+
+ let padded: Vec<T> = (0..padded_size).map(|i| T::from(0.1 * ((i as f64 % 11.0) - 5.0)).unwrap()).collect();
+ let w: Vec<T> = (0..w_size).map(|i| T::from(0.05 * ((i as f64 % 5.0) - 2.0)).unwrap()).collect();
+ let b = if has_bias {
+ Some(
+ (0..num_channels as usize)
+ .map(|i| T::from(0.01 * (i as f64 + 1.0)).unwrap())
+ .collect::<Vec<T>>()
+ .into_boxed_slice(),
+ )
+ } else {
+ None
+ };
+
+ Input {
+ padded: padded.into_boxed_slice(),
+ w: w.into_boxed_slice(),
+ b,
+ suffix_len,
+ kernel_size,
+ row_stride,
+ state_stride,
+ num_channels,
+ inner_dim,
+ proj_dim,
+ activation_type,
+ }
+}
+
+fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let has_bias = input.b.is_some();
+ let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&context, T::data_type(), has_bias)
+ .expect("Failed to create Conv1dScanKernel");
+
+ let x_out_size = input.suffix_len as usize * input.inner_dim as usize;
+ let b_out_size = input.suffix_len as usize * input.proj_dim as usize;
+ let c_out_size = input.suffix_len as usize * input.proj_dim as usize;
+ let state_size = input.num_channels as usize * input.state_stride as usize;
+
+ let padded_array = context.create_array_from(&[input.padded.len()], &input.padded, "padded");
+ let w_array = context.create_array_from(&[input.w.len()], &input.w, "w");
+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, "b"));
+
+ let x_out_array = context.create_array_uninitialized(&[x_out_size], T::data_type(), "x_out");
+ let b_out_array = context.create_array_uninitialized(&[b_out_size], T::data_type(), "b_out");
+ let c_out_array = context.create_array_uninitialized(&[c_out_size], T::data_type(), "c_out");
+ let state_out_array = context.create_array_uninitialized(&[state_size], T::data_type(), "state_out");
+
+ let b_buf = b_array.as_ref().map(|a| a.buffer());
+ let b_borrow = b_buf.as_ref().map(|rc| rc.borrow());
+ let b_deref: Option<&B::Buffer> = b_borrow.as_ref().map(|b| b.deref());
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ padded_array.buffer().borrow().deref(),
+ w_array.buffer().borrow().deref(),
+ b_deref,
+ x_out_array.buffer().borrow_mut().deref_mut(),
+ b_out_array.buffer().borrow_mut().deref_mut(),
+ c_out_array.buffer().borrow_mut().deref_mut(),
+ state_out_array.buffer().borrow_mut().deref_mut(),
+ input.suffix_len,
+ input.kernel_size,
+ input.row_stride,
+ input.state_stride,
+ input.num_channels,
+ input.inner_dim,
+ input.proj_dim,
+ input.activation_type,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ Output {
+ x_out: x_out_array.as_slice().to_vec(),
+ b_out: b_out_array.as_slice().to_vec(),
+ c_out: c_out_array.as_slice().to_vec(),
+ state_out: state_out_array.as_slice().to_vec(),
+ }
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ kernel_size: u32,
+ num_channels: u32,
+ inner_dim: u32,
+ proj_dim: u32,
+ suffix_len: u32,
+ has_bias: bool,
+ activation_type: ActivationType,
+) -> (Input<T>, Output<T>) {
+ let input = get_input::<T>(kernel_size, num_channels, inner_dim, proj_dim, suffix_len, has_bias, activation_type);
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &Output<T>,
+ label: &str,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-2f32
+ } else {
+ 1e-5
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input);
+ let backend_name = std::any::type_name::<B>();
+ let type_name = std::any::type_name::<T>();
+
+ assert_eq_float::<T>(
+ &expected.x_out,
+ &output.x_out,
+ eps,
+ &format!("Conv1dScan x_out {backend_name} {label} (type={type_name})"),
+ );
+ assert_eq_float::<T>(
+ &expected.b_out,
+ &output.b_out,
+ eps,
+ &format!("Conv1dScan b_out {backend_name} {label} (type={type_name})"),
+ );
+ assert_eq_float::<T>(
+ &expected.c_out,
+ &output.c_out,
+ eps,
+ &format!("Conv1dScan c_out {backend_name} {label} (type={type_name})"),
+ );
+ assert_eq_float::<T>(
+ &expected.state_out,
+ &output.state_out,
+ eps,
+ &format!("Conv1dScan state_out {backend_name} {label} (type={type_name})"),
+ );
+ });
+}
+
+// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4, suffix_len=2
+fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+ for has_bias in [true, false] {
+ let label = format!("basic(bias={has_bias})");
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, has_bias, ActivationType::SILU);
+ test_internal(&input, &expected, &label);
+ }
+}
+
+// kernel_size=2 (state_stride=1, minimal state)
+fn test_small_kernel<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 3, true, ActivationType::SILU);
+ test_internal(&input, &expected, "small_kernel");
+}
+
+// Single token suffix
+fn test_single_token<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, ActivationType::SILU);
+ test_internal(&input, &expected, "single_token");
+}
+
+// Many tokens
+fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 16, true, ActivationType::SILU);
+ test_internal(&input, &expected, "many_tokens");
+}
+
+// Larger dimensions closer to real usage
+fn test_large<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 8, true, ActivationType::SILU);
+ test_internal(&input, &expected, "large");
+}
+
+fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, false, ActivationType::IDENTITY);
+ test_internal(&input, &expected, "identity_activation");
+}
+
+fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::GELU);
+ test_internal(&input, &expected, "gelu_activation");
+}
+
+fn test_tanh_activation<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::TANH);
+ test_internal(&input, &expected, "tanh_activation");
+}
+
+// f32
+#[test]
+fn test_basic_f32() {
+ test_basic::<f32>();
+}
+
+#[test]
+fn test_small_kernel_f32() {
+ test_small_kernel::<f32>();
+}
+
+#[test]
+fn test_single_token_f32() {
+ test_single_token::<f32>();
+}
+
+#[test]
+fn test_many_tokens_f32() {
+ test_many_tokens::<f32>();
+}
+
+#[test]
+fn test_large_f32() {
+ test_large::<f32>();
+}
+
+#[test]
+fn test_identity_activation_f32() {
+ test_identity_activation::<f32>();
+}
+
+#[test]
+fn test_gelu_activation_f32() {
+ test_gelu_activation::<f32>();
+}
+
+#[test]
+fn test_tanh_activation_f32() {
+ test_tanh_activation::<f32>();
+}
+
+// f16
+#[test]
+fn test_basic_f16() {
+ test_basic::<f16>();
+}
+
+#[test]
+fn test_small_kernel_f16() {
+ test_small_kernel::<f16>();
+}
+
+#[test]
+fn test_single_token_f16() {
+ test_single_token::<f16>();
+}
+
+#[test]
+fn test_many_tokens_f16() {
+ test_many_tokens::<f16>();
+}
+
+#[test]
+fn test_large_f16() {
+ test_large::<f16>();
+}
+
+// bf16
+#[test]
+fn test_basic_bf16() {
+ test_basic::<bf16>();
+}
+
+#[test]
+fn test_small_kernel_bf16() {
+ test_small_kernel::<bf16>();
+}
+
+#[test]
+fn test_single_token_bf16() {
+ test_single_token::<bf16>();
+}
+
+#[test]
+fn test_many_tokens_bf16() {
+ test_many_tokens::<bf16>();
+}
+
+#[test]
+fn test_large_bf16() {
+ test_large::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs
new file mode 100644
index 000000000..dc2b25316
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs
@@ -0,0 +1,290 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{
+ Backend, Context, Encoder, Kernels,
+ kernel::{SSDPrefillKernel, SSDPrefillSequentialKernel},
+ },
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ x: Box<[T]>,
+ dt: Box<[T]>,
+ b: Box<[T]>,
+ c: Box<[T]>,
+ d: Box<[T]>,
+ z: Box<[T]>,
+ state: Box<[T]>,
+ suffix_len: usize,
+ num_heads: usize,
+ head_dim: usize,
+ state_dim: usize,
+ group_size: u32,
+ x_strides: [usize; 3],
+ dt_strides: [usize; 2],
+ cb_strides: [usize; 3],
+ state_strides: [usize; 3],
+}
+
+struct Output<T: ArrayElement + Float> {
+ y: Vec<T>,
+ state: Vec<T>,
+}
+
+enum KernelType {
+ Prefill,
+ Sequential,
+}
+
+fn get_input<T: ArrayElement + Float>(
+ suffix_len: usize,
+ num_heads: usize,
+ head_dim: usize,
+ state_dim: usize,
+ group_size: u32,
+) -> Input<T> {
+ let safe_group = group_size.max(1) as usize;
+ let group_count = num_heads / safe_group;
+ let total_x = suffix_len * num_heads * head_dim;
+ let total_dt = suffix_len * num_heads;
+ let total_cb = suffix_len * group_count * state_dim;
+ let total_state = num_heads * head_dim * state_dim;
+
+ let x: Vec<T> = (0..total_x).map(|i| T::from(((i % 17) as f64) * 0.01 - 0.05).unwrap()).collect();
+ let dt: Vec<T> = (0..total_dt).map(|i| T::from(((i % 13) as f64) * 0.2 - 1.5).unwrap()).collect();
+ let b: Vec<T> = (0..total_cb).map(|i| T::from(((i % 11) as f64) * 0.02 - 0.05).unwrap()).collect();
+ let c: Vec<T> = (0..total_cb).map(|i| T::from(((i % 19) as f64) * 0.01 - 0.02).unwrap()).collect();
+ let d: Vec<T> = (0..num_heads).map(|i| T::from(((i % 3) as f64) * 0.05 - 0.05).unwrap()).collect();
+ let z: Vec<T> = (0..total_x).map(|i| T::from(((i % 23) as f64) * 0.02 - 0.1).unwrap()).collect();
+ let state: Vec<T> = (0..total_state).map(|i| T::from(((i % 29) as f64) * 0.03 - 0.4).unwrap()).collect();
+
+ let x_strides = [num_heads * head_dim, head_dim, 1];
+ let dt_strides = [num_heads, 1];
+ let cb_strides = [group_count * state_dim, state_dim, 1];
+ let state_strides = [head_dim * state_dim, state_dim, 1];
+
+ Input {
+ x: x.into_boxed_slice(),
+ dt: dt.into_boxed_slice(),
+ b: b.into_boxed_slice(),
+ c: c.into_boxed_slice(),
+ d: d.into_boxed_slice(),
+ z: z.into_boxed_slice(),
+ state: state.into_boxed_slice(),
+ suffix_len,
+ num_heads,
+ head_dim,
+ state_dim,
+ group_size,
+ x_strides,
+ dt_strides,
+ cb_strides,
+ state_strides,
+ }
+}
+
+fn get_output<B: Backend, T: ArrayElement + Float>(
+ input: &Input<T>,
+ kernel_type: &KernelType,
+) -> Output<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let total_x = input.suffix_len * input.num_heads * input.head_dim;
+
+ let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");
+ let dt_array = context.create_array_from(&[input.dt.len()], &input.dt, "dt");
+ let b_array = context.create_array_from(&[input.b.len()], &input.b, "b");
+ let c_array = context.create_array_from(&[input.c.len()], &input.c, "c");
+ let d_array = context.create_array_from(&[input.d.len()], &input.d, "d");
+ let z_array = context.create_array_from(&[input.z.len()], &input.z, "z");
+ let state_array = context.create_array_from(&[input.state.len()], &input.state, "state");
+ let y_array = context.create_array_uninitialized(&[total_x], T::data_type(), "y");
+
+ let x_strides: Vec<u32> = input.x_strides.iter().map(|&s| s as u32).collect();
+ let dt_strides: Vec<u32> = input.dt_strides.iter().map(|&s| s as u32).collect();
+ let cb_strides: Vec<u32> = input.cb_strides.iter().map(|&s| s as u32).collect();
+ let state_strides: Vec<u32> = input.state_strides.iter().map(|&s| s as u32).collect();
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ match kernel_type {
+ KernelType::Prefill => {
+ let kernel = <<B as Backend>::Kernels as Kernels>::SSDPrefillKernel::new(&context, T::data_type())
+ .expect("Failed to create SSDPrefillKernel");
+ kernel.encode(
+ x_array.buffer().borrow().deref(),
+ dt_array.buffer().borrow().deref(),
+ b_array.buffer().borrow().deref(),
+ c_array.buffer().borrow().deref(),
+ d_array.buffer().borrow().deref(),
+ z_array.buffer().borrow().deref(),
+ state_array.buffer().borrow_mut().deref_mut(),
+ y_array.buffer().borrow_mut().deref_mut(),
+ input.suffix_len as u32,
+ input.group_size as u32,
+ input.state_dim as u32,
+ &x_strides,
+ &dt_strides,
+ &cb_strides,
+ &state_strides,
+ input.num_heads as u32,
+ input.head_dim as u32,
+ &mut encoder,
+ );
+ },
+ KernelType::Sequential => {
+ let kernel =
+ <<B as Backend>::Kernels as Kernels>::SSDPrefillSequentialKernel::new(&context, T::data_type())
+ .expect("Failed to create SSDPrefillSequentialKernel");
+ kernel.encode(
+ x_array.buffer().borrow().deref(),
+ dt_array.buffer().borrow().deref(),
+ b_array.buffer().borrow().deref(),
+ c_array.buffer().borrow().deref(),
+ d_array.buffer().borrow().deref(),
+ z_array.buffer().borrow().deref(),
+ state_array.buffer().borrow_mut().deref_mut(),
+ y_array.buffer().borrow_mut().deref_mut(),
+ input.suffix_len as u32,
+ input.group_size as u32,
+ input.state_dim as u32,
+ &x_strides,
+ &dt_strides,
+ &cb_strides,
+ &state_strides,
+ input.num_heads as u32,
+ input.head_dim as u32,
+ &mut encoder,
+ );
+ },
+ };
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ Output {
+ y: y_array.as_slice().to_vec(),
+ state: state_array.as_slice().to_vec(),
+ }
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &Output<T>,
+ kernel_type: &KernelType,
+ label: &str,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 2e-2f32
+ } else {
+ 5e-5
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input, kernel_type);
+ let backend_name = std::any::type_name::<B>();
+ let type_name = std::any::type_name::<T>();
+
+ assert_eq_float::<T>(
+ &expected.y,
+ &output.y,
+ eps,
+ &format!("SSDPrefillSequential y {backend_name} {label} (type={type_name})"),
+ );
+ assert_eq_float::<T>(
+ &expected.state,
+ &output.state,
+ eps,
+ &format!("SSDPrefillSequential state {backend_name} {label} (type={type_name})"),
+ );
+ });
+}
+
+// --- test shapes ---
+
+fn test_shape(
+ kernel_type: &KernelType,
+ suffix_len: usize,
+ num_heads: usize,
+ head_dim: usize,
+ state_dim: usize,
+ group_size: u32,
+ label: &str,
+) {
+ fn run<T: ArrayElement + Float + Debug + Display>(
+ kernel_type: &KernelType,
+ suffix_len: usize,
+ num_heads: usize,
+ head_dim: usize,
+ state_dim: usize,
+ group_size: u32,
+ label: &str,
+ ) {
+ let input = get_input::<T>(suffix_len, num_heads, head_dim, state_dim, group_size);
+ let expected = get_output::<Cpu, T>(&input, &KernelType::Sequential);
+ test_internal(&input, &expected, kernel_type, label);
+ }
+ run::<f32>(kernel_type, suffix_len, num_heads, head_dim, state_dim, group_size, label);
+ run::<f16>(kernel_type, suffix_len, num_heads, head_dim, state_dim, group_size, label);
+ run::<bf16>(kernel_type, suffix_len, num_heads, head_dim, state_dim, group_size, label);
+}
+
+// --- Sequential ---
+#[test]
+fn test_sequential_basic() {
+ test_shape(&KernelType::Sequential, 512, 32, 64, 64, 1, "sequential_basic");
+}
+
+#[test]
+fn test_sequential_small() {
+ test_shape(&KernelType::Sequential, 4, 4, 4, 8, 1, "sequential_small");
+}
+
+#[test]
+fn test_sequential_minimal() {
+ test_shape(&KernelType::Sequential, 1, 1, 1, 1, 1, "sequential_minimal");
+}
+
+#[test]
+fn test_sequential_multi_group() {
+ test_shape(&KernelType::Sequential, 8, 8, 4, 16, 4, "sequential_multi_group");
+}
+
+#[test]
+fn test_sequential_group_per_head() {
+ test_shape(&KernelType::Sequential, 8, 4, 4, 8, 1, "sequential_group_per_head");
+}
+
+// --- Prefill ---
+#[test]
+fn test_prefill_basic() {
+ test_shape(&KernelType::Prefill, 512, 32, 64, 64, 1, "prefill_basic");
+}
+
+#[test]
+fn test_prefill_small() {
+ test_shape(&KernelType::Prefill, 4, 4, 4, 8, 1, "prefill_small");
+}
+
+#[test]
+fn test_prefill_minimal() {
+ test_shape(&KernelType::Prefill, 1, 1, 1, 1, 1, "prefill_minimal");
+}
+
+#[test]
+fn test_prefill_multi_group() {
+ test_shape(&KernelType::Prefill, 8, 8, 4, 16, 4, "prefill_multi_group");
+}
+
+#[test]
+fn test_prefill_group_per_head() {
+ test_shape(&KernelType::Prefill, 8, 4, 4, 8, 1, "prefill_group_per_head");
+}
diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs
new file mode 100644
index 000000000..e7ed1d3a1
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs
@@ -0,0 +1,240 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::SplitInProjKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ input: Box<[T]>,
+ z_bias: Box<[T]>,
+ suffix_length: u32,
+ total_dim: u32,
+ conv_dim: u32,
+ inner_dim: u32,
+ num_heads: u32,
+}
+
+struct Output<T: ArrayElement + Float> {
+ conv_out: Vec<T>,
+ z_out: Vec<T>,
+ dt_out: Vec<T>,
+}
+
+fn get_input<T: ArrayElement + Float>(
+ suffix_length: u32,
+ conv_dim: u32,
+ inner_dim: u32,
+ num_heads: u32,
+) -> Input<T> {
+ let total_dim = conv_dim + inner_dim + num_heads;
+ let input_size = suffix_length as usize * total_dim as usize;
+
+ let input: Vec<T> = (0..input_size).map(|i| T::from(0.1 * ((i as f64 % 13.0) - 6.0)).unwrap()).collect();
+ let z_bias: Vec<T> = (0..inner_dim as usize).map(|i| T::from(0.05 * ((i as f64 % 7.0) - 3.0)).unwrap()).collect();
+
+ Input {
+ input: input.into_boxed_slice(),
+ z_bias: z_bias.into_boxed_slice(),
+ suffix_length,
+ total_dim,
+ conv_dim,
+ inner_dim,
+ num_heads,
+ }
+}
+
+fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::SplitInProjKernel::new(&context, T::data_type())
+ .expect("Failed to create SplitInProjKernel");
+
+ let conv_out_size = input.suffix_length as usize * input.conv_dim as usize;
+ let z_out_size = input.suffix_length as usize * input.inner_dim as usize;
+ let dt_out_size = input.suffix_length as usize * input.num_heads as usize;
+
+ let input_array = context.create_array_from(&[input.input.len()], &input.input, "input");
+ let z_bias_array = context.create_array_from(&[input.z_bias.len()], &input.z_bias, "z_bias");
+
+ let conv_out_array = context.create_array_uninitialized(&[conv_out_size], T::data_type(), "conv_out");
+ let z_out_array = context.create_array_uninitialized(&[z_out_size], T::data_type(), "z_out");
+ let dt_out_array = context.create_array_uninitialized(&[dt_out_size], T::data_type(), "dt_out");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ input_array.buffer().borrow().deref(),
+ conv_out_array.buffer().borrow_mut().deref_mut(),
+ z_out_array.buffer().borrow_mut().deref_mut(),
+ dt_out_array.buffer().borrow_mut().deref_mut(),
+ z_bias_array.buffer().borrow().deref(),
+ input.suffix_length,
+ input.total_dim,
+ input.conv_dim,
+ input.inner_dim,
+ input.num_heads,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ Output {
+ conv_out: conv_out_array.as_slice().to_vec(),
+ z_out: z_out_array.as_slice().to_vec(),
+ dt_out: dt_out_array.as_slice().to_vec(),
+ }
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ suffix_length: u32,
+ conv_dim: u32,
+ inner_dim: u32,
+ num_heads: u32,
+) -> (Input<T>, Output<T>) {
+ let input = get_input::<T>(suffix_length, conv_dim, inner_dim, num_heads);
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &Output<T>,
+ label: &str,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-2f32
+ } else {
+ 1e-5
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input);
+ let backend_name = std::any::type_name::<B>();
+ let type_name = std::any::type_name::<T>();
+
+ assert_eq_float::<T>(
+ &expected.conv_out,
+ &output.conv_out,
+ eps,
+ &format!("SplitInProj conv_out {backend_name} {label} (type={type_name})"),
+ );
+ assert_eq_float::<T>(
+ &expected.z_out,
+ &output.z_out,
+ eps,
+ &format!("SplitInProj z_out {backend_name} {label} (type={type_name})"),
+ );
+ assert_eq_float::<T>(
+ &expected.dt_out,
+ &output.dt_out,
+ eps,
+ &format!("SplitInProj dt_out {backend_name} {label} (type={type_name})"),
+ );
+ });
+}
+
+// conv_dim=8, inner_dim=4, num_heads=2, suffix_length=2
+fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(2, 8, 4, 2);
+ test_internal(&input, &expected, "basic");
+}
+
+// Single token
+fn test_single_token<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(1, 8, 4, 2);
+ test_internal(&input, &expected, "single_token");
+}
+
+// Many tokens
+fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(16, 8, 4, 2);
+ test_internal(&input, &expected, "many_tokens");
+}
+
+// Larger dimensions closer to real usage
+fn test_large<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(8, 128, 64, 32);
+ test_internal(&input, &expected, "large");
+}
+
+// Minimal dimensions
+fn test_minimal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(1, 1, 1, 1);
+ test_internal(&input, &expected, "minimal");
+}
+
+// f32
+#[test]
+fn test_basic_f32() {
+ test_basic::<f32>();
+}
+
+#[test]
+fn test_single_token_f32() {
+ test_single_token::<f32>();
+}
+
+#[test]
+fn test_many_tokens_f32() {
+ test_many_tokens::<f32>();
+}
+
+#[test]
+fn test_large_f32() {
+ test_large::<f32>();
+}
+
+#[test]
+fn test_minimal_f32() {
+ test_minimal::<f32>();
+}
+
+// f16
+#[test]
+fn test_basic_f16() {
+ test_basic::<f16>();
+}
+
+#[test]
+fn test_single_token_f16() {
+ test_single_token::<f16>();
+}
+
+#[test]
+fn test_many_tokens_f16() {
+ test_many_tokens::<f16>();
+}
+
+#[test]
+fn test_large_f16() {
+ test_large::<f16>();
+}
+
+// bf16
+#[test]
+fn test_basic_bf16() {
+ test_basic::<bf16>();
+}
+
+#[test]
+fn test_single_token_bf16() {
+ test_single_token::<bf16>();
+}
+
+#[test]
+fn test_many_tokens_bf16() {
+ test_many_tokens::<bf16>();
+}
+
+#[test]
+fn test_large_bf16() {
+ test_large::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_update_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_update_test.rs
new file mode 100644
index 000000000..2cd1c4cd8
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_update_test.rs
@@ -0,0 +1,325 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::SSDUpdateKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ x: Box<[T]>,
+ dt: Box<[T]>,
+ b: Box<[T]>,
+ c: Box<[T]>,
+ d: Box<[T]>,
+ z: Box<[T]>,
+ state: Box<[T]>,
+ bsz: u32,
+ h: u32,
+ dh: u32,
+ g: u32,
+ n: u32,
+ state_in_place: bool,
+}
+
+struct Output<T: ArrayElement + Float> {
+ y: Vec<T>,
+ next_state: Vec<T>,
+}
+
+fn get_input<T: ArrayElement + Float>(
+ bsz: u32,
+ h: u32,
+ dh: u32,
+ g: u32,
+ n: u32,
+ state_in_place: bool,
+) -> Input<T> {
+ let bsz_u = bsz as usize;
+ let h_u = h as usize;
+ let dh_u = dh as usize;
+ let g_u = g as usize;
+ let n_u = n as usize;
+
+ let x: Vec<T> = (0..bsz_u * h_u * dh_u).map(|i| T::from(((i % 7) as f64) * 0.1 - 0.2).unwrap()).collect();
+ let z: Vec<T> = (0..bsz_u * h_u * dh_u).map(|i| T::from(((i % 5) as f64) * 0.1 - 0.1).unwrap()).collect();
+ let dt: Vec<T> = (0..bsz_u * h_u).map(|i| T::from(((i % 5) as f64) * 0.3 - 1.0).unwrap()).collect();
+ let b: Vec<T> = (0..bsz_u * g_u * n_u).map(|i| T::from(((i % 11) as f64) * 0.02 - 0.05).unwrap()).collect();
+ let c: Vec<T> = (0..bsz_u * g_u * n_u).map(|i| T::from(((i % 13) as f64) * 0.015).unwrap()).collect();
+ let d: Vec<T> = (0..h_u).map(|i| T::from(((i % 3) as f64) * 0.05).unwrap()).collect();
+ let state: Vec<T> =
+ (0..bsz_u * h_u * dh_u * n_u).map(|i| T::from(((i % 23) as f64) * 0.01 - 0.05).unwrap()).collect();
+
+ Input {
+ x: x.into_boxed_slice(),
+ dt: dt.into_boxed_slice(),
+ b: b.into_boxed_slice(),
+ c: c.into_boxed_slice(),
+ d: d.into_boxed_slice(),
+ z: z.into_boxed_slice(),
+ state: state.into_boxed_slice(),
+ bsz,
+ h,
+ dh,
+ g,
+ n,
+ state_in_place,
+ }
+}
+
+fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel =
+ <<B as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&context, T::data_type(), input.state_in_place)
+ .expect("Failed to create SSDUpdateKernel");
+
+ let bsz = input.bsz as usize;
+ let h = input.h as usize;
+ let dh = input.dh as usize;
+ let g = input.g as usize;
+ let n = input.n as usize;
+
+ let x_strides = [(h * dh) as u32, dh as u32, 1u32];
+ let dt_strides = [h as u32, 1u32];
+ let cb_strides = [(g * n) as u32, n as u32, 1u32];
+ let state_strides = [(h * dh * n) as u32, (dh * n) as u32, n as u32, 1u32];
+
+ let y_size = bsz * h * dh;
+ let ns_size = bsz * h * dh * n;
+
+ let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");
+ let dt_array = context.create_array_from(&[input.dt.len()], &input.dt, "dt");
+ let b_array = context.create_array_from(&[input.b.len()], &input.b, "b");
+ let c_array = context.create_array_from(&[input.c.len()], &input.c, "c");
+ let d_array = context.create_array_from(&[input.d.len()], &input.d, "d");
+ let z_array = context.create_array_from(&[input.z.len()], &input.z, "z");
+
+ let y_array = context.create_array_uninitialized(&[y_size], T::data_type(), "y");
+
+ if input.state_in_place {
+ let ns_array = context.create_array_from(&[ns_size], &input.state, "next_state");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ x_array.buffer().borrow().deref(),
+ dt_array.buffer().borrow().deref(),
+ b_array.buffer().borrow().deref(),
+ c_array.buffer().borrow().deref(),
+ d_array.buffer().borrow().deref(),
+ z_array.buffer().borrow().deref(),
+ None::<&<B as Backend>::Buffer>,
+ y_array.buffer().borrow_mut().deref_mut(),
+ ns_array.buffer().borrow_mut().deref_mut(),
+ (h / g) as u32,
+ input.n,
+ x_strides.as_slice(),
+ dt_strides.as_slice(),
+ cb_strides.as_slice(),
+ state_strides.as_slice(),
+ input.bsz,
+ input.h,
+ input.dh,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ Output {
+ y: y_array.as_slice().to_vec(),
+ next_state: ns_array.as_slice().to_vec(),
+ }
+ } else {
+ let state_array = context.create_array_from(&[ns_size], &input.state, "state");
+ let ns_array = context.create_array_uninitialized(&[ns_size], T::data_type(), "next_state");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ x_array.buffer().borrow().deref(),
+ dt_array.buffer().borrow().deref(),
+ b_array.buffer().borrow().deref(),
+ c_array.buffer().borrow().deref(),
+ d_array.buffer().borrow().deref(),
+ z_array.buffer().borrow().deref(),
+ Some(state_array.buffer().borrow().deref()),
+ y_array.buffer().borrow_mut().deref_mut(),
+ ns_array.buffer().borrow_mut().deref_mut(),
+ (h / g) as u32,
+ input.n,
+ x_strides.as_slice(),
+ dt_strides.as_slice(),
+ cb_strides.as_slice(),
+ state_strides.as_slice(),
+ input.bsz,
+ input.h,
+ input.dh,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ Output {
+ y: y_array.as_slice().to_vec(),
+ next_state: ns_array.as_slice().to_vec(),
+ }
+ }
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ bsz: u32,
+ h: u32,
+ dh: u32,
+ g: u32,
+ n: u32,
+ state_in_place: bool,
+) -> (Input<T>, Output<T>) {
+ let input = get_input::<T>(bsz, h, dh, g, n, state_in_place);
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &Output<T>,
+ label: &str,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 2e-2f32
+ } else {
+ 1e-5
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input);
+ let backend_name = std::any::type_name::<B>();
+ let type_name = std::any::type_name::<T>();
+
+ assert_eq_float::<T>(
+ &expected.y,
+ &output.y,
+ eps,
+ &format!("SSDUpdate y {backend_name} {label} (type={type_name})"),
+ );
+ assert_eq_float::<T>(
+ &expected.next_state,
+ &output.next_state,
+ eps,
+ &format!("SSDUpdate next_state {backend_name} {label} (type={type_name})"),
+ );
+ });
+}
+
+// bsz=1, h=4, dh=3, g=2, n=8 (matches existing Metal test)
+fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, false);
+ test_internal(&input, &expected, "basic");
+}
+
+// bsz=2 multi-batch
+fn test_multi_batch<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(2, 4, 3, 2, 8, false);
+ test_internal(&input, &expected, "multi_batch");
+}
+
+// state_in_place=true
+fn test_state_in_place<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, true);
+ test_internal(&input, &expected, "state_in_place");
+}
+
+// Larger dimensions closer to real usage
+fn test_large<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(1, 8, 4, 4, 16, false);
+ test_internal(&input, &expected, "large");
+}
+
+// Minimal: single head, single group, dh=1, n=1
+fn test_minimal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(1, 1, 1, 1, 1, false);
+ test_internal(&input, &expected, "minimal");
+}
+
+// g == h (each head is its own group)
+fn test_group_per_head<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data::<T>(1, 4, 2, 4, 8, false);
+ test_internal(&input, &expected, "group_per_head");
+}
+
+// f32
+#[test]
+fn test_basic_f32() {
+ test_basic::<f32>();
+}
+
+#[test]
+fn test_multi_batch_f32() {
+ test_multi_batch::<f32>();
+}
+
+#[test]
+fn test_state_in_place_f32() {
+ test_state_in_place::<f32>();
+}
+
+#[test]
+fn test_large_f32() {
+ test_large::<f32>();
+}
+
+#[test]
+fn test_minimal_f32() {
+ test_minimal::<f32>();
+}
+
+#[test]
+fn test_group_per_head_f32() {
+ test_group_per_head::<f32>();
+}
+
+// f16
+#[test]
+fn test_basic_f16() {
+ test_basic::<f16>();
+}
+
+#[test]
+fn test_multi_batch_f16() {
+ test_multi_batch::<f16>();
+}
+
+#[test]
+fn test_state_in_place_f16() {
+ test_state_in_place::<f16>();
+}
+
+#[test]
+fn test_large_f16() {
+ test_large::<f16>();
+}
+
+// bf16
+#[test]
+fn test_basic_bf16() {
+ test_basic::<bf16>();
+}
+
+#[test]
+fn test_multi_batch_bf16() {
+ test_multi_batch::<bf16>();
+}
+
+#[test]
+fn test_state_in_place_bf16() {
+ test_state_in_place::<bf16>();
+}
+
+#[test]
+fn test_large_bf16() {
+ test_large::<bf16>();
+}