A high-performance inference engine for AI models
Ssm cpu kernels (#308)
Aleksandr Golokoz971cd98d192e
diff --git a/crates/uzu/src/backends/common/gpu_types/activation_type.rs b/crates/uzu/src/backends/common/gpu_types/activation_type.rsindex 4a902e9d4..7524bff9b 100644--- a/crates/uzu/src/backends/common/gpu_types/activation_type.rs+++ b/crates/uzu/src/backends/common/gpu_types/activation_type.rs@@ -10,6 +10,7 @@ pub enum ActivationType {GELU,TANH,IDENTITY,+ SOFTPLUS,}impl ActivationType {@@ -24,6 +25,7 @@ impl ActivationType {ActivationType::GELU => gelu(x),ActivationType::TANH => tanh_activation(x),ActivationType::IDENTITY => x,+ ActivationType::SOFTPLUS => softplus(x),}}}@@ -52,3 +54,13 @@ fn tanh_activation<T: Float>(x: T) -> T {let x_float = x.to_f32().unwrap();T::from(x_float.tanh()).unwrap()}++fn softplus<T: Float>(x: T) -> T {+ let x_float = x.to_f32().unwrap();+ if x_float > 20f32 {+ return x;+ }++ let result = (1f32 + x_float.exp()).ln();+ T::from(result).unwrap()+}diff --git a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/uzu/src/backends/common/kernel/ssd_prefill.rsindex 7d944cf18..b14dfdb03 100644--- a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs+++ b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs@@ -22,8 +22,8 @@ pub struct SSDPrefillArguments<'a, B: Backend> {pub state: &'a mut B::Buffer,pub y: &'a mut B::Buffer,pub suffix_len: usize,- pub group_size: i32,- pub state_size: i32,+ pub group_size: u32,+ pub state_size: u32,pub x_strides: [usize; 3],pub dt_strides: [usize; 2],pub cb_strides: [usize; 3],diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rsindex d00fedb10..1fb06b66d 100644--- a/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs+++ b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs@@ -7,73 +7,194 @@ use crate::ArrayElement;#[kernel(Conv1dPack)]#[variants(T, f32, f16, bf16)]pub fn conv1d_pack<T: ArrayElement + Float>(- #[allow(unused)] state_in: *const T,- #[allow(unused)] x: *const T,- #[allow(unused)] padded: *mut T,- #[allow(unused)] state_stride: u32,- #[allow(unused)] row_stride: u32,- #[allow(unused)] suffix_len: u32,- #[allow(unused)] num_channels: u32,+ state_in: *const T,+ x: *const T,+ padded: *mut T,+ state_stride: u32,+ row_stride: u32,+ suffix_len: u32,+ num_channels: u32,) {- todo!()+ let state_stride = state_stride as usize;+ let row_stride = row_stride as usize;+ let suffix_len = suffix_len as usize;+ let num_channels = num_channels as usize;++ unsafe {+ for channel_idx in 0..num_channels {+ for row_idx in 0..state_stride + suffix_len {+ let padded_index = row_idx * row_stride + channel_idx;+ if row_idx < state_stride {+ let state_index = channel_idx * state_stride + row_idx;+ *padded.add(padded_index) = *state_in.add(state_index);+ } else {+ let token = row_idx - state_stride;+ let x_index = token * row_stride + channel_idx;+ *padded.add(padded_index) = *x.add(x_index);+ }+ }+ }+ }}#[kernel(Conv1dDecode)]#[variants(T, f32, f16, bf16)]pub fn conv1d_decode<T: ArrayElement + Float>(- #[allow(unused)] x: *const T,- #[allow(unused)] w: *const T,- #[allow(unused)]- #[optional(has_bias)]- b: Option<*const T>,- #[allow(unused)]- #[optional(!state_in_place)]- state: Option<*const T>,- #[allow(unused)] x_out: *mut T,- #[allow(unused)] b_out: *mut T,- #[allow(unused)] c_out: *mut T,- #[allow(unused)] next_state: *mut T,- #[allow(unused)] kernel_size: u32,- #[allow(unused)] row_stride: u32,- #[allow(unused)] state_stride: u32,- #[allow(unused)] num_channels: u32,- #[allow(unused)] suffix_len: u32,- #[allow(unused)] inner_dim: u32,- #[allow(unused)] proj_dim: u32,- #[allow(unused)] activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,- #[allow(unused)]- #[specialize]- has_bias: bool,- #[allow(unused)]- #[specialize]- state_in_place: bool,+ x: *const T,+ w: *const T,+ #[optional(has_bias)] b: Option<*const T>,+ #[optional(!state_in_place)] state: Option<*const T>,+ x_out: *mut T,+ b_out: *mut T,+ c_out: *mut T,+ next_state: *mut T,+ kernel_size: u32,+ row_stride: u32,+ state_stride: u32,+ num_channels: u32,+ suffix_len: u32,+ inner_dim: u32,+ proj_dim: u32,+ activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,+ #[specialize] has_bias: bool,+ #[specialize] state_in_place: bool,) {- todo!()+ let kernel_size = kernel_size as usize;+ let row_stride = row_stride as usize;+ let state_stride = state_stride as usize;+ let num_channels = num_channels as usize;+ let suffix_len = suffix_len as usize;+ let inner_dim = inner_dim as usize;+ let proj_dim = proj_dim as usize;++ let state = match state_in_place {+ true => next_state,+ false => state.unwrap(),+ };++ unsafe {+ for token_idx in 0..suffix_len {+ for channel_idx in 0..num_channels {+ let x_idx = token_idx * row_stride + channel_idx;+ let state_offset = channel_idx * state_stride;+ let w_row = w.add(channel_idx * kernel_size);++ let mut acc = 0.0f32;+ if has_bias {+ acc = (*b.unwrap().add(channel_idx)).to_f32().unwrap();+ }++ let state_taps = kernel_size.saturating_sub(1);+ for tap in 0..state_taps {+ let state_index = state_offset + tap;+ let sample = (*state.add(state_index)).to_f32().unwrap();+ acc += (*w_row.add(tap)).to_f32().unwrap() * sample;+ }++ let current = (*x.add(x_idx)).to_f32().unwrap();+ let current_tap_index = state_taps;+ if current_tap_index < kernel_size {+ acc += (*w_row.add(current_tap_index)).to_f32().unwrap() * current;+ }++ let activated = activation_type.activate(T::from(acc).unwrap());+ if channel_idx < inner_dim {+ let dst = token_idx * inner_dim + channel_idx;+ *x_out.add(dst) = activated;+ } else if channel_idx < inner_dim + proj_dim {+ let dst = token_idx * proj_dim + (channel_idx - inner_dim);+ *b_out.add(dst) = activated;+ } else if channel_idx < inner_dim + 2 * proj_dim {+ let dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);+ *c_out.add(dst) = activated;+ }+ if state_taps == 0 {+ continue;+ }++ for tap in 0..state_taps - 1 {+ let src_index = state_offset + tap + 1;+ let dst_index = state_offset + tap;+ *next_state.add(dst_index) = *state.add(src_index);+ }++ let tail_index = state_offset + state_taps - 1;+ *next_state.add(tail_index) = T::from(current).unwrap();+ }+ }+ }}#[kernel(Conv1dScan)]#[variants(T, f32, f16, bf16)]pub fn conv1d_scan<T: ArrayElement + Float>(- #[allow(unused)] padded: *const T,- #[allow(unused)] w: *const T,- #[allow(unused)]- #[optional(has_bias)]- b: Option<*const T>,- #[allow(unused)] x_out: *mut T,- #[allow(unused)] b_out: *mut T,- #[allow(unused)] c_out: *mut T,- #[allow(unused)] state_out: *mut T,- #[allow(unused)] suffix_len: u32,- #[allow(unused)] kernel_size: u32,- #[allow(unused)] row_stride: u32,- #[allow(unused)] state_stride: u32,- #[allow(unused)] num_channels: u32,- #[allow(unused)] inner_dim: u32,- #[allow(unused)] proj_dim: u32,- #[allow(unused)] activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,- #[allow(unused)]- #[specialize]- has_bias: bool,+ padded: *const T,+ w: *const T,+ #[optional(has_bias)] b: Option<*const T>,+ x_out: *mut T,+ b_out: *mut T,+ c_out: *mut T,+ state_out: *mut T,+ suffix_len: u32,+ kernel_size: u32,+ row_stride: u32,+ state_stride: u32,+ num_channels: u32,+ inner_dim: u32,+ proj_dim: u32,+ activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,+ #[specialize] has_bias: bool,) {- todo!()+ let suffix_len = suffix_len as usize;+ let kernel_size = kernel_size as usize;+ let row_stride = row_stride as usize;+ let state_stride = state_stride as usize;+ let num_channels = num_channels as usize;+ let inner_dim = inner_dim as usize;+ let proj_dim = proj_dim as usize;++ unsafe {+ for token_idx in 0..suffix_len + kernel_size.saturating_sub(1) {+ for channel_idx in 0..num_channels {+ let tap_count = kernel_size.saturating_sub(1);+ let state_offset = channel_idx * state_stride;+ let weight_offset = channel_idx * kernel_size;+ let w_row = w.add(weight_offset);++ if token_idx < suffix_len {+ let mut acc = 0.0f32;+ if has_bias {+ acc = (*b.unwrap().add(channel_idx)).to_f32().unwrap();+ }++ for tap in 0..kernel_size {+ let padded_index = (token_idx + tap) * row_stride + channel_idx;+ let sample = (*padded.add(padded_index)).to_f32().unwrap();+ acc += (*w_row.add(tap)).to_f32().unwrap() * sample;+ }++ let activated = activation_type.activate(T::from(acc).unwrap());+ if channel_idx < inner_dim {+ let dst = token_idx * inner_dim + channel_idx;+ *x_out.add(dst) = activated;+ } else if channel_idx < inner_dim + proj_dim {+ let dst = token_idx * proj_dim + (channel_idx - inner_dim);+ *b_out.add(dst) = activated;+ } else if channel_idx < inner_dim + 2 * proj_dim {+ let dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);+ *c_out.add(dst) = activated;+ }+ } else if tap_count > 0 {+ let tap = token_idx - suffix_len;+ if tap >= tap_count {+ return;+ }++ let padded_index = token_idx * row_stride + channel_idx;+ let sample = (*padded.add(padded_index)).to_f32().unwrap();+ *state_out.add(state_offset + tap) = T::from(sample).unwrap();+ }+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs b/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rsindex 873e4908f..8df79081c 100644--- a/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs+++ b/crates/uzu/src/backends/cpu/kernel/ssm/split_inproj.rs@@ -7,16 +7,39 @@ use crate::ArrayElement;#[kernel(SplitInProj)]#[variants(T, f32, f16, bf16)]pub fn split_in_proj<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] conv_out: *mut T,- #[allow(unused)] z_out: *mut T,- #[allow(unused)] dt_out: *mut T,- #[allow(unused)] z_bias: *const T,- #[allow(unused)] suffix_length: u32,- #[allow(unused)] total_dim: u32,- #[allow(unused)] conv_dim: u32,- #[allow(unused)] inner_dim: u32,- #[allow(unused)] num_heads: u32,+ input: *const T,+ conv_out: *mut T,+ z_out: *mut T,+ dt_out: *mut T,+ z_bias: *const T,+ suffix_length: u32,+ total_dim: u32,+ conv_dim: u32,+ inner_dim: u32,+ num_heads: u32,) {- todo!()+ let suffix_length = suffix_length as usize;+ let total_dim = total_dim as usize;+ let conv_dim = conv_dim as usize;+ let inner_dim = inner_dim as usize;+ let num_heads = num_heads as usize;++ unsafe {+ for row in 0..suffix_length {+ for col in 0..total_dim {+ let input_idx = row * total_dim + col;+ if col < conv_dim {+ let dst = row * conv_dim + col;+ *conv_out.add(dst) = *input.add(input_idx);+ } else if col < conv_dim + inner_dim {+ let dst = row * inner_dim + (col - conv_dim);+ let bias_idx = col - conv_dim;+ *z_out.add(dst) = *input.add(input_idx) + *z_bias.add(bias_idx);+ } else if col < conv_dim + inner_dim + num_heads {+ let dst = row * num_heads + (col - conv_dim - inner_dim);+ *dt_out.add(dst) = *input.add(input_idx);+ }+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rsindex 0ca159e13..3a673f923 100644--- a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs+++ b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_prefill.rs@@ -2,7 +2,7 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{ArrayElement, backends::common::gpu_types::ActivationType};#[kernel(SSDPrefill64)]#[variants(T, f32, f16, bf16)]@@ -16,8 +16,8 @@ pub fn ssd_prefill64<T: ArrayElement + Float>(#[allow(unused)] state: *mut T,#[allow(unused)] y: *mut T,#[allow(unused)] suffix_len: u32,- #[allow(unused)] group_size: i32,- #[allow(unused)] state_size: i32,+ #[allow(unused)] group_size: u32,+ #[allow(unused)] state_size: u32,#[allow(unused)] x_strides: &[u32],#[allow(unused)] dt_strides: &[u32],#[allow(unused)] cb_strides: &[u32],@@ -40,8 +40,8 @@ pub fn ssd_prefill<T: ArrayElement + Float>(#[allow(unused)] state: *mut T,#[allow(unused)] y: *mut T,#[allow(unused)] suffix_len: u32,- #[allow(unused)] group_size: i32,- #[allow(unused)] state_size: i32,+ #[allow(unused)] group_size: u32,+ #[allow(unused)] state_size: u32,#[allow(unused)] x_strides: &[u32],#[allow(unused)] dt_strides: &[u32],#[allow(unused)] cb_strides: &[u32],@@ -55,23 +55,64 @@ pub fn ssd_prefill<T: ArrayElement + Float>(#[kernel(SSDPrefillSequential)]#[variants(T, f32, f16, bf16)]pub fn ssd_prefill_sequential<T: ArrayElement + Float>(- #[allow(unused)] x: *const T,- #[allow(unused)] dt_raw: *const T,- #[allow(unused)] b: *const T,- #[allow(unused)] c: *const T,- #[allow(unused)] d: *const T,- #[allow(unused)] z: *const T,- #[allow(unused)] state: *mut T,- #[allow(unused)] y: *mut T,- #[allow(unused)] suffix_len: u32,- #[allow(unused)] group_size: i32,- #[allow(unused)] state_size: i32,- #[allow(unused)] x_strides: &[u32],- #[allow(unused)] dt_strides: &[u32],- #[allow(unused)] cb_strides: &[u32],- #[allow(unused)] state_strides: &[u32],- #[allow(unused)] channels: u32,- #[allow(unused)] head_dim: u32,+ x: *const T,+ dt_raw: *const T,+ b: *const T,+ c: *const T,+ d: *const T,+ z: *const T,+ state: *mut T,+ y: *mut T,+ suffix_len: u32,+ group_size: u32,+ state_size: u32,+ x_strides: &[u32],+ dt_strides: &[u32],+ cb_strides: &[u32],+ state_strides: &[u32],+ channels: u32,+ head_dim: u32,) {- todo!()+ let suffix_len = suffix_len as usize;+ let group_size = group_size as usize;+ let state_size = state_size as usize;+ let channels = channels as usize;+ let head_dim = head_dim as usize;++ let total_pairs = suffix_len * channels * head_dim;+ let safe_group = group_size.max(1) as usize;++ unsafe {+ for h in 0..channels {+ let group_idx = h / safe_group;+ for dh in 0..head_dim {+ let state_base = h * state_strides[0] as usize + dh * state_strides[1] as usize;+ for token in 0..suffix_len {+ let x_idx = token * x_strides[0] as usize + h * x_strides[1] as usize + dh * x_strides[2] as usize;+ let dt_idx = token * dt_strides[0] as usize + h * dt_strides[1] as usize;+ let cb_base = token * cb_strides[0] as usize + group_idx * cb_strides[1] as usize;++ let x_val = *x.add(x_idx);+ let dt_raw = *dt_raw.add(dt_idx);+ let dt_val = ActivationType::SOFTPLUS.activate(dt_raw);+ let decay_val = (-dt_val).exp();+ let dt_scaled_input = x_val;++ let mut acc = *d.add(h) * x_val;+ for s in 0..state_size {+ let state_idx = state_base + s * state_strides[2] as usize;+ let cb_idx = cb_base + s * cb_strides[2] as usize;+ let b_coeff = *b.add(cb_idx);+ let c_coeff = *c.add(cb_idx);+ let new_state = decay_val * *state.add(state_idx) + dt_scaled_input * b_coeff;+ *state.add(state_idx) = new_state;+ acc = acc + new_state * c_coeff;+ }++ let gate = ActivationType::SILU.activate(*z.add(x_idx));+ *y.add(x_idx) = acc * gate;+ }+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rsindex 59d069521..2de6ecea9 100644--- a/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs+++ b/crates/uzu/src/backends/cpu/kernel/ssm/ssd_update.rs@@ -2,34 +2,74 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{ArrayElement, backends::common::gpu_types::ActivationType};#[kernel(SSDUpdate)]#[variants(T, f32, f16, bf16)]pub fn ssd_update<T: ArrayElement + Float>(- #[allow(unused)] x: *const T,- #[allow(unused)] dt_raw: *const T,- #[allow(unused)] b: *const T,- #[allow(unused)] c: *const T,- #[allow(unused)] d: *const T,- #[allow(unused)] z: *const T,- #[allow(unused)]- #[optional(!state_in_place)]- state: Option<*const T>,- #[allow(unused)] y: *mut T,- #[allow(unused)] next_state: *mut T,- #[allow(unused)] group_size: u32,- #[allow(unused)] state_size: u32,- #[allow(unused)] x_strides: &[u32],- #[allow(unused)] dt_strides: &[u32],- #[allow(unused)] cb_strides: &[u32],- #[allow(unused)] state_strides: &[u32],- #[allow(unused)] b_size: u32,- #[allow(unused)] h_size: u32,- #[allow(unused)] dh_size: u32,- #[allow(unused)]- #[specialize]- state_in_place: bool,+ x: *const T,+ dt_raw: *const T,+ b: *const T,+ c: *const T,+ d: *const T,+ z: *const T,+ #[optional(!state_in_place)] state: Option<*const T>,+ y: *mut T,+ next_state: *mut T,+ group_size: u32,+ state_size: u32,+ x_strides: &[u32],+ dt_strides: &[u32],+ cb_strides: &[u32],+ state_strides: &[u32],+ b_size: u32,+ h_size: u32,+ dh_size: u32,+ #[specialize] state_in_place: bool,) {- todo!()+ let group_size = group_size as usize;+ let state_size = state_size as usize;+ let b_size = b_size as usize;+ let h_size = h_size as usize;+ let dh_size = dh_size as usize;++ let state = match state_in_place {+ true => next_state,+ false => state.unwrap(),+ };++ unsafe {+ for b_idx in 0..b_size {+ for h_idx in 0..h_size {+ for dh_idx in 0..dh_size {+ let cb_start_idx = b_idx * cb_strides[0] as usize + (h_idx / group_size) * cb_strides[1] as usize;+ let x_idx =+ b_idx * x_strides[0] as usize + h_idx * x_strides[1] as usize + dh_idx * x_strides[2] as usize;+ let dt_idx = b_idx * dt_strides[0] as usize + h_idx * dt_strides[1] as usize;+ let state_start_idx = b_idx * state_strides[0] as usize+ + h_idx * state_strides[1] as usize+ + dh_idx * state_strides[2] as usize;++ let this_x = *x.add(x_idx);+ let dt_raw_val = *dt_raw.add(dt_idx);+ let this_dt = ActivationType::SOFTPLUS.activate(dt_raw_val);+ let this_decay = T::from((-this_dt.to_f32().unwrap()).exp()).unwrap();+ let this_d = *d.add(h_idx);+ let this_z = ActivationType::SILU.activate(*z.add(x_idx));++ let mut temp = T::zero();+ for i in 0..state_size {+ let cb_idx = cb_start_idx + i;+ let state_idx = state_start_idx + i;+ let this_new_state = *state.add(state_idx) * this_decay + *b.add(cb_idx) * this_x;+ *next_state.add(state_idx) = this_new_state;+ temp = temp + this_new_state * *c.add(cb_idx);+ }+ temp = temp + this_d * this_x;+ temp = temp * this_z;+ *y.add(x_idx) = temp;+ }+ }+ }+ }}diff --git a/crates/uzu/src/backends/metal/generated/activation_type.h b/crates/uzu/src/backends/metal/generated/activation_type.hindex 290f091ea..2cf6beef1 100644--- a/crates/uzu/src/backends/metal/generated/activation_type.h+++ b/crates/uzu/src/backends/metal/generated/activation_type.h@@ -19,6 +19,7 @@ enum ActivationType : uint32_t {GELU,TANH,IDENTITY,+ SOFTPLUS,};#ifdef __METAL_VERSION__diff --git a/crates/uzu/src/backends/metal/kernel/activation/activations.h b/crates/uzu/src/backends/metal/kernel/activation/activations.hindex b09b0b45d..6df1548ae 100644--- a/crates/uzu/src/backends/metal/kernel/activation/activations.h+++ b/crates/uzu/src/backends/metal/kernel/activation/activations.h@@ -35,6 +35,17 @@ inline T activate_leaky_relu(T x, float alpha) {return static_cast<T>(yf);}+template <typename T>+inline T activate_softplus(T x) {+ float xf = static_cast<float>(x);+ if (xf > 20.0f) {+ return x;+ }++ float result = fast::log(1.0f + fast::exp(xf));+ return static_cast<T>(result);+}+template <typename T>inline T activate_tanh(T x) {return static_cast<T>(metal::tanh(float(x)));@@ -51,5 +62,7 @@ inline T activate(T x, ActivationType type) {return activate_tanh(x);case IDENTITY:return x;+ case SOFTPLUS:+ return activate_softplus(x);}}diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal b/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metalindex e0474b957..685fce193 100644--- a/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal+++ b/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal@@ -1,7 +1,6 @@#include <metal_stdlib>#include "../activation/activations.h"#include "../common/dsl.h"-#include "../ssm/ssm_common.h"using namespace metal;diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal b/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metalindex 7132561aa..8b9119f41 100644--- a/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal+++ b/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal@@ -1,7 +1,6 @@#include <metal_stdlib>#include "../activation/activations.h"#include "../common/dsl.h"-#include "../ssm/ssm_common.h"using namespace metal;diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metalindex 2a7020b36..75e19ddf6 100644--- a/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal+++ b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal@@ -3,7 +3,6 @@#include "../common/dsl.h"#include "../common/thread_context.h"#include "../common/threadgroup_reduce.h"-#include "../ssm/ssm_common.h"using namespace metal;diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metalindex 3e5f78703..b26376bb2 100644--- a/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal@@ -1,7 +1,6 @@#include <metal_stdlib>#include "../common/defines.h"#include "../common/dsl.h"-#include "../ssm/ssm_common.h"using namespace metal;diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal b/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metalindex 65256e6b1..6365c2092 100644--- a/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal+++ b/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal@@ -1,7 +1,7 @@#include <metal_stdlib>+#include "../activation/activations.h"#include "../common/defines.h"#include "../common/dsl.h"-#include "../ssm/ssm_common.h"using namespace metal;@@ -88,7 +88,7 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(float a_raw =float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);- float sp = softplus(a_raw + float(dt_bias[hv]));+ float sp = activate_softplus(a_raw + float(dt_bias[hv]));float decay = fast::exp(-fast::exp(float(a_log[hv])) * sp);beta_out[token_idx * num_v_heads + hv] = beta;diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/uzu/src/backends/metal/kernel/delta_net/update.metalindex a12e283a4..094cfe0c3 100644--- a/crates/uzu/src/backends/metal/kernel/delta_net/update.metal+++ b/crates/uzu/src/backends/metal/kernel/delta_net/update.metal@@ -4,7 +4,6 @@#include "../common/dsl.h"#include "../common/thread_context.h"#include "../common/threadgroup_reduce.h"-#include "../ssm/ssm_common.h"using namespace metal;@@ -97,7 +96,7 @@ PUBLIC KERNEL(DeltaNetUpdate)(float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);float beta = 1.0f / (1.0f + fast::exp(-beta_raw));float a_raw = float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);- float sp = softplus(a_raw + float(dt_bias[hv_idx]));+ float sp = activate_softplus(a_raw + float(dt_bias[hv_idx]));float decay = fast::exp(-fast::exp(float(a_log[hv_idx])) * sp);float v_i =diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metalindex 2fc6e8114..e6100b52e 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal@@ -137,33 +137,31 @@ PUBLIC KERNEL(Conv1dScan)(if (has_bias) {acc = float(b[channel_idx]);}+for (uint tap = 0; tap < kernel_size; ++tap) {- const uint padded_idx = size_t(token_idx) + size_t(tap);- float sample = 0.0f;- const uint padded_index = size_t(padded_idx) * row_stride + channel_idx;- sample = float(padded[padded_index]);+ const uint padded_index = (token_idx + tap) * row_stride + channel_idx;+ float sample = float(padded[padded_index]);acc += float(w_row[tap]) * sample;}const T activated = activate(static_cast<T>(acc), activation_type);if (channel_idx < inner_dim) {- const size_t dst = size_t(token_idx) * inner_dim + channel_idx;+ const uint dst = token_idx * inner_dim + channel_idx;x_out[dst] = activated;} else if (channel_idx < inner_dim + proj_dim) {- const size_t dst =- size_t(token_idx) * proj_dim + (channel_idx - inner_dim);+ const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);b_out[dst] = activated;} else if (channel_idx < inner_dim + 2 * proj_dim) {- const size_t dst =- size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);+ const uint dst =+ token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);c_out[dst] = activated;}} else if (tap_count > 0) {- const size_t tap = size_t(token_idx - suffix_len);- if (tap >= size_t(tap_count)) {+ const uint tap = token_idx - suffix_len;+ if (tap >= tap_count) {return;}- const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;+ const uint padded_index = token_idx * row_stride + channel_idx;const float sample = float(padded[padded_index]);state_out[state_offset + tap] = static_cast<T>(sample);}diff --git a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metalindex fb1b85ebd..18bd8805b 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal@@ -17,16 +17,16 @@ PUBLIC KERNEL(SplitInProj)(const uint row AXIS(suffix_length, 16),const uint col AXIS(total_dim, 16)) {- const int input_idx = row * total_dim + col;+ const uint input_idx = row * total_dim + col;if (col < conv_dim) {- const int dst = row * conv_dim + col;+ const uint dst = row * conv_dim + col;conv_out[dst] = input[input_idx];} else if (col < conv_dim + inner_dim) {- const int dst = row * inner_dim + (col - conv_dim);- const int bias_idx = col - conv_dim;+ const uint dst = row * inner_dim + (col - conv_dim);+ const uint bias_idx = col - conv_dim;z_out[dst] = input[input_idx] + z_bias[bias_idx];} else if (col < conv_dim + inner_dim + num_heads) {- const int dst = row * num_heads + (col - conv_dim - inner_dim);+ const uint dst = row * num_heads + (col - conv_dim - inner_dim);dt_out[dst] = input[input_idx];}}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex 6330408ee..d795c70a4 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -3,7 +3,6 @@#include "../activation/activations.h"#include "../common/dsl.h"#include "../common/thread_context.h"-#include "ssm_common.h"using namespace metal;@@ -21,8 +20,8 @@ PUBLIC KERNEL(SSDPrefill64)(device T* state, // (h, dh, n)device T* y, // (suffix, h, dh)constant const uint& suffix_len,- constant const int& group_size,- constant const int& state_size,+ constant const uint& group_size,+ constant const uint& state_size,constant const uint* x_strides,constant const uint* dt_strides,constant const uint* cb_strides,@@ -35,7 +34,7 @@ PUBLIC KERNEL(SSDPrefill64)() {const uint h_idx = pair_idx / head_dim;const uint dh_idx = pair_idx % head_dim;- const uint safe_group = uint(max(group_size, 1));+ const uint safe_group = max(group_size, 1u);const uint group_idx = h_idx / safe_group;const uint x_token_stride = x_strides[0];@@ -59,27 +58,21 @@ PUBLIC KERNEL(SSDPrefill64)(const uint idx0 = lane_idx;const uint idx1 = lane_idx + thread_context.simdgroup_size;- float state0 = 0.0f;- float state1 = 0.0f;- uint state_idx0 = 0;- uint state_idx1 = 0;- state_idx0 = state_base + idx0 * state_inner_stride;- state0 = float(state[state_idx0]);+ const uint state_idx0 = state_base + idx0 * state_inner_stride;+ const uint state_idx1 = state_base + idx1 * state_inner_stride;+ float state0 = float(state[state_idx0]);+ float state1 = float(state[state_idx1]);- state_idx1 = state_base + idx1 * state_inner_stride;- state1 = float(state[state_idx1]);-- uint cb_idx0 = 0;- uint cb_idx1 = 0;- cb_idx0 = cb_group_base + idx0 * cb_state_stride;- cb_idx1 = cb_group_base + idx1 * cb_state_stride;+ uint cb_idx0 = cb_group_base + idx0 * cb_state_stride;+ uint cb_idx1 = cb_group_base + idx1 * cb_state_stride;for (uint token = 0; token < suffix_len; ++token) {const uint x_idx = token * x_token_stride + x_base;const uint dt_idx = token * dt_token_stride + dt_base;const float x_val = float(x[x_idx]);- const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));+ const float decay_val =+ fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));const float gate = float(activate_silu(z[x_idx]));const float skip = d_scalar * x_val;const float dt_scaled_input = x_val;@@ -119,8 +112,8 @@ PUBLIC KERNEL(SSDPrefill)(device T* state, // (h, dh, n)device T* y, // (suffix, h, dh)constant const uint& suffix_len,- constant const int& group_size,- constant const int& state_size,+ constant const uint& group_size,+ constant const uint& state_size,constant const uint* x_strides,constant const uint* dt_strides,constant const uint* cb_strides,@@ -134,7 +127,7 @@ PUBLIC KERNEL(SSDPrefill)(const int state_dim = state_size;const uint h_idx = pair_idx / head_dim;const uint dh_idx = pair_idx % head_dim;- const uint safe_group = uint(max(group_size, 1));+ const uint safe_group = max(group_size, 1u);const uint group_idx = h_idx / safe_group;const uint x_token_stride = x_strides[0];@@ -167,8 +160,12 @@ PUBLIC KERNEL(SSDPrefill)(#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);- const uint state_idx = state_base + idx * state_inner_stride;- lane_states[chunk] = float(state[state_idx]);+ if (idx < state_dim) {+ const uint state_idx = state_base + idx * state_inner_stride;+ lane_states[chunk] = float(state[state_idx]);+ } else {+ lane_states[chunk] = 0.0f;+ }}for (uint token = 0; token < suffix_len; ++token) {@@ -176,7 +173,8 @@ PUBLIC KERNEL(SSDPrefill)(const uint dt_idx = token * dt_token_stride + dt_base;const float x_val = float(x[x_idx]);- const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));+ const float decay_val =+ fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));const float gate = float(activate_silu(z[x_idx]));const float skip = d_scalar * x_val;const float dt_scaled_input = x_val;@@ -186,12 +184,14 @@ PUBLIC KERNEL(SSDPrefill)(for (int chunk = 0; chunk < chunk_count; ++chunk) {const int idx =chunk * int(thread_context.simdgroup_size) + int(lane_idx);- const uint cb_idx =- cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;- const float new_state =- decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);- lane_states[chunk] = new_state;- contrib_sum += new_state * float(c[cb_idx]);+ if (idx < state_dim) {+ const uint cb_idx = cb_group_base + uint(idx) * cb_state_stride ++ token * cb_token_stride;+ const float new_state =+ decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);+ lane_states[chunk] = new_state;+ contrib_sum += new_state * float(c[cb_idx]);+ }}float dot = simd_sum(contrib_sum);@@ -203,8 +203,10 @@ PUBLIC KERNEL(SSDPrefill)(#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);- const uint state_idx = state_base + uint(idx) * state_inner_stride;- state[state_idx] = static_cast<T>(lane_states[chunk]);+ if (idx < state_dim) {+ const uint state_idx = state_base + uint(idx) * state_inner_stride;+ state[state_idx] = static_cast<T>(lane_states[chunk]);+ }}}@@ -220,8 +222,8 @@ PUBLIC KERNEL(SSDPrefillSequential)(device T* state, // (h, dh, n)device T* y, // (suffix, h, dh)constant const uint& suffix_len,- constant const int& group_size,- constant const int& state_size,+ constant const uint& group_size,+ constant const uint& state_size,constant const uint* x_strides,constant const uint* dt_strides,constant const uint* cb_strides,@@ -231,7 +233,7 @@ PUBLIC KERNEL(SSDPrefillSequential)(const uint h_idx AXIS(channels, 32),const uint dh_idx AXIS(head_dim, 32)) {- const uint safe_group = uint(max(group_size, 1));+ const uint safe_group = max(group_size, 1u);const uint group_idx = h_idx / safe_group;device T* state_row = state + size_t(h_idx) * state_strides[0] +size_t(dh_idx) * state_strides[1];@@ -244,7 +246,7 @@ PUBLIC KERNEL(SSDPrefillSequential)(const T this_x = x[x_idx];const T dt_raw_val = dt_raw[dt_idx];- const T this_dt = softplus(dt_raw_val);+ const T this_dt = activate_softplus(dt_raw_val);const T this_decay = static_cast<T>(fast::exp(-float(this_dt)));const T this_D = d[h_idx];const T this_z = activate_silu(z[x_idx]);diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metalindex 4afcc0302..c4564d601 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal@@ -1,7 +1,6 @@#include <metal_stdlib>#include "../activation/activations.h"#include "../common/dsl.h"-#include "ssm_common.h"using namespace metal;@@ -52,7 +51,7 @@ PUBLIC KERNEL(SSDUpdate)(// load dataT this_x = x[x_idx];T dt_raw_val = dt_raw[dt_idx];- T this_dt = softplus(dt_raw_val);+ T this_dt = activate_softplus(dt_raw_val);T this_decay = static_cast<T>(fast::exp(-float(this_dt)));T this_d = d[h_idx];T this_z = z[x_idx];diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.hdeleted file mode 100644index 1884fc210..000000000--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h+++ /dev/null@@ -1,12 +0,0 @@-#pragma once--#include <metal_stdlib>--template <typename T>-inline T softplus(T x) {- float xf = float(x);- if (xf > 20.0f) {- return x;- }- return static_cast<T>(log(1.0f + fast::exp(xf)));-}diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex 183ba1f7a..0863028d5 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -292,8 +292,8 @@ impl<B: Backend> MambaMixer<B> {state: state_arr.buffer().borrow_mut().deref_mut(),y: out.buffer().borrow_mut().deref_mut(),suffix_len: suffix_length,- group_size: (self.config.num_heads / self.config.num_groups) as i32,- state_size: self.config.state_dim as i32,+ group_size: (self.config.num_heads / self.config.num_groups) as u32,+ state_size: self.config.state_dim as u32,x_strides: [self.config.num_heads * self.config.head_dim, self.config.head_dim, 1],dt_strides: [self.config.num_heads, 1],cb_strides: [self.config.num_groups * self.config.state_dim, self.config.state_dim, 1],diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rsindex 1dbd6a2ae..a14a00f7b 100644--- a/crates/uzu/src/encodable_block/mlp/moe.rs+++ b/crates/uzu/src/encodable_block/mlp/moe.rs@@ -182,11 +182,8 @@ impl<B: Backend> MoeBlock<B> {ActivationType::SILU {..} => 2,- ActivationType::TANH => {- panic!("Tanh activation is not supported for MoE kernels")- },- ActivationType::IDENTITY => {- panic!("Identity activation is not supported for MoE kernels")+ _ => {+ panic!("{:?} is not supported for MoE kernels", activation)},}}diff --git a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rsindex 34f9dcc9e..2904de720 100644--- a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs+++ b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs@@ -1,64 +1,46 @@-#![cfg(metal_backend)]--use std::mem::size_of;+use std::{+ mem::size_of,+ ops::{Deref, DerefMut},+};-use bytemuck;-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};-use objc2::runtime::ProtocolObject;+use uzu::backends::common::Buffer;use crate::{- DataType,- backends::{- common::{- Backend, Context, Encoder, Kernels,- gpu_types::ActivationType,- kernel::{- Conv1dScanKernel,- ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},- },+ ArrayContextExt, DataType,+ backends::common::{+ Backend, Context, Encoder, Kernels,+ gpu_types::ActivationType,+ kernel::{+ Conv1dScanKernel,+ ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},- metal::Metal,},+ for_each_non_cpu_backend,};-const STORAGE_MODE: MTLResourceOptions = MTLResourceOptions::STORAGE_MODE_SHARED;--fn silu_scalar(x: f32) -> f32 {- let y = 1.0 / (1.0 + (-x).exp());- x * y-}--fn softplus_f32(x: f32) -> f32 {- if x > 20.0 {- x- } else {- (1.0 + x.exp()).ln()- }-}--fn write_buffer(- buf: &ProtocolObject<dyn MTLBuffer>,+fn write_buffer<B: Backend>(+ buf: &B::Buffer,data: &[f32],) {unsafe {- std::ptr::copy_nonoverlapping(data.as_ptr(), buf.contents().as_ptr() as *mut f32, data.len());+ std::ptr::copy_nonoverlapping(data.as_ptr(), buf.cpu_ptr().as_ptr() as *mut f32, data.len());}}-fn read_buffer(- buf: &ProtocolObject<dyn MTLBuffer>,+fn read_buffer<B: Backend>(+ buf: &B::Buffer,len: usize,) -> Vec<f32> {let mut out = vec![0.0f32; len];unsafe {- std::ptr::copy_nonoverlapping(buf.contents().as_ptr() as *const f32, out.as_mut_ptr(), len);+ std::ptr::copy_nonoverlapping(buf.cpu_ptr().as_ptr() as *const f32, out.as_mut_ptr(), len);}out}-fn zero_buffer(buf: &ProtocolObject<dyn MTLBuffer>) {+fn zero_buffer<B: Backend>(buf: &B::Buffer) {unsafe {- std::ptr::write_bytes(buf.contents().as_ptr(), 0, buf.length() as usize);+ std::ptr::write_bytes(buf.cpu_ptr().as_ptr(), 0, buf.length());}}@@ -96,10 +78,10 @@ fn ssd_prefill_cpu_reference(let x_val = x_data[x_idx];let dt_raw = dt_raw_data[dt_idx];- let dt_val = softplus_f32(dt_raw);+ let dt_val = ActivationType::SOFTPLUS.activate(dt_raw);let decay_val = (-dt_val).exp();let dt_scaled_input = x_val;- let gate = silu_scalar(z_data[x_idx]);+ let gate = ActivationType::SILU.activate(z_data[x_idx]);let mut acc = d_data[h] * x_val;for s in 0..state_dim {@@ -196,49 +178,65 @@ impl SSDPrefillFixture {}}-fn run_prefill_kernel_mode(- ctx: &<Metal as Backend>::Context,- kernel: &SSDPrefillKernels<Metal>,+fn run_prefill_kernel_mode<B: Backend>(+ ctx: &B::Context,+ kernel: &SSDPrefillKernels<B>,fixture: &SSDPrefillFixture,mode: SSDPrefillMode,) -> (Vec<f32>, Vec<f32>, Option<(Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>)>) {- let device = &ctx.device;- let x_buf = device- .new_buffer_with_data(bytemuck::cast_slice(&fixture.x_data), STORAGE_MODE)- .expect("Failed to create buffer");- let dt_buf = device- .new_buffer_with_data(bytemuck::cast_slice(&fixture.dt_data), STORAGE_MODE)- .expect("Failed to create buffer");- let b_buf = device- .new_buffer_with_data(bytemuck::cast_slice(&fixture.b_data), STORAGE_MODE)- .expect("Failed to create buffer");- let c_buf = device- .new_buffer_with_data(bytemuck::cast_slice(&fixture.c_data), STORAGE_MODE)- .expect("Failed to create buffer");- let d_buf = device- .new_buffer_with_data(bytemuck::cast_slice(&fixture.d_data), STORAGE_MODE)- .expect("Failed to create buffer");- let z_buf = device- .new_buffer_with_data(bytemuck::cast_slice(&fixture.z_data), STORAGE_MODE)- .expect("Failed to create buffer");- let mut state_buf = device.new_buffer(fixture.total_state * 4, STORAGE_MODE).expect("Failed to create buffer");- let mut y_buf = device.new_buffer(fixture.total_x * 4, STORAGE_MODE).expect("Failed to create buffer");-- write_buffer(&state_buf, &fixture.state_init);- zero_buffer(&y_buf);-- let args = SSDPrefillArguments::<Metal> {- x: &x_buf,- dt: &dt_buf,- b: &b_buf,- c: &c_buf,- d: &d_buf,- z: &z_buf,- state: &mut state_buf,- y: &mut y_buf,+ let x_array = ctx.create_array_from(&[fixture.x_data.len()], &fixture.x_data, "");+ let x_array_buf = x_array.buffer();+ let x_array_buf_borrow = x_array_buf.borrow();+ let x_buf = x_array_buf_borrow.deref();++ let dt_array = ctx.create_array_from(&[fixture.dt_data.len()], &fixture.dt_data, "");+ let dt_array_buf = dt_array.buffer();+ let dt_array_buf_borrow = dt_array_buf.borrow();+ let dt_buf = dt_array_buf_borrow.deref();++ let b_array = ctx.create_array_from(&[fixture.b_data.len()], &fixture.b_data, "");+ let b_array_buf = b_array.buffer();+ let b_array_buf_borrow = b_array_buf.borrow();+ let b_buf = b_array_buf_borrow.deref();++ let c_array = ctx.create_array_from(&[fixture.c_data.len()], &fixture.c_data, "");+ let c_array_buf = c_array.buffer();+ let c_array_buf_borrow = c_array_buf.borrow();+ let c_buf = c_array_buf_borrow.deref();++ let d_array = ctx.create_array_from(&[fixture.d_data.len()], &fixture.d_data, "");+ let d_array_buf = d_array.buffer();+ let d_array_buf_borrow = d_array_buf.borrow();+ let d_buf = d_array_buf_borrow.deref();++ let z_array = ctx.create_array_from(&[fixture.z_data.len()], &fixture.z_data, "");+ let z_array_buf = z_array.buffer();+ let z_array_buf_borrow = z_array_buf.borrow();+ let z_buf = z_array_buf_borrow.deref();++ let state_array = ctx.create_array_from(&[fixture.state_init.len()], &fixture.state_init, "");+ let state_array_buf = state_array.buffer();+ let mut state_array_buf_borrow = state_array_buf.borrow_mut();+ let state_buf = state_array_buf_borrow.deref_mut();++ let y = vec![0f32; fixture.total_x];+ let y_array = ctx.create_array_from(&[y.len()], &y, "");+ let y_array_buf = y_array.buffer();+ let mut y_array_buf_borrow = y_array_buf.borrow_mut();+ let y_buf = y_array_buf_borrow.deref_mut();++ let args = SSDPrefillArguments::<B> {+ x: x_buf,+ dt: dt_buf,+ b: b_buf,+ c: c_buf,+ d: d_buf,+ z: z_buf,+ state: state_buf,+ y: y_buf,suffix_len: fixture.suffix_len,- group_size: fixture.group_size,- state_size: fixture.state_dim as i32,+ group_size: fixture.group_size as u32,+ state_size: fixture.state_dim as u32,x_strides: fixture.x_strides,dt_strides: fixture.dt_strides,cb_strides: fixture.cb_strides,@@ -251,14 +249,14 @@ fn run_prefill_kernel_mode(kernel.encode(&mut encoder, args, mode);encoder.end_encoding().submit().wait_until_completed().unwrap();- let y_vec = read_buffer(&y_buf, fixture.total_x);- let state_vec = read_buffer(&state_buf, fixture.total_state);+ let y_vec = read_buffer::<B>(y_buf, fixture.total_x);+ let state_vec = read_buffer::<B>(state_buf, fixture.total_state);(y_vec, state_vec, None)}-fn run_conv_scan_once(- ctx: &<Metal as Backend>::Context,- kernel: &<<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel,+fn run_conv_scan_once<B: Backend>(+ ctx: &<B as Backend>::Context,+ kernel: &<<B as Backend>::Kernels as Kernels>::Conv1dScanKernel,suffix_len: usize,channels: usize,kernel_size: i32,@@ -270,39 +268,40 @@ fn run_conv_scan_once(use_scratch: bool,alias_io: bool,) -> (Vec<f32>, Vec<f32>) {- let device = &ctx.device;let total_x = suffix_len * channels;let _total_w = channels * kernel_size as usize;let total_state = channels * tap_count;- let mut y_buf = if alias_io {- device.new_buffer_with_data(bytemuck::cast_slice(x_data), STORAGE_MODE).expect("Failed to create buffer")- } else {- let buf = device.new_buffer(total_x * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");- zero_buffer(&buf);- buf- };- let w_buf =- device.new_buffer_with_data(bytemuck::cast_slice(w_data), STORAGE_MODE).expect("Failed to create buffer");- let b_buf =- device.new_buffer_with_data(bytemuck::cast_slice(b_data), STORAGE_MODE).expect("Failed to create buffer");- let mut state_buf =- device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");- let scratch_buf = if use_scratch && tap_count > 0 {- Some(device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer"))+ let y_array = if alias_io {+ ctx.create_array_from(&[x_data.len()], x_data, "")} else {- None+ ctx.create_array_zeros(&[total_x], DataType::F32, "")};+ let b_out_array = ctx.create_array_zeros(&[total_x], DataType::F32, "");+ let c_out_array = ctx.create_array_zeros(&[total_x], DataType::F32, "");++ let w_array = ctx.create_array_from(&[w_data.len()], w_data, "");+ let b_array = ctx.create_array_from(&[b_data.len()], b_data, "");- write_buffer(&state_buf, state_init);- if let Some(ref scratch) = scratch_buf {- zero_buffer(scratch);+ let state_array = ctx.create_array_uninitialized(&[total_state], DataType::F32, "");+ let scratch_array = ctx.create_array_uninitialized(&[total_state], DataType::F32, "");++ {+ let state_buf = state_array.buffer();+ let state_borrow = state_buf.borrow();+ write_buffer::<B>(state_borrow.deref(), state_init);+ if use_scratch && tap_count > 0 {+ let scratch_buf = scratch_array.buffer();+ let scratch_borrow = scratch_buf.borrow();+ zero_buffer::<B>(scratch_borrow.deref());+ }}let padded_len = tap_count + suffix_len;- let padded_buf =- device.new_buffer(padded_len * channels * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");+ let padded_array = ctx.create_array_uninitialized(&[padded_len * channels], DataType::F32, "");{+ let padded_buf = padded_array.buffer();+ let padded_borrow = padded_buf.borrow();let mut host = vec![0.0f32; padded_len * channels];for tap in 0..tap_count {for ch in 0..channels {@@ -314,67 +313,89 @@ fn run_conv_scan_once(host[(tap_count + token) * channels + ch] = x_data[token * channels + ch];}}- write_buffer(&padded_buf, &host);+ write_buffer::<B>(padded_borrow.deref(), &host);}let mut encoder = Encoder::new(ctx).unwrap();- let mut b_out = y_buf.clone();- let mut c_out = y_buf.clone();- let mut state_out = scratch_buf.as_ref().unwrap_or(&state_buf).clone();- kernel.encode(- &padded_buf,- &w_buf,- Some(&b_buf),- &mut y_buf,- &mut b_out,- &mut c_out,- &mut state_out,- suffix_len as u32,- kernel_size as u32,- channels as u32,- tap_count as u32,- channels as u32,- channels as u32,- 0u32,- ActivationType::SILU,- &mut encoder,- );+ {+ let padded_buf = padded_array.buffer();+ let padded_borrow = padded_buf.borrow();+ let w_buf = w_array.buffer();+ let w_borrow = w_buf.borrow();+ let b_buf = b_array.buffer();+ let b_borrow = b_buf.borrow();+ let y_buf = y_array.buffer();+ let mut y_borrow = y_buf.borrow_mut();+ let b_out_buf = b_out_array.buffer();+ let mut b_out_borrow = b_out_buf.borrow_mut();+ let c_out_buf = c_out_array.buffer();+ let mut c_out_borrow = c_out_buf.borrow_mut();+ let state_buf = state_array.buffer();+ let mut state_borrow = state_buf.borrow_mut();++ kernel.encode(+ padded_borrow.deref(),+ w_borrow.deref(),+ Some(b_borrow.deref()),+ y_borrow.deref_mut(),+ b_out_borrow.deref_mut(),+ c_out_borrow.deref_mut(),+ state_borrow.deref_mut(),+ suffix_len as u32,+ kernel_size as u32,+ channels as u32,+ tap_count as u32,+ channels as u32,+ channels as u32,+ 0u32,+ ActivationType::SILU,+ &mut encoder,+ );+ }- if let Some(ref scratch) = scratch_buf {+ if use_scratch && tap_count > 0 {let bytes = channels * tap_count * size_of::<f32>();if bytes > 0 {- encoder.encode_copy(scratch, 0..bytes, &mut state_buf, 0..bytes);+ let scratch_buf = scratch_array.buffer();+ let scratch_borrow = scratch_buf.borrow();+ let state_buf = state_array.buffer();+ let mut state_borrow = state_buf.borrow_mut();+ encoder.encode_copy(scratch_borrow.deref(), 0..bytes, state_borrow.deref_mut(), 0..bytes);}}encoder.end_encoding().submit().wait_until_completed().unwrap();- let y_vec = read_buffer(&y_buf, total_x);- let state_vec = read_buffer(&state_buf, total_state);+ let y_buf = y_array.buffer();+ let y_borrow = y_buf.borrow();+ let state_buf = state_array.buffer();+ let state_borrow = state_buf.borrow();+ let y_vec = read_buffer::<B>(y_borrow.deref(), total_x);+ let state_vec = read_buffer::<B>(state_borrow.deref(), total_state);(y_vec, state_vec)}-fn assert_deterministic_for_mode(mode: SSDPrefillMode) {- let Some(ctx) = <Metal as Backend>::Context::new().ok() else {+fn assert_deterministic_for_mode<B: Backend>(mode: SSDPrefillMode) {+ let Some(ctx) = <B as Backend>::Context::new().ok() else {eprintln!("Skipping SSD prefill determinism test: no Metal device");return;};- let kernel = SSDPrefillKernels::<Metal>::new(&ctx, DataType::F32).unwrap();+ let kernel = SSDPrefillKernels::<B>::new(&ctx, DataType::F32).unwrap();let fixture = SSDPrefillFixture::new();- let (y_a, state_a, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);- let (y_b, state_b, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);+ let (y_a, state_a, _) = run_prefill_kernel_mode::<B>(&ctx, &kernel, &fixture, mode);+ let (y_b, state_b, _) = run_prefill_kernel_mode::<B>(&ctx, &kernel, &fixture, mode);assert_eq!(y_a, y_b, "Prefill outputs differ in {:?} mode", mode);assert_eq!(state_a, state_b, "Prefill states differ in {:?} mode", mode);}-fn assert_matches_cpu_reference(mode: SSDPrefillMode) {- let Some(ctx) = <Metal as Backend>::Context::new().ok() else {+fn assert_matches_cpu_reference<B: Backend>(mode: SSDPrefillMode) {+ let Some(ctx) = <B as Backend>::Context::new().ok() else {eprintln!("Skipping SSD prefill reference test: no Metal device");return;};- let kernel = SSDPrefillKernels::<Metal>::new(&ctx, DataType::F32).unwrap();+ let kernel = SSDPrefillKernels::<B>::new(&ctx, DataType::F32).unwrap();let fixture = SSDPrefillFixture::new();let (y_ref, state_ref) = ssd_prefill_cpu_reference(@@ -396,7 +417,7 @@ fn assert_matches_cpu_reference(mode: SSDPrefillMode) {fixture.state_strides,);- let (y_gpu, state_gpu, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);+ let (y_gpu, state_gpu, _) = run_prefill_kernel_mode::<B>(&ctx, &kernel, &fixture, mode);let tolerance = 5e-5f32;let mut max_y_diff = 0.0f32;@@ -434,33 +455,12 @@ fn assert_matches_cpu_reference(mode: SSDPrefillMode) {);}-#[test]-fn ssd_prefill_sequential_is_deterministic() {- assert_deterministic_for_mode(SSDPrefillMode::Sequential);-}--#[test]-fn ssd_prefill_single_pass_is_deterministic() {- assert_deterministic_for_mode(SSDPrefillMode::SinglePass);-}--#[test]-fn ssd_prefill_sequential_matches_cpu_reference() {- assert_matches_cpu_reference(SSDPrefillMode::Sequential);-}--#[test]-fn ssd_prefill_single_pass_matches_cpu_reference() {- assert_matches_cpu_reference(SSDPrefillMode::SinglePass);-}--#[test]-fn conv1d_scan_is_deterministic() {- let Some(ctx) = <Metal as Backend>::Context::new().ok() else {+fn conv1d_scan_deterministic_internal<B: Backend>() {+ let Some(ctx) = <B as Backend>::Context::new().ok() else {eprintln!("Skipping conv1d scan determinism test: no Metal device");return;};- let kernel = <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, true).unwrap();+ let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, true).unwrap();let suffix_len = 192usize;let channels = 8usize;@@ -478,7 +478,7 @@ fn conv1d_scan_is_deterministic() {let use_scratch = tap_count > 0 && suffix_len > 1;for &alias_io in &[false, true] {- let first = run_conv_scan_once(+ let first = run_conv_scan_once::<B>(&ctx,&kernel,suffix_len,@@ -492,7 +492,7 @@ fn conv1d_scan_is_deterministic() {use_scratch,alias_io,);- let second = run_conv_scan_once(+ let second = run_conv_scan_once::<B>(&ctx,&kernel,suffix_len,@@ -511,3 +511,38 @@ fn conv1d_scan_is_deterministic() {assert_eq!(first.1, second.1, "Conv states differ (alias_io={alias_io})");}}++#[test]+fn ssd_prefill_sequential_is_deterministic() {+ for_each_non_cpu_backend!(|B| {+ assert_deterministic_for_mode::<B>(SSDPrefillMode::Sequential);+ });+}++#[test]+fn ssd_prefill_single_pass_is_deterministic() {+ for_each_non_cpu_backend!(|B| {+ assert_deterministic_for_mode::<B>(SSDPrefillMode::SinglePass);+ });+}++#[test]+fn ssd_prefill_sequential_matches_cpu_reference() {+ for_each_non_cpu_backend!(|B| {+ assert_matches_cpu_reference::<B>(SSDPrefillMode::Sequential);+ });+}++#[test]+fn ssd_prefill_single_pass_matches_cpu_reference() {+ for_each_non_cpu_backend!(|B| {+ assert_matches_cpu_reference::<B>(SSDPrefillMode::SinglePass);+ });+}++#[test]+fn conv1d_scan_is_deterministic() {+ for_each_non_cpu_backend!(|B| {+ conv1d_scan_deterministic_internal::<B>();+ });+}diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rsindex 7e47c316a..e9290f87c 100644--- a/crates/uzu/tests/unit/kernel/mod.rs+++ b/crates/uzu/tests/unit/kernel/mod.rs@@ -18,6 +18,7 @@ mod rms_norm_test;mod rope_test;mod sampling;mod short_conv;+mod ssm;mod tensor_add_bias_test;mod tensor_add_scale_test;mod tensor_add_swap_test;diff --git a/crates/uzu/tests/unit/kernel/ssd_update_test.rs b/crates/uzu/tests/unit/kernel/ssd_update_test.rsdeleted file mode 100644index 84c885c75..000000000--- a/crates/uzu/tests/unit/kernel/ssd_update_test.rs+++ /dev/null@@ -1,281 +0,0 @@-#![cfg(metal_backend)]--use bytemuck;-use half::bf16;-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};-use uzu::{- DataType,- backends::{- common::{- Backend, Context, Encoder, Kernels, kernel::SSDUpdateKernel,- },- metal::Metal,- },-};--#[allow(dead_code)]-fn ssm_update_ref_bf16(- x: &[bf16],- dt: &[bf16],- a: &[bf16],- b: &[bf16],- c: &[bf16],- d: &[bf16],- z: &[bf16],- state: &[bf16],- bsz: usize,- h: usize,-) -> (Vec<bf16>, Vec<bf16>) {- let n: usize = 16;- let mut y = vec![bf16::from_f32(0.0); bsz * h];- let mut next_state = vec![bf16::from_f32(0.0); bsz * h * n];- for bb in 0..bsz {- for ch in 0..h {- let idx = bb * h + ch;- let vx = x[idx].to_f32();- let sig = 1.0 / (1.0 + (-vx.abs()).exp());- let this_x = if vx < 0.0 {- (1.0 - sig) * vx- } else {- sig * vx- };- let vz = z[idx].to_f32();- let sigz = 1.0 / (1.0 + (-vz.abs()).exp());- let this_z = if vz < 0.0 {- (1.0 - sigz) * vz- } else {- sigz * vz- };- let delta = (1.0 + dt[idx].to_f32().exp()).ln();- let cb_offset = bb * n;- let state_offset = idx * n;- let mut acc = 0.0f32;- for i in 0..n {- let new_s = state[state_offset + i].to_f32() * (a[i].to_f32() * delta).exp()- + b[cb_offset + i].to_f32() * delta * this_x;- next_state[state_offset + i] = bf16::from_f32(new_s);- acc += new_s * c[cb_offset + i].to_f32();- }- acc += d[ch].to_f32() * this_x;- acc *= this_z;- y[idx] = bf16::from_f32(acc);- }- }- (y, next_state)-}--fn softplus_f32(x: f32) -> f32 {- if x > 20.0 {- x- } else {- (1.0 + x.exp()).ln()- }-}--#[allow(dead_code)]-fn ssd_update_no_z_ref_bf16(- x: &[bf16],- dt_raw: &[bf16],- b: &[bf16],- c: &[bf16],- d: &[bf16],- state: &[bf16],- bsz: usize,- h: usize,- dh: usize,- g: usize,- n: usize,-) -> (Vec<bf16>, Vec<bf16>) {- let group_size = h / g;- let mut y = vec![bf16::from_f32(0.0); bsz * h * dh];- let mut next_state = vec![bf16::from_f32(0.0); bsz * h * dh * n];- for bb in 0..bsz {- for hh in 0..h {- for ddh in 0..dh {- let x_idx = bb * h * dh + hh * dh + ddh;- let dt_idx = bb * h + hh;- let cb_idx0 = bb * g * n + (hh / group_size) * n;- let state_idx0 = (bb * h * dh + hh * dh + ddh) * n;- let this_x = x[x_idx].to_f32();- let dt_raw_val = dt_raw[dt_idx].to_f32();- let this_dt = softplus_f32(dt_raw_val);- let this_decay = (-this_dt).exp();- let this_d = d[hh].to_f32();- let dt_safe = this_dt.max(1e-6);- let normalized_x = this_x / dt_safe;- let dt_scaled_input = normalized_x * this_dt;- let mut acc = 0.0f32;- for i in 0..n {- let s_old = state[state_idx0 + i].to_f32();- let new_s = s_old * this_decay + b[cb_idx0 + i].to_f32() * dt_scaled_input;- next_state[state_idx0 + i] = bf16::from_f32(new_s);- acc += new_s * c[cb_idx0 + i].to_f32();- }- acc += this_d * this_x;- y[x_idx] = bf16::from_f32(acc);- }- }- }- (y, next_state)-}--fn ssd_update_ref_bf16(- x: &[bf16],- dt_raw: &[bf16],- b: &[bf16],- c: &[bf16],- d: &[bf16],- z: &[bf16],- state: &[bf16],- bsz: usize,- h: usize,- dh: usize,- g: usize,- n: usize,-) -> (Vec<bf16>, Vec<bf16>) {- let group_size = h / g;- let mut y = vec![bf16::from_f32(0.0); bsz * h * dh];- let mut next_state = vec![bf16::from_f32(0.0); bsz * h * dh * n];- for bb in 0..bsz {- for hh in 0..h {- for ddh in 0..dh {- let x_idx = bb * h * dh + hh * dh + ddh;- let dt_idx = bb * h + hh;- let cb_idx0 = bb * g * n + (hh / group_size) * n;- let state_idx0 = (bb * h * dh + hh * dh + ddh) * n;- let this_x = x[x_idx].to_f32();- let dt_raw_val = dt_raw[dt_idx].to_f32();- let this_dt = softplus_f32(dt_raw_val);- let this_decay = (-this_dt).exp();- let this_d = d[hh].to_f32();- let dt_safe = this_dt.max(1e-6);- let normalized_x = this_x / dt_safe;- let dt_scaled_input = normalized_x * this_dt;- let vz = z[x_idx].to_f32();- let sigz = 1.0 / (1.0 + (-vz.abs()).exp());- let this_z = if vz < 0.0 {- (1.0 - sigz) * vz- } else {- sigz * vz- };- let mut acc = 0.0f32;- for i in 0..n {- let s_old = state[state_idx0 + i].to_f32();- let new_s = s_old * this_decay + b[cb_idx0 + i].to_f32() * dt_scaled_input;- next_state[state_idx0 + i] = bf16::from_f32(new_s);- acc += new_s * c[cb_idx0 + i].to_f32();- }- acc += this_d * this_x;- acc *= this_z;- y[x_idx] = bf16::from_f32(acc);- }- }- }- (y, next_state)-}--#[test]-fn ssd_update_with_z_bf16() {- let Some(ctx) = <Metal as Backend>::Context::new().ok() else {- eprintln!("Skipping: no Metal device");- return;- };- let bsz = 1usize;- let h = 4usize;- let dh = 3usize;- let g = 2usize;- let n = 8usize;-- let x: Vec<bf16> = (0..bsz * h * dh).map(|i| bf16::from_f32(((i % 7) as f32) * 0.1 - 0.2)).collect();- let z: Vec<bf16> = (0..bsz * h * dh).map(|i| bf16::from_f32(((i % 5) as f32) * 0.1 - 0.1)).collect();- let dt: Vec<bf16> = (0..bsz * h).map(|i| bf16::from_f32(((i % 5) as f32) * 0.3 - 1.0)).collect();- let b: Vec<bf16> = (0..bsz * g * n).map(|i| bf16::from_f32(((i % 11) as f32) * 0.02 - 0.05)).collect();- let c: Vec<bf16> = (0..bsz * g * n).map(|i| bf16::from_f32(((i % 13) as f32) * 0.015)).collect();- let d: Vec<bf16> = (0..h).map(|i| bf16::from_f32(((i % 3) as f32) * 0.05)).collect();- let state: Vec<bf16> = (0..bsz * h * dh * n).map(|i| bf16::from_f32(((i % 23) as f32) * 0.01 - 0.05)).collect();-- let (y_exp, ns_exp) = ssd_update_ref_bf16(&x, &dt, &b, &c, &d, &z, &state, bsz, h, dh, g, n);- let y_exp_f32: Vec<f32> = y_exp.iter().map(|&v| v.to_f32()).collect();- let ns_exp_f32: Vec<f32> = ns_exp.iter().map(|&v| v.to_f32()).collect();-- let x_strides = [(h * dh) as u32, dh as u32, 1u32];- let dt_strides = [h as u32, 1u32];- let cb_strides = [(g * n) as u32, n as u32, 1u32];- let state_strides = [(h * dh * n) as u32, (dh * n) as u32, n as u32, 1u32];-- let x_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(&x), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let dt_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(&dt), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let b_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(&b), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let c_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(&c), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let d_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(&d), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let z_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(&z), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let state_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(&state), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let mut y_buf = ctx- .device- .new_buffer(bsz * h * dh * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let mut ns_buf = ctx- .device- .new_buffer(bsz * h * dh * n * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let kernel = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&ctx, DataType::BF16, false).unwrap();- let mut encoder = Encoder::new(ctx.as_ref()).unwrap();- kernel.encode(- &x_buf,- &dt_buf,- &b_buf,- &c_buf,- &d_buf,- &z_buf,- Some(&state_buf),- &mut y_buf,- &mut ns_buf,- (h / g) as u32,- n as u32,- x_strides.as_slice(),- dt_strides.as_slice(),- cb_strides.as_slice(),- state_strides.as_slice(),- bsz as u32,- h as u32,- dh as u32,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().unwrap();-- let y_ptr = y_buf.contents().as_ptr() as *const bf16;- let y_out = unsafe { std::slice::from_raw_parts(y_ptr, bsz * h * dh) };- let y_out_f32: Vec<f32> = y_out.iter().map(|&v| v.to_f32()).collect();- let ns_ptr = ns_buf.contents().as_ptr() as *const bf16;- let ns_out = unsafe { std::slice::from_raw_parts(ns_ptr, bsz * h * dh * n) };- let ns_out_f32: Vec<f32> = ns_out.iter().map(|&v| v.to_f32()).collect();- let tol = 2e-2;- for (i, (a, b)) in y_out_f32.iter().zip(y_exp_f32.iter()).enumerate() {- assert!((a - b).abs() < tol, "y {} {} {}", i, a, b);- }- for (i, (a, b)) in ns_out_f32.iter().zip(ns_exp_f32.iter()).enumerate() {- assert!((a - b).abs() < tol, "ns {} {} {}", i, a, b);- }-}diff --git a/crates/uzu/tests/unit/kernel/ssm/mod.rs b/crates/uzu/tests/unit/kernel/ssm/mod.rsnew file mode 100644index 000000000..f704533c4--- /dev/null+++ b/crates/uzu/tests/unit/kernel/ssm/mod.rs@@ -0,0 +1,6 @@+mod ssm_conv1d_decode_test;+mod ssm_conv1d_pack_test;+mod ssm_conv1d_scan_test;+mod ssm_prefill_test;+mod ssm_split_inproj_test;+mod ssm_update_test;diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rsnew file mode 100644index 000000000..40210ca72--- /dev/null+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs@@ -0,0 +1,348 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::Conv1dDecodeKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ x: Box<[T]>,+ w: Box<[T]>,+ b: Option<Box<[T]>>,+ state: Box<[T]>,+ kernel_size: u32,+ row_stride: u32,+ state_stride: u32,+ num_channels: u32,+ suffix_len: u32,+ inner_dim: u32,+ proj_dim: u32,+ activation_type: ActivationType,+ state_in_place: bool,+}++struct Output<T: ArrayElement + Float> {+ x_out: Vec<T>,+ b_out: Vec<T>,+ c_out: Vec<T>,+ next_state: Vec<T>,+}++fn get_input<T: ArrayElement + Float>(+ kernel_size: u32,+ num_channels: u32,+ inner_dim: u32,+ proj_dim: u32,+ suffix_len: u32,+ has_bias: bool,+ state_in_place: bool,+ activation_type: ActivationType,+) -> Input<T> {+ let row_stride = num_channels;+ let state_stride = kernel_size.saturating_sub(1);+ let state_size = num_channels as usize * state_stride as usize;+ let x_size = suffix_len as usize * row_stride as usize;+ let w_size = num_channels as usize * kernel_size as usize;++ let x: Vec<T> = (0..x_size).map(|i| T::from(0.1 * ((i as f64 % 7.0) - 3.0)).unwrap()).collect();+ let w: Vec<T> = (0..w_size).map(|i| T::from(0.05 * ((i as f64 % 5.0) - 2.0)).unwrap()).collect();+ let b = if has_bias {+ Some(+ (0..num_channels as usize)+ .map(|i| T::from(0.01 * (i as f64 + 1.0)).unwrap())+ .collect::<Vec<T>>()+ .into_boxed_slice(),+ )+ } else {+ None+ };+ let state: Vec<T> = (0..state_size).map(|i| T::from(0.02 * ((i as f64 % 9.0) - 4.0)).unwrap()).collect();++ Input {+ x: x.into_boxed_slice(),+ w: w.into_boxed_slice(),+ b,+ state: state.into_boxed_slice(),+ kernel_size,+ row_stride,+ state_stride,+ num_channels,+ suffix_len,+ inner_dim,+ proj_dim,+ activation_type,+ state_in_place,+ }+}++fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let has_bias = input.b.is_some();+ let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dDecodeKernel::new(+ &context,+ T::data_type(),+ has_bias,+ input.state_in_place,+ )+ .expect("Failed to create Conv1dDecodeKernel");++ let x_out_size = input.suffix_len as usize * input.inner_dim as usize;+ let b_out_size = input.suffix_len as usize * input.proj_dim as usize;+ let c_out_size = input.suffix_len as usize * input.proj_dim as usize;+ let state_size = input.num_channels as usize * input.state_stride as usize;++ let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");+ let w_array = context.create_array_from(&[input.w.len()], &input.w, "w");+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, "b"));++ let x_out_array = context.create_array_uninitialized(&[x_out_size], T::data_type(), "x_out");+ let b_out_array = context.create_array_uninitialized(&[b_out_size], T::data_type(), "b_out");+ let c_out_array = context.create_array_uninitialized(&[c_out_size], T::data_type(), "c_out");++ let b_buf = b_array.as_ref().map(|a| a.buffer());+ let b_borrow = b_buf.as_ref().map(|rc| rc.borrow());+ let b_deref: Option<&B::Buffer> = b_borrow.as_ref().map(|b| b.deref());++ if input.state_in_place {+ let next_state_array = context.create_array_from(&[state_size], &input.state, "next_state");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ x_array.buffer().borrow().deref(),+ w_array.buffer().borrow().deref(),+ b_deref,+ None::<&B::Buffer>,+ x_out_array.buffer().borrow_mut().deref_mut(),+ b_out_array.buffer().borrow_mut().deref_mut(),+ c_out_array.buffer().borrow_mut().deref_mut(),+ next_state_array.buffer().borrow_mut().deref_mut(),+ input.kernel_size,+ input.row_stride,+ input.state_stride,+ input.num_channels,+ input.suffix_len,+ input.inner_dim,+ input.proj_dim,+ input.activation_type,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ Output {+ x_out: x_out_array.as_slice().to_vec(),+ b_out: b_out_array.as_slice().to_vec(),+ c_out: c_out_array.as_slice().to_vec(),+ next_state: next_state_array.as_slice().to_vec(),+ }+ } else {+ let state_array = context.create_array_from(&[state_size], &input.state, "state");+ let next_state_array = context.create_array_uninitialized(&[state_size], T::data_type(), "next_state");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ x_array.buffer().borrow().deref(),+ w_array.buffer().borrow().deref(),+ b_deref,+ Some(state_array.buffer().borrow().deref()),+ x_out_array.buffer().borrow_mut().deref_mut(),+ b_out_array.buffer().borrow_mut().deref_mut(),+ c_out_array.buffer().borrow_mut().deref_mut(),+ next_state_array.buffer().borrow_mut().deref_mut(),+ input.kernel_size,+ input.row_stride,+ input.state_stride,+ input.num_channels,+ input.suffix_len,+ input.inner_dim,+ input.proj_dim,+ input.activation_type,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ Output {+ x_out: x_out_array.as_slice().to_vec(),+ b_out: b_out_array.as_slice().to_vec(),+ c_out: c_out_array.as_slice().to_vec(),+ next_state: next_state_array.as_slice().to_vec(),+ }+ }+}++fn get_test_data<T: ArrayElement + Float>(+ kernel_size: u32,+ num_channels: u32,+ inner_dim: u32,+ proj_dim: u32,+ suffix_len: u32,+ has_bias: bool,+ state_in_place: bool,+ activation_type: ActivationType,+) -> (Input<T>, Output<T>) {+ let input = get_input::<T>(+ kernel_size,+ num_channels,+ inner_dim,+ proj_dim,+ suffix_len,+ has_bias,+ state_in_place,+ activation_type,+ );+ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &Output<T>,+ label: &str,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-2f32+ } else {+ 1e-5+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input);+ let backend_name = std::any::type_name::<B>();+ let type_name = std::any::type_name::<T>();++ assert_eq_float::<T>(+ &expected.x_out,+ &output.x_out,+ eps,+ &format!("Conv1dDecode x_out {backend_name} {label} (type={type_name})"),+ );+ assert_eq_float::<T>(+ &expected.b_out,+ &output.b_out,+ eps,+ &format!("Conv1dDecode b_out {backend_name} {label} (type={type_name})"),+ );+ assert_eq_float::<T>(+ &expected.c_out,+ &output.c_out,+ eps,+ &format!("Conv1dDecode c_out {backend_name} {label} (type={type_name})"),+ );+ assert_eq_float::<T>(+ &expected.next_state,+ &output.next_state,+ eps,+ &format!("Conv1dDecode next_state {backend_name} {label} (type={type_name})"),+ );+ });+}++// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4+fn test_basic<T: ArrayElement + Float + Debug + Display>() {+ for state_in_place in [true, false] {+ for has_bias in [true, false] {+ let label = format!("basic(bias={has_bias},in_place={state_in_place})");+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, has_bias, state_in_place, ActivationType::SILU);+ test_internal(&input, &expected, &label);+ }+ }+}++// kernel_size=2 (state_stride=1, minimal state)+fn test_small_kernel<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 1, true, true, ActivationType::SILU);+ test_internal(&input, &expected, "small_kernel");+}++// Larger dimensions closer to real usage+fn test_large<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 1, true, true, ActivationType::SILU);+ test_internal(&input, &expected, "large");+}++fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, false, true, ActivationType::IDENTITY);+ test_internal(&input, &expected, "identity_activation");+}++fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::GELU);+ test_internal(&input, &expected, "gelu_activation");+}++fn test_tanh_activation<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::TANH);+ test_internal(&input, &expected, "tanh_activation");+}++// f32+#[test]+fn test_basic_f32() {+ test_basic::<f32>();+}++#[test]+fn test_small_kernel_f32() {+ test_small_kernel::<f32>();+}++#[test]+fn test_large_f32() {+ test_large::<f32>();+}++#[test]+fn test_identity_activation_f32() {+ test_identity_activation::<f32>();+}++#[test]+fn test_gelu_activation_f32() {+ test_gelu_activation::<f32>();+}++#[test]+fn test_tanh_activation_f32() {+ test_tanh_activation::<f32>();+}++// f16+#[test]+fn test_basic_f16() {+ test_basic::<f16>();+}++#[test]+fn test_small_kernel_f16() {+ test_small_kernel::<f16>();+}++#[test]+fn test_large_f16() {+ test_large::<f16>();+}++// bf16+#[test]+fn test_basic_bf16() {+ test_basic::<bf16>();+}++#[test]+fn test_small_kernel_bf16() {+ test_small_kernel::<bf16>();+}++#[test]+fn test_large_bf16() {+ test_large::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rsnew file mode 100644index 000000000..29d026dcd--- /dev/null+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs@@ -0,0 +1,204 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::Conv1dPackKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ state_in: Box<[T]>,+ x: Box<[T]>,+ state_stride: u32,+ row_stride: u32,+ suffix_len: u32,+ num_channels: u32,+}++fn get_input<T: ArrayElement + Float>(+ state_stride: u32,+ row_stride: u32,+ suffix_len: u32,+ num_channels: u32,+) -> Input<T> {+ let state_size = num_channels as usize * state_stride as usize;+ let x_size = suffix_len as usize * row_stride as usize;++ let state_in: Vec<T> = (0..state_size).map(|i| T::from(0.1 * (i as f64 + 1.0)).unwrap()).collect();+ let x: Vec<T> = (0..x_size).map(|i| T::from(-0.05 * (i as f64 + 1.0)).unwrap()).collect();++ Input {+ state_in: state_in.into_boxed_slice(),+ x: x.into_boxed_slice(),+ state_stride,+ row_stride,+ suffix_len,+ num_channels,+ }+}++fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, T::data_type())+ .expect("Failed to create Conv1dPackKernel");++ let state_size = input.state_in.len();+ let x_size = input.x.len();+ let total_rows = input.state_stride as usize + input.suffix_len as usize;+ let padded_size = total_rows * input.row_stride as usize;++ let state_array = context.create_array_from(&[state_size], &input.state_in, "state_in");+ let x_array = context.create_array_from(&[x_size], &input.x, "x");+ let padded_array = context.create_array_uninitialized(&[padded_size], T::data_type(), "padded");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ state_array.buffer().borrow().deref(),+ x_array.buffer().borrow().deref(),+ padded_array.buffer().borrow_mut().deref_mut(),+ input.state_stride,+ input.row_stride,+ input.suffix_len,+ input.num_channels,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ padded_array.as_slice().to_vec()+}++fn get_test_data<T: ArrayElement + Float>(+ state_stride: u32,+ row_stride: u32,+ suffix_len: u32,+ num_channels: u32,+) -> (Input<T>, Vec<T>) {+ let input = get_input::<T>(state_stride, row_stride, suffix_len, num_channels);+ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &[T],+ label: &str,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-3f32+ } else {+ 1e-6+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input);+ let msg = format!("Conv1dPack {} {} (type={})", std::any::type_name::<B>(), label, std::any::type_name::<T>(),);+ assert_eq_float::<T>(expected, &output, eps, &msg);+ });+}++fn test_basic<T: ArrayElement + Float + Debug + Display>() {+ // 4 channels, state_stride=3 (kernel_size=4), 2 tokens+ let (input, expected) = get_test_data::<T>(3, 4, 2, 4);+ test_internal(&input, &expected, "basic");+}++fn test_single_token<T: ArrayElement + Float + Debug + Display>() {+ // 8 channels, state_stride=1, 1 token+ let (input, expected) = get_test_data::<T>(1, 8, 1, 8);+ test_internal(&input, &expected, "single_token");+}++fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {+ // Minimal state (kernel_size=2), many tokens+ let (input, expected) = get_test_data::<T>(1, 4, 8, 4);+ test_internal(&input, &expected, "many_tokens");+}++fn test_large<T: ArrayElement + Float + Debug + Display>() {+ // Larger dimensions closer to real usage+ let (input, expected) = get_test_data::<T>(3, 128, 16, 128);+ test_internal(&input, &expected, "large");+}++fn test_channels_lt_row_stride<T: ArrayElement + Float + Debug + Display>() {+ // num_channels < row_stride (padded channels that kernel doesn't write)+ let (input, expected) = get_test_data::<T>(2, 8, 3, 6);+ test_internal(&input, &expected, "channels_lt_row_stride");+}++// f32+#[test]+fn test_basic_f32() {+ test_basic::<f32>();+}++#[test]+fn test_single_token_f32() {+ test_single_token::<f32>();+}++#[test]+fn test_many_tokens_f32() {+ test_many_tokens::<f32>();+}++#[test]+fn test_large_f32() {+ test_large::<f32>();+}++#[test]+fn test_channels_lt_row_stride_f32() {+ test_channels_lt_row_stride::<f32>();+}++// f16+#[test]+fn test_basic_f16() {+ test_basic::<f16>();+}++#[test]+fn test_single_token_f16() {+ test_single_token::<f16>();+}++#[test]+fn test_many_tokens_f16() {+ test_many_tokens::<f16>();+}++#[test]+fn test_large_f16() {+ test_large::<f16>();+}++// bf16+#[test]+fn test_basic_bf16() {+ test_basic::<bf16>();+}++#[test]+fn test_single_token_bf16() {+ test_single_token::<bf16>();+}++#[test]+fn test_many_tokens_bf16() {+ test_many_tokens::<bf16>();+}++#[test]+fn test_large_bf16() {+ test_large::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rsnew file mode 100644index 000000000..9d249dd38--- /dev/null+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs@@ -0,0 +1,331 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::Conv1dScanKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ padded: Box<[T]>,+ w: Box<[T]>,+ b: Option<Box<[T]>>,+ suffix_len: u32,+ kernel_size: u32,+ row_stride: u32,+ state_stride: u32,+ num_channels: u32,+ inner_dim: u32,+ proj_dim: u32,+ activation_type: ActivationType,+}++struct Output<T: ArrayElement + Float> {+ x_out: Vec<T>,+ b_out: Vec<T>,+ c_out: Vec<T>,+ state_out: Vec<T>,+}++fn get_input<T: ArrayElement + Float>(+ kernel_size: u32,+ num_channels: u32,+ inner_dim: u32,+ proj_dim: u32,+ suffix_len: u32,+ has_bias: bool,+ activation_type: ActivationType,+) -> Input<T> {+ let row_stride = num_channels;+ let state_stride = kernel_size.saturating_sub(1);+ let total_rows = (suffix_len + state_stride) as usize;+ let padded_size = total_rows * row_stride as usize;+ let w_size = num_channels as usize * kernel_size as usize;++ let padded: Vec<T> = (0..padded_size).map(|i| T::from(0.1 * ((i as f64 % 11.0) - 5.0)).unwrap()).collect();+ let w: Vec<T> = (0..w_size).map(|i| T::from(0.05 * ((i as f64 % 5.0) - 2.0)).unwrap()).collect();+ let b = if has_bias {+ Some(+ (0..num_channels as usize)+ .map(|i| T::from(0.01 * (i as f64 + 1.0)).unwrap())+ .collect::<Vec<T>>()+ .into_boxed_slice(),+ )+ } else {+ None+ };++ Input {+ padded: padded.into_boxed_slice(),+ w: w.into_boxed_slice(),+ b,+ suffix_len,+ kernel_size,+ row_stride,+ state_stride,+ num_channels,+ inner_dim,+ proj_dim,+ activation_type,+ }+}++fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let has_bias = input.b.is_some();+ let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&context, T::data_type(), has_bias)+ .expect("Failed to create Conv1dScanKernel");++ let x_out_size = input.suffix_len as usize * input.inner_dim as usize;+ let b_out_size = input.suffix_len as usize * input.proj_dim as usize;+ let c_out_size = input.suffix_len as usize * input.proj_dim as usize;+ let state_size = input.num_channels as usize * input.state_stride as usize;++ let padded_array = context.create_array_from(&[input.padded.len()], &input.padded, "padded");+ let w_array = context.create_array_from(&[input.w.len()], &input.w, "w");+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, "b"));++ let x_out_array = context.create_array_uninitialized(&[x_out_size], T::data_type(), "x_out");+ let b_out_array = context.create_array_uninitialized(&[b_out_size], T::data_type(), "b_out");+ let c_out_array = context.create_array_uninitialized(&[c_out_size], T::data_type(), "c_out");+ let state_out_array = context.create_array_uninitialized(&[state_size], T::data_type(), "state_out");++ let b_buf = b_array.as_ref().map(|a| a.buffer());+ let b_borrow = b_buf.as_ref().map(|rc| rc.borrow());+ let b_deref: Option<&B::Buffer> = b_borrow.as_ref().map(|b| b.deref());++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ padded_array.buffer().borrow().deref(),+ w_array.buffer().borrow().deref(),+ b_deref,+ x_out_array.buffer().borrow_mut().deref_mut(),+ b_out_array.buffer().borrow_mut().deref_mut(),+ c_out_array.buffer().borrow_mut().deref_mut(),+ state_out_array.buffer().borrow_mut().deref_mut(),+ input.suffix_len,+ input.kernel_size,+ input.row_stride,+ input.state_stride,+ input.num_channels,+ input.inner_dim,+ input.proj_dim,+ input.activation_type,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ Output {+ x_out: x_out_array.as_slice().to_vec(),+ b_out: b_out_array.as_slice().to_vec(),+ c_out: c_out_array.as_slice().to_vec(),+ state_out: state_out_array.as_slice().to_vec(),+ }+}++fn get_test_data<T: ArrayElement + Float>(+ kernel_size: u32,+ num_channels: u32,+ inner_dim: u32,+ proj_dim: u32,+ suffix_len: u32,+ has_bias: bool,+ activation_type: ActivationType,+) -> (Input<T>, Output<T>) {+ let input = get_input::<T>(kernel_size, num_channels, inner_dim, proj_dim, suffix_len, has_bias, activation_type);+ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &Output<T>,+ label: &str,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-2f32+ } else {+ 1e-5+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input);+ let backend_name = std::any::type_name::<B>();+ let type_name = std::any::type_name::<T>();++ assert_eq_float::<T>(+ &expected.x_out,+ &output.x_out,+ eps,+ &format!("Conv1dScan x_out {backend_name} {label} (type={type_name})"),+ );+ assert_eq_float::<T>(+ &expected.b_out,+ &output.b_out,+ eps,+ &format!("Conv1dScan b_out {backend_name} {label} (type={type_name})"),+ );+ assert_eq_float::<T>(+ &expected.c_out,+ &output.c_out,+ eps,+ &format!("Conv1dScan c_out {backend_name} {label} (type={type_name})"),+ );+ assert_eq_float::<T>(+ &expected.state_out,+ &output.state_out,+ eps,+ &format!("Conv1dScan state_out {backend_name} {label} (type={type_name})"),+ );+ });+}++// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4, suffix_len=2+fn test_basic<T: ArrayElement + Float + Debug + Display>() {+ for has_bias in [true, false] {+ let label = format!("basic(bias={has_bias})");+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, has_bias, ActivationType::SILU);+ test_internal(&input, &expected, &label);+ }+}++// kernel_size=2 (state_stride=1, minimal state)+fn test_small_kernel<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 3, true, ActivationType::SILU);+ test_internal(&input, &expected, "small_kernel");+}++// Single token suffix+fn test_single_token<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, ActivationType::SILU);+ test_internal(&input, &expected, "single_token");+}++// Many tokens+fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 16, true, ActivationType::SILU);+ test_internal(&input, &expected, "many_tokens");+}++// Larger dimensions closer to real usage+fn test_large<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 8, true, ActivationType::SILU);+ test_internal(&input, &expected, "large");+}++fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, false, ActivationType::IDENTITY);+ test_internal(&input, &expected, "identity_activation");+}++fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::GELU);+ test_internal(&input, &expected, "gelu_activation");+}++fn test_tanh_activation<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::TANH);+ test_internal(&input, &expected, "tanh_activation");+}++// f32+#[test]+fn test_basic_f32() {+ test_basic::<f32>();+}++#[test]+fn test_small_kernel_f32() {+ test_small_kernel::<f32>();+}++#[test]+fn test_single_token_f32() {+ test_single_token::<f32>();+}++#[test]+fn test_many_tokens_f32() {+ test_many_tokens::<f32>();+}++#[test]+fn test_large_f32() {+ test_large::<f32>();+}++#[test]+fn test_identity_activation_f32() {+ test_identity_activation::<f32>();+}++#[test]+fn test_gelu_activation_f32() {+ test_gelu_activation::<f32>();+}++#[test]+fn test_tanh_activation_f32() {+ test_tanh_activation::<f32>();+}++// f16+#[test]+fn test_basic_f16() {+ test_basic::<f16>();+}++#[test]+fn test_small_kernel_f16() {+ test_small_kernel::<f16>();+}++#[test]+fn test_single_token_f16() {+ test_single_token::<f16>();+}++#[test]+fn test_many_tokens_f16() {+ test_many_tokens::<f16>();+}++#[test]+fn test_large_f16() {+ test_large::<f16>();+}++// bf16+#[test]+fn test_basic_bf16() {+ test_basic::<bf16>();+}++#[test]+fn test_small_kernel_bf16() {+ test_small_kernel::<bf16>();+}++#[test]+fn test_single_token_bf16() {+ test_single_token::<bf16>();+}++#[test]+fn test_many_tokens_bf16() {+ test_many_tokens::<bf16>();+}++#[test]+fn test_large_bf16() {+ test_large::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_prefill_test.rsnew file mode 100644index 000000000..dc2b25316--- /dev/null+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs@@ -0,0 +1,290 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{+ Backend, Context, Encoder, Kernels,+ kernel::{SSDPrefillKernel, SSDPrefillSequentialKernel},+ },+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ x: Box<[T]>,+ dt: Box<[T]>,+ b: Box<[T]>,+ c: Box<[T]>,+ d: Box<[T]>,+ z: Box<[T]>,+ state: Box<[T]>,+ suffix_len: usize,+ num_heads: usize,+ head_dim: usize,+ state_dim: usize,+ group_size: u32,+ x_strides: [usize; 3],+ dt_strides: [usize; 2],+ cb_strides: [usize; 3],+ state_strides: [usize; 3],+}++struct Output<T: ArrayElement + Float> {+ y: Vec<T>,+ state: Vec<T>,+}++enum KernelType {+ Prefill,+ Sequential,+}++fn get_input<T: ArrayElement + Float>(+ suffix_len: usize,+ num_heads: usize,+ head_dim: usize,+ state_dim: usize,+ group_size: u32,+) -> Input<T> {+ let safe_group = group_size.max(1) as usize;+ let group_count = num_heads / safe_group;+ let total_x = suffix_len * num_heads * head_dim;+ let total_dt = suffix_len * num_heads;+ let total_cb = suffix_len * group_count * state_dim;+ let total_state = num_heads * head_dim * state_dim;++ let x: Vec<T> = (0..total_x).map(|i| T::from(((i % 17) as f64) * 0.01 - 0.05).unwrap()).collect();+ let dt: Vec<T> = (0..total_dt).map(|i| T::from(((i % 13) as f64) * 0.2 - 1.5).unwrap()).collect();+ let b: Vec<T> = (0..total_cb).map(|i| T::from(((i % 11) as f64) * 0.02 - 0.05).unwrap()).collect();+ let c: Vec<T> = (0..total_cb).map(|i| T::from(((i % 19) as f64) * 0.01 - 0.02).unwrap()).collect();+ let d: Vec<T> = (0..num_heads).map(|i| T::from(((i % 3) as f64) * 0.05 - 0.05).unwrap()).collect();+ let z: Vec<T> = (0..total_x).map(|i| T::from(((i % 23) as f64) * 0.02 - 0.1).unwrap()).collect();+ let state: Vec<T> = (0..total_state).map(|i| T::from(((i % 29) as f64) * 0.03 - 0.4).unwrap()).collect();++ let x_strides = [num_heads * head_dim, head_dim, 1];+ let dt_strides = [num_heads, 1];+ let cb_strides = [group_count * state_dim, state_dim, 1];+ let state_strides = [head_dim * state_dim, state_dim, 1];++ Input {+ x: x.into_boxed_slice(),+ dt: dt.into_boxed_slice(),+ b: b.into_boxed_slice(),+ c: c.into_boxed_slice(),+ d: d.into_boxed_slice(),+ z: z.into_boxed_slice(),+ state: state.into_boxed_slice(),+ suffix_len,+ num_heads,+ head_dim,+ state_dim,+ group_size,+ x_strides,+ dt_strides,+ cb_strides,+ state_strides,+ }+}++fn get_output<B: Backend, T: ArrayElement + Float>(+ input: &Input<T>,+ kernel_type: &KernelType,+) -> Output<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let total_x = input.suffix_len * input.num_heads * input.head_dim;++ let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");+ let dt_array = context.create_array_from(&[input.dt.len()], &input.dt, "dt");+ let b_array = context.create_array_from(&[input.b.len()], &input.b, "b");+ let c_array = context.create_array_from(&[input.c.len()], &input.c, "c");+ let d_array = context.create_array_from(&[input.d.len()], &input.d, "d");+ let z_array = context.create_array_from(&[input.z.len()], &input.z, "z");+ let state_array = context.create_array_from(&[input.state.len()], &input.state, "state");+ let y_array = context.create_array_uninitialized(&[total_x], T::data_type(), "y");++ let x_strides: Vec<u32> = input.x_strides.iter().map(|&s| s as u32).collect();+ let dt_strides: Vec<u32> = input.dt_strides.iter().map(|&s| s as u32).collect();+ let cb_strides: Vec<u32> = input.cb_strides.iter().map(|&s| s as u32).collect();+ let state_strides: Vec<u32> = input.state_strides.iter().map(|&s| s as u32).collect();++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ match kernel_type {+ KernelType::Prefill => {+ let kernel = <<B as Backend>::Kernels as Kernels>::SSDPrefillKernel::new(&context, T::data_type())+ .expect("Failed to create SSDPrefillKernel");+ kernel.encode(+ x_array.buffer().borrow().deref(),+ dt_array.buffer().borrow().deref(),+ b_array.buffer().borrow().deref(),+ c_array.buffer().borrow().deref(),+ d_array.buffer().borrow().deref(),+ z_array.buffer().borrow().deref(),+ state_array.buffer().borrow_mut().deref_mut(),+ y_array.buffer().borrow_mut().deref_mut(),+ input.suffix_len as u32,+ input.group_size as u32,+ input.state_dim as u32,+ &x_strides,+ &dt_strides,+ &cb_strides,+ &state_strides,+ input.num_heads as u32,+ input.head_dim as u32,+ &mut encoder,+ );+ },+ KernelType::Sequential => {+ let kernel =+ <<B as Backend>::Kernels as Kernels>::SSDPrefillSequentialKernel::new(&context, T::data_type())+ .expect("Failed to create SSDPrefillSequentialKernel");+ kernel.encode(+ x_array.buffer().borrow().deref(),+ dt_array.buffer().borrow().deref(),+ b_array.buffer().borrow().deref(),+ c_array.buffer().borrow().deref(),+ d_array.buffer().borrow().deref(),+ z_array.buffer().borrow().deref(),+ state_array.buffer().borrow_mut().deref_mut(),+ y_array.buffer().borrow_mut().deref_mut(),+ input.suffix_len as u32,+ input.group_size as u32,+ input.state_dim as u32,+ &x_strides,+ &dt_strides,+ &cb_strides,+ &state_strides,+ input.num_heads as u32,+ input.head_dim as u32,+ &mut encoder,+ );+ },+ };+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ Output {+ y: y_array.as_slice().to_vec(),+ state: state_array.as_slice().to_vec(),+ }+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &Output<T>,+ kernel_type: &KernelType,+ label: &str,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 2e-2f32+ } else {+ 5e-5+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input, kernel_type);+ let backend_name = std::any::type_name::<B>();+ let type_name = std::any::type_name::<T>();++ assert_eq_float::<T>(+ &expected.y,+ &output.y,+ eps,+ &format!("SSDPrefillSequential y {backend_name} {label} (type={type_name})"),+ );+ assert_eq_float::<T>(+ &expected.state,+ &output.state,+ eps,+ &format!("SSDPrefillSequential state {backend_name} {label} (type={type_name})"),+ );+ });+}++// --- test shapes ---++fn test_shape(+ kernel_type: &KernelType,+ suffix_len: usize,+ num_heads: usize,+ head_dim: usize,+ state_dim: usize,+ group_size: u32,+ label: &str,+) {+ fn run<T: ArrayElement + Float + Debug + Display>(+ kernel_type: &KernelType,+ suffix_len: usize,+ num_heads: usize,+ head_dim: usize,+ state_dim: usize,+ group_size: u32,+ label: &str,+ ) {+ let input = get_input::<T>(suffix_len, num_heads, head_dim, state_dim, group_size);+ let expected = get_output::<Cpu, T>(&input, &KernelType::Sequential);+ test_internal(&input, &expected, kernel_type, label);+ }+ run::<f32>(kernel_type, suffix_len, num_heads, head_dim, state_dim, group_size, label);+ run::<f16>(kernel_type, suffix_len, num_heads, head_dim, state_dim, group_size, label);+ run::<bf16>(kernel_type, suffix_len, num_heads, head_dim, state_dim, group_size, label);+}++// --- Sequential ---+#[test]+fn test_sequential_basic() {+ test_shape(&KernelType::Sequential, 512, 32, 64, 64, 1, "sequential_basic");+}++#[test]+fn test_sequential_small() {+ test_shape(&KernelType::Sequential, 4, 4, 4, 8, 1, "sequential_small");+}++#[test]+fn test_sequential_minimal() {+ test_shape(&KernelType::Sequential, 1, 1, 1, 1, 1, "sequential_minimal");+}++#[test]+fn test_sequential_multi_group() {+ test_shape(&KernelType::Sequential, 8, 8, 4, 16, 4, "sequential_multi_group");+}++#[test]+fn test_sequential_group_per_head() {+ test_shape(&KernelType::Sequential, 8, 4, 4, 8, 1, "sequential_group_per_head");+}++// --- Prefill ---+#[test]+fn test_prefill_basic() {+ test_shape(&KernelType::Prefill, 512, 32, 64, 64, 1, "prefill_basic");+}++#[test]+fn test_prefill_small() {+ test_shape(&KernelType::Prefill, 4, 4, 4, 8, 1, "prefill_small");+}++#[test]+fn test_prefill_minimal() {+ test_shape(&KernelType::Prefill, 1, 1, 1, 1, 1, "prefill_minimal");+}++#[test]+fn test_prefill_multi_group() {+ test_shape(&KernelType::Prefill, 8, 8, 4, 16, 4, "prefill_multi_group");+}++#[test]+fn test_prefill_group_per_head() {+ test_shape(&KernelType::Prefill, 8, 4, 4, 8, 1, "prefill_group_per_head");+}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rsnew file mode 100644index 000000000..e7ed1d3a1--- /dev/null+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs@@ -0,0 +1,240 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::SplitInProjKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ input: Box<[T]>,+ z_bias: Box<[T]>,+ suffix_length: u32,+ total_dim: u32,+ conv_dim: u32,+ inner_dim: u32,+ num_heads: u32,+}++struct Output<T: ArrayElement + Float> {+ conv_out: Vec<T>,+ z_out: Vec<T>,+ dt_out: Vec<T>,+}++fn get_input<T: ArrayElement + Float>(+ suffix_length: u32,+ conv_dim: u32,+ inner_dim: u32,+ num_heads: u32,+) -> Input<T> {+ let total_dim = conv_dim + inner_dim + num_heads;+ let input_size = suffix_length as usize * total_dim as usize;++ let input: Vec<T> = (0..input_size).map(|i| T::from(0.1 * ((i as f64 % 13.0) - 6.0)).unwrap()).collect();+ let z_bias: Vec<T> = (0..inner_dim as usize).map(|i| T::from(0.05 * ((i as f64 % 7.0) - 3.0)).unwrap()).collect();++ Input {+ input: input.into_boxed_slice(),+ z_bias: z_bias.into_boxed_slice(),+ suffix_length,+ total_dim,+ conv_dim,+ inner_dim,+ num_heads,+ }+}++fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::SplitInProjKernel::new(&context, T::data_type())+ .expect("Failed to create SplitInProjKernel");++ let conv_out_size = input.suffix_length as usize * input.conv_dim as usize;+ let z_out_size = input.suffix_length as usize * input.inner_dim as usize;+ let dt_out_size = input.suffix_length as usize * input.num_heads as usize;++ let input_array = context.create_array_from(&[input.input.len()], &input.input, "input");+ let z_bias_array = context.create_array_from(&[input.z_bias.len()], &input.z_bias, "z_bias");++ let conv_out_array = context.create_array_uninitialized(&[conv_out_size], T::data_type(), "conv_out");+ let z_out_array = context.create_array_uninitialized(&[z_out_size], T::data_type(), "z_out");+ let dt_out_array = context.create_array_uninitialized(&[dt_out_size], T::data_type(), "dt_out");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ input_array.buffer().borrow().deref(),+ conv_out_array.buffer().borrow_mut().deref_mut(),+ z_out_array.buffer().borrow_mut().deref_mut(),+ dt_out_array.buffer().borrow_mut().deref_mut(),+ z_bias_array.buffer().borrow().deref(),+ input.suffix_length,+ input.total_dim,+ input.conv_dim,+ input.inner_dim,+ input.num_heads,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ Output {+ conv_out: conv_out_array.as_slice().to_vec(),+ z_out: z_out_array.as_slice().to_vec(),+ dt_out: dt_out_array.as_slice().to_vec(),+ }+}++fn get_test_data<T: ArrayElement + Float>(+ suffix_length: u32,+ conv_dim: u32,+ inner_dim: u32,+ num_heads: u32,+) -> (Input<T>, Output<T>) {+ let input = get_input::<T>(suffix_length, conv_dim, inner_dim, num_heads);+ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &Output<T>,+ label: &str,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-2f32+ } else {+ 1e-5+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input);+ let backend_name = std::any::type_name::<B>();+ let type_name = std::any::type_name::<T>();++ assert_eq_float::<T>(+ &expected.conv_out,+ &output.conv_out,+ eps,+ &format!("SplitInProj conv_out {backend_name} {label} (type={type_name})"),+ );+ assert_eq_float::<T>(+ &expected.z_out,+ &output.z_out,+ eps,+ &format!("SplitInProj z_out {backend_name} {label} (type={type_name})"),+ );+ assert_eq_float::<T>(+ &expected.dt_out,+ &output.dt_out,+ eps,+ &format!("SplitInProj dt_out {backend_name} {label} (type={type_name})"),+ );+ });+}++// conv_dim=8, inner_dim=4, num_heads=2, suffix_length=2+fn test_basic<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(2, 8, 4, 2);+ test_internal(&input, &expected, "basic");+}++// Single token+fn test_single_token<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(1, 8, 4, 2);+ test_internal(&input, &expected, "single_token");+}++// Many tokens+fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(16, 8, 4, 2);+ test_internal(&input, &expected, "many_tokens");+}++// Larger dimensions closer to real usage+fn test_large<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(8, 128, 64, 32);+ test_internal(&input, &expected, "large");+}++// Minimal dimensions+fn test_minimal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(1, 1, 1, 1);+ test_internal(&input, &expected, "minimal");+}++// f32+#[test]+fn test_basic_f32() {+ test_basic::<f32>();+}++#[test]+fn test_single_token_f32() {+ test_single_token::<f32>();+}++#[test]+fn test_many_tokens_f32() {+ test_many_tokens::<f32>();+}++#[test]+fn test_large_f32() {+ test_large::<f32>();+}++#[test]+fn test_minimal_f32() {+ test_minimal::<f32>();+}++// f16+#[test]+fn test_basic_f16() {+ test_basic::<f16>();+}++#[test]+fn test_single_token_f16() {+ test_single_token::<f16>();+}++#[test]+fn test_many_tokens_f16() {+ test_many_tokens::<f16>();+}++#[test]+fn test_large_f16() {+ test_large::<f16>();+}++// bf16+#[test]+fn test_basic_bf16() {+ test_basic::<bf16>();+}++#[test]+fn test_single_token_bf16() {+ test_single_token::<bf16>();+}++#[test]+fn test_many_tokens_bf16() {+ test_many_tokens::<bf16>();+}++#[test]+fn test_large_bf16() {+ test_large::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/ssm/ssm_update_test.rs b/crates/uzu/tests/unit/kernel/ssm/ssm_update_test.rsnew file mode 100644index 000000000..2cd1c4cd8--- /dev/null+++ b/crates/uzu/tests/unit/kernel/ssm/ssm_update_test.rs@@ -0,0 +1,325 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::SSDUpdateKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ x: Box<[T]>,+ dt: Box<[T]>,+ b: Box<[T]>,+ c: Box<[T]>,+ d: Box<[T]>,+ z: Box<[T]>,+ state: Box<[T]>,+ bsz: u32,+ h: u32,+ dh: u32,+ g: u32,+ n: u32,+ state_in_place: bool,+}++struct Output<T: ArrayElement + Float> {+ y: Vec<T>,+ next_state: Vec<T>,+}++fn get_input<T: ArrayElement + Float>(+ bsz: u32,+ h: u32,+ dh: u32,+ g: u32,+ n: u32,+ state_in_place: bool,+) -> Input<T> {+ let bsz_u = bsz as usize;+ let h_u = h as usize;+ let dh_u = dh as usize;+ let g_u = g as usize;+ let n_u = n as usize;++ let x: Vec<T> = (0..bsz_u * h_u * dh_u).map(|i| T::from(((i % 7) as f64) * 0.1 - 0.2).unwrap()).collect();+ let z: Vec<T> = (0..bsz_u * h_u * dh_u).map(|i| T::from(((i % 5) as f64) * 0.1 - 0.1).unwrap()).collect();+ let dt: Vec<T> = (0..bsz_u * h_u).map(|i| T::from(((i % 5) as f64) * 0.3 - 1.0).unwrap()).collect();+ let b: Vec<T> = (0..bsz_u * g_u * n_u).map(|i| T::from(((i % 11) as f64) * 0.02 - 0.05).unwrap()).collect();+ let c: Vec<T> = (0..bsz_u * g_u * n_u).map(|i| T::from(((i % 13) as f64) * 0.015).unwrap()).collect();+ let d: Vec<T> = (0..h_u).map(|i| T::from(((i % 3) as f64) * 0.05).unwrap()).collect();+ let state: Vec<T> =+ (0..bsz_u * h_u * dh_u * n_u).map(|i| T::from(((i % 23) as f64) * 0.01 - 0.05).unwrap()).collect();++ Input {+ x: x.into_boxed_slice(),+ dt: dt.into_boxed_slice(),+ b: b.into_boxed_slice(),+ c: c.into_boxed_slice(),+ d: d.into_boxed_slice(),+ z: z.into_boxed_slice(),+ state: state.into_boxed_slice(),+ bsz,+ h,+ dh,+ g,+ n,+ state_in_place,+ }+}++fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let kernel =+ <<B as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&context, T::data_type(), input.state_in_place)+ .expect("Failed to create SSDUpdateKernel");++ let bsz = input.bsz as usize;+ let h = input.h as usize;+ let dh = input.dh as usize;+ let g = input.g as usize;+ let n = input.n as usize;++ let x_strides = [(h * dh) as u32, dh as u32, 1u32];+ let dt_strides = [h as u32, 1u32];+ let cb_strides = [(g * n) as u32, n as u32, 1u32];+ let state_strides = [(h * dh * n) as u32, (dh * n) as u32, n as u32, 1u32];++ let y_size = bsz * h * dh;+ let ns_size = bsz * h * dh * n;++ let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");+ let dt_array = context.create_array_from(&[input.dt.len()], &input.dt, "dt");+ let b_array = context.create_array_from(&[input.b.len()], &input.b, "b");+ let c_array = context.create_array_from(&[input.c.len()], &input.c, "c");+ let d_array = context.create_array_from(&[input.d.len()], &input.d, "d");+ let z_array = context.create_array_from(&[input.z.len()], &input.z, "z");++ let y_array = context.create_array_uninitialized(&[y_size], T::data_type(), "y");++ if input.state_in_place {+ let ns_array = context.create_array_from(&[ns_size], &input.state, "next_state");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ x_array.buffer().borrow().deref(),+ dt_array.buffer().borrow().deref(),+ b_array.buffer().borrow().deref(),+ c_array.buffer().borrow().deref(),+ d_array.buffer().borrow().deref(),+ z_array.buffer().borrow().deref(),+ None::<&<B as Backend>::Buffer>,+ y_array.buffer().borrow_mut().deref_mut(),+ ns_array.buffer().borrow_mut().deref_mut(),+ (h / g) as u32,+ input.n,+ x_strides.as_slice(),+ dt_strides.as_slice(),+ cb_strides.as_slice(),+ state_strides.as_slice(),+ input.bsz,+ input.h,+ input.dh,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ Output {+ y: y_array.as_slice().to_vec(),+ next_state: ns_array.as_slice().to_vec(),+ }+ } else {+ let state_array = context.create_array_from(&[ns_size], &input.state, "state");+ let ns_array = context.create_array_uninitialized(&[ns_size], T::data_type(), "next_state");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ x_array.buffer().borrow().deref(),+ dt_array.buffer().borrow().deref(),+ b_array.buffer().borrow().deref(),+ c_array.buffer().borrow().deref(),+ d_array.buffer().borrow().deref(),+ z_array.buffer().borrow().deref(),+ Some(state_array.buffer().borrow().deref()),+ y_array.buffer().borrow_mut().deref_mut(),+ ns_array.buffer().borrow_mut().deref_mut(),+ (h / g) as u32,+ input.n,+ x_strides.as_slice(),+ dt_strides.as_slice(),+ cb_strides.as_slice(),+ state_strides.as_slice(),+ input.bsz,+ input.h,+ input.dh,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ Output {+ y: y_array.as_slice().to_vec(),+ next_state: ns_array.as_slice().to_vec(),+ }+ }+}++fn get_test_data<T: ArrayElement + Float>(+ bsz: u32,+ h: u32,+ dh: u32,+ g: u32,+ n: u32,+ state_in_place: bool,+) -> (Input<T>, Output<T>) {+ let input = get_input::<T>(bsz, h, dh, g, n, state_in_place);+ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &Output<T>,+ label: &str,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 2e-2f32+ } else {+ 1e-5+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input);+ let backend_name = std::any::type_name::<B>();+ let type_name = std::any::type_name::<T>();++ assert_eq_float::<T>(+ &expected.y,+ &output.y,+ eps,+ &format!("SSDUpdate y {backend_name} {label} (type={type_name})"),+ );+ assert_eq_float::<T>(+ &expected.next_state,+ &output.next_state,+ eps,+ &format!("SSDUpdate next_state {backend_name} {label} (type={type_name})"),+ );+ });+}++// bsz=1, h=4, dh=3, g=2, n=8 (matches existing Metal test)+fn test_basic<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, false);+ test_internal(&input, &expected, "basic");+}++// bsz=2 multi-batch+fn test_multi_batch<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(2, 4, 3, 2, 8, false);+ test_internal(&input, &expected, "multi_batch");+}++// state_in_place=true+fn test_state_in_place<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, true);+ test_internal(&input, &expected, "state_in_place");+}++// Larger dimensions closer to real usage+fn test_large<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(1, 8, 4, 4, 16, false);+ test_internal(&input, &expected, "large");+}++// Minimal: single head, single group, dh=1, n=1+fn test_minimal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(1, 1, 1, 1, 1, false);+ test_internal(&input, &expected, "minimal");+}++// g == h (each head is its own group)+fn test_group_per_head<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data::<T>(1, 4, 2, 4, 8, false);+ test_internal(&input, &expected, "group_per_head");+}++// f32+#[test]+fn test_basic_f32() {+ test_basic::<f32>();+}++#[test]+fn test_multi_batch_f32() {+ test_multi_batch::<f32>();+}++#[test]+fn test_state_in_place_f32() {+ test_state_in_place::<f32>();+}++#[test]+fn test_large_f32() {+ test_large::<f32>();+}++#[test]+fn test_minimal_f32() {+ test_minimal::<f32>();+}++#[test]+fn test_group_per_head_f32() {+ test_group_per_head::<f32>();+}++// f16+#[test]+fn test_basic_f16() {+ test_basic::<f16>();+}++#[test]+fn test_multi_batch_f16() {+ test_multi_batch::<f16>();+}++#[test]+fn test_state_in_place_f16() {+ test_state_in_place::<f16>();+}++#[test]+fn test_large_f16() {+ test_large::<f16>();+}++// bf16+#[test]+fn test_basic_bf16() {+ test_basic::<bf16>();+}++#[test]+fn test_multi_batch_bf16() {+ test_multi_batch::<bf16>();+}++#[test]+fn test_state_in_place_bf16() {+ test_state_in_place::<bf16>();+}++#[test]+fn test_large_bf16() {+ test_large::<bf16>();+}