A high-performance inference engine for AI models
Small cleanup part 6 (#957)
uuuvn3602af9b09e5
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rsnew file mode 100644index 000000000..9f97cdadc--- /dev/null+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs@@ -0,0 +1,39 @@+use half::bf16;+use num_traits::Float;+use uzu_engine_macros::kernel;++use crate::data_type::DataTypeRepr;++#[kernel(Conv1dPack)]+#[variants(StateT, f32, bf16)]+#[variants(InputT, f32, bf16)]+pub fn conv1d_pack<StateT: DataTypeRepr + Float, InputT: DataTypeRepr + Float>(+ state_in: *const StateT,+ x: *const InputT,+ padded: *mut StateT,+ state_stride: u32,+ row_stride: u32,+ suffix_len: u32,+ num_channels: u32,+) {+ let state_stride = state_stride as usize;+ let row_stride = row_stride as usize;+ let suffix_len = suffix_len as usize;+ let num_channels = num_channels as usize;++ unsafe {+ for channel_idx in 0..num_channels {+ for row_idx in 0..state_stride + suffix_len {+ let padded_index = row_idx * row_stride + channel_idx;+ if row_idx < state_stride {+ let state_index = channel_idx * state_stride + row_idx;+ *padded.add(padded_index) = *state_in.add(state_index);+ } else {+ let token = row_idx - state_stride;+ let x_index = token * row_stride + channel_idx;+ *padded.add(padded_index) = StateT::from((*x.add(x_index)).to_f32().unwrap()).unwrap();+ }+ }+ }+ }+}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/mod.rsindex e0c4e7865..075767ac1 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/mod.rs@@ -1,3 +1,4 @@+pub mod conv_pack;pub mod conv_scan;pub mod conv_update;pub mod norm_gate;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/mod.rsindex e96c27e1c..26f1d4619 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs@@ -18,7 +18,6 @@ mod radix_top_k_small;mod sampling;mod short_conv;mod softmax;-mod ssm;mod tensor_add_bias;mod tensor_add_scale;mod weaver;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rsdeleted file mode 100644index f16d261c7..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs+++ /dev/null@@ -1,201 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::data_type::DataTypeRepr;--#[kernel(Conv1dPack)]-#[variants(StateT, f32, bf16)]-#[variants(InputT, f32, bf16)]-pub fn conv1d_pack<StateT: DataTypeRepr + Float, InputT: DataTypeRepr + Float>(- state_in: *const StateT,- x: *const InputT,- padded: *mut StateT,- state_stride: u32,- row_stride: u32,- suffix_len: u32,- num_channels: u32,-) {- let state_stride = state_stride as usize;- let row_stride = row_stride as usize;- let suffix_len = suffix_len as usize;- let num_channels = num_channels as usize;-- unsafe {- for channel_idx in 0..num_channels {- for row_idx in 0..state_stride + suffix_len {- let padded_index = row_idx * row_stride + channel_idx;- if row_idx < state_stride {- let state_index = channel_idx * state_stride + row_idx;- *padded.add(padded_index) = *state_in.add(state_index);- } else {- let token = row_idx - state_stride;- let x_index = token * row_stride + channel_idx;- *padded.add(padded_index) = StateT::from((*x.add(x_index)).to_f32().unwrap()).unwrap();- }- }- }- }-}--#[kernel(Conv1dDecode)]-#[variants(T, f32, f16, bf16)]-pub fn conv1d_decode<T: DataTypeRepr + Float>(- x: *const T,- w: *const T,- #[optional(has_bias)] b: Option<*const T>,- #[optional(!state_in_place)] state: Option<*const T>,- x_out: *mut T,- b_out: *mut T,- c_out: *mut T,- next_state: *mut T,- kernel_size: u32,- row_stride: u32,- state_stride: u32,- num_channels: u32,- suffix_len: u32,- inner_dim: u32,- proj_dim: u32,- activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,- #[specialize] has_bias: bool,- #[specialize] state_in_place: bool,-) {- let kernel_size = kernel_size as usize;- let row_stride = row_stride as usize;- let state_stride = state_stride as usize;- let num_channels = num_channels as usize;- let suffix_len = suffix_len as usize;- let inner_dim = inner_dim as usize;- let proj_dim = proj_dim as usize;-- let state = match state_in_place {- true => next_state,- false => state.unwrap(),- };-- unsafe {- for token_idx in 0..suffix_len {- for channel_idx in 0..num_channels {- let x_idx = token_idx * row_stride + channel_idx;- let state_offset = channel_idx * state_stride;- let w_row = w.add(channel_idx * kernel_size);-- let mut acc = 0.0f32;- if has_bias {- acc = (*b.unwrap().add(channel_idx)).to_f32().unwrap();- }-- let state_taps = kernel_size.saturating_sub(1);- for tap in 0..state_taps {- let state_index = state_offset + tap;- let sample = (*state.add(state_index)).to_f32().unwrap();- acc += (*w_row.add(tap)).to_f32().unwrap() * sample;- }-- let current = (*x.add(x_idx)).to_f32().unwrap();- let current_tap_index = state_taps;- if current_tap_index < kernel_size {- acc += (*w_row.add(current_tap_index)).to_f32().unwrap() * current;- }-- let activated = activation_type.activate(T::from(acc).unwrap());- if channel_idx < inner_dim {- let dst = token_idx * inner_dim + channel_idx;- *x_out.add(dst) = activated;- } else if channel_idx < inner_dim + proj_dim {- let dst = token_idx * proj_dim + (channel_idx - inner_dim);- *b_out.add(dst) = activated;- } else if channel_idx < inner_dim + 2 * proj_dim {- let dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);- *c_out.add(dst) = activated;- }- if state_taps == 0 {- continue;- }-- for tap in 0..state_taps - 1 {- let src_index = state_offset + tap + 1;- let dst_index = state_offset + tap;- *next_state.add(dst_index) = *state.add(src_index);- }-- let tail_index = state_offset + state_taps - 1;- *next_state.add(tail_index) = T::from(current).unwrap();- }- }- }-}--#[kernel(Conv1dScan)]-#[variants(T, f32, f16, bf16)]-pub fn conv1d_scan<T: DataTypeRepr + Float>(- padded: *const T,- w: *const T,- #[optional(has_bias)] b: Option<*const T>,- x_out: *mut T,- b_out: *mut T,- c_out: *mut T,- state_out: *mut T,- suffix_len: u32,- kernel_size: u32,- row_stride: u32,- state_stride: u32,- num_channels: u32,- inner_dim: u32,- proj_dim: u32,- activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,- #[specialize] has_bias: bool,-) {- let suffix_len = suffix_len as usize;- let kernel_size = kernel_size as usize;- let row_stride = row_stride as usize;- let state_stride = state_stride as usize;- let num_channels = num_channels as usize;- let inner_dim = inner_dim as usize;- let proj_dim = proj_dim as usize;-- unsafe {- for token_idx in 0..suffix_len + kernel_size.saturating_sub(1) {- for channel_idx in 0..num_channels {- let tap_count = kernel_size.saturating_sub(1);- let state_offset = channel_idx * state_stride;- let weight_offset = channel_idx * kernel_size;- let w_row = w.add(weight_offset);-- if token_idx < suffix_len {- let mut acc = 0.0f32;- if has_bias {- acc = (*b.unwrap().add(channel_idx)).to_f32().unwrap();- }-- for tap in 0..kernel_size {- let padded_index = (token_idx + tap) * row_stride + channel_idx;- let sample = (*padded.add(padded_index)).to_f32().unwrap();- acc += (*w_row.add(tap)).to_f32().unwrap() * sample;- }-- let activated = activation_type.activate(T::from(acc).unwrap());- if channel_idx < inner_dim {- let dst = token_idx * inner_dim + channel_idx;- *x_out.add(dst) = activated;- } else if channel_idx < inner_dim + proj_dim {- let dst = token_idx * proj_dim + (channel_idx - inner_dim);- *b_out.add(dst) = activated;- } else if channel_idx < inner_dim + 2 * proj_dim {- let dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);- *c_out.add(dst) = activated;- }- } else if tap_count > 0 {- let tap = token_idx - suffix_len;- if tap >= tap_count {- return;- }-- let padded_index = token_idx * row_stride + channel_idx;- let sample = (*padded.add(padded_index)).to_f32().unwrap();- *state_out.add(state_offset + tap) = T::from(sample).unwrap();- }- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/mod.rsdeleted file mode 100644index fb01e93a7..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/mod.rs+++ /dev/null@@ -1,4 +0,0 @@-pub mod conv1d;-pub mod split_inproj;-pub mod ssd_prefill;-pub mod ssd_update;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rsdeleted file mode 100644index e6268c9a4..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs+++ /dev/null@@ -1,45 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::data_type::DataTypeRepr;--#[kernel(SplitInProj)]-#[variants(T, f32, f16, bf16)]-pub fn split_in_proj<T: DataTypeRepr + Float>(- input: *const T,- conv_out: *mut T,- z_out: *mut T,- dt_out: *mut T,- z_bias: *const T,- suffix_length: u32,- total_dim: u32,- conv_dim: u32,- inner_dim: u32,- num_heads: u32,-) {- let suffix_length = suffix_length as usize;- let total_dim = total_dim as usize;- let conv_dim = conv_dim as usize;- let inner_dim = inner_dim as usize;- let num_heads = num_heads as usize;-- unsafe {- for row in 0..suffix_length {- for col in 0..total_dim {- let input_idx = row * total_dim + col;- if col < conv_dim {- let dst = row * conv_dim + col;- *conv_out.add(dst) = *input.add(input_idx);- } else if col < conv_dim + inner_dim {- let dst = row * inner_dim + (col - conv_dim);- let bias_idx = col - conv_dim;- *z_out.add(dst) = *input.add(input_idx) + *z_bias.add(bias_idx);- } else if col < conv_dim + inner_dim + num_heads {- let dst = row * num_heads + (col - conv_dim - inner_dim);- *dt_out.add(dst) = *input.add(input_idx);- }- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rsdeleted file mode 100644index 84ec1a5b2..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs+++ /dev/null@@ -1,53 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::data_type::DataTypeRepr;--#[kernel(SSDPrefill64)]-#[variants(T, f32, f16, bf16)]-pub fn ssd_prefill64<T: DataTypeRepr + Float>(- #[allow(unused)] x: *const T,- #[allow(unused)] dt_raw: *const T,- #[allow(unused)] b: *const T,- #[allow(unused)] c: *const T,- #[allow(unused)] d: *const T,- #[allow(unused)] z: *const T,- #[allow(unused)] state: *mut T,- #[allow(unused)] y: *mut T,- #[allow(unused)] suffix_len: u32,- #[allow(unused)] group_size: u32,- #[allow(unused)] state_size: u32,- #[allow(unused)] x_strides: &[u32; 3],- #[allow(unused)] dt_strides: &[u32; 2],- #[allow(unused)] cb_strides: &[u32; 3],- #[allow(unused)] state_strides: &[u32; 3],- #[allow(unused)] num_heads: u32,- #[allow(unused)] head_dim: u32,-) {- todo!()-}--#[kernel(SSDPrefill)]-#[variants(T, f32, f16, bf16)]-pub fn ssd_prefill<T: DataTypeRepr + Float>(- #[allow(unused)] x: *const T,- #[allow(unused)] dt_raw: *const T,- #[allow(unused)] b: *const T,- #[allow(unused)] c: *const T,- #[allow(unused)] d: *const T,- #[allow(unused)] z: *const T,- #[allow(unused)] state: *mut T,- #[allow(unused)] y: *mut T,- #[allow(unused)] suffix_len: u32,- #[allow(unused)] group_size: u32,- #[allow(unused)] state_size: u32,- #[allow(unused)] x_strides: &[u32; 3],- #[allow(unused)] dt_strides: &[u32; 2],- #[allow(unused)] cb_strides: &[u32; 3],- #[allow(unused)] state_strides: &[u32; 3],- #[allow(unused)] num_heads: u32,- #[allow(unused)] head_dim: u32,-) {- todo!()-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rsdeleted file mode 100644index c7669cb04..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs+++ /dev/null@@ -1,75 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};--#[kernel(SSDUpdate)]-#[variants(T, f32, f16, bf16)]-pub fn ssd_update<T: DataTypeRepr + Float>(- x: *const T,- dt_raw: *const T,- b: *const T,- c: *const T,- d: *const T,- z: *const T,- #[optional(!state_in_place)] state: Option<*const T>,- y: *mut T,- next_state: *mut T,- group_size: u32,- state_size: u32,- x_strides: &[u32; 3],- dt_strides: &[u32; 2],- cb_strides: &[u32; 3],- state_strides: &[u32; 4],- b_size: u32,- h_size: u32,- dh_size: u32,- #[specialize] state_in_place: bool,-) {- let group_size = group_size as usize;- let state_size = state_size as usize;- let b_size = b_size as usize;- let h_size = h_size as usize;- let dh_size = dh_size as usize;-- let state = match state_in_place {- true => next_state,- false => state.unwrap(),- };-- unsafe {- for b_idx in 0..b_size {- for h_idx in 0..h_size {- for dh_idx in 0..dh_size {- let cb_start_idx = b_idx * cb_strides[0] as usize + (h_idx / group_size) * cb_strides[1] as usize;- let x_idx =- b_idx * x_strides[0] as usize + h_idx * x_strides[1] as usize + dh_idx * x_strides[2] as usize;- let dt_idx = b_idx * dt_strides[0] as usize + h_idx * dt_strides[1] as usize;- let state_start_idx = b_idx * state_strides[0] as usize- + h_idx * state_strides[1] as usize- + dh_idx * state_strides[2] as usize;-- let this_x = *x.add(x_idx);- let dt_raw_val = *dt_raw.add(dt_idx);- let this_dt = ActivationType::SOFTPLUS.activate(dt_raw_val);- let this_decay = T::from((-this_dt.to_f32().unwrap()).exp()).unwrap();- let this_d = *d.add(h_idx);- let this_z = ActivationType::SILU.activate(*z.add(x_idx));-- let mut temp = T::zero();- for i in 0..state_size {- let cb_idx = cb_start_idx + i;- let state_idx = state_start_idx + i;- let this_new_state = *state.add(state_idx) * this_decay + *b.add(cb_idx) * this_x;- *next_state.add(state_idx) = this_new_state;- temp = temp + this_new_state * *c.add(cb_idx);- }- temp = temp + this_d * this_x;- temp = temp * this_z;- *y.add(x_idx) = temp;- }- }- }- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metalnew file mode 100644index 000000000..9e076f61f--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal@@ -0,0 +1,27 @@+#include <metal_stdlib>+#include "../common/dsl.h"++template <typename StateT, typename InputT>+VARIANTS(StateT, float, bfloat)+VARIANTS(InputT, float, bfloat)+PUBLIC KERNEL(Conv1dPack)(+ device const StateT* state_in,+ device const InputT* x,+ device StateT* padded,+ constant const uint& state_stride,+ constant const uint& row_stride,+ constant const uint& suffix_len,+ constant const uint& num_channels,+ const uint channel_idx AXIS(num_channels, 32),+ const uint row_idx AXIS(state_stride + suffix_len, 1)+) {+ const uint padded_index = row_idx * row_stride + channel_idx;+ if (row_idx < state_stride) {+ const uint state_index = channel_idx * state_stride + row_idx;+ padded[padded_index] = state_in[state_index];+ } else {+ const uint token = row_idx - state_stride;+ const uint x_index = token * row_stride + channel_idx;+ padded[padded_index] = static_cast<StateT>(x[x_index]);+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu-engine/src/backends/metal/kernel/ssm/conv1d.metaldeleted file mode 100644index 9d997c001..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/ssm/conv1d.metal+++ /dev/null@@ -1,167 +0,0 @@-#include <metal_stdlib>-#include "../activation/activations.h"-#include "../common/dsl.h"--using namespace uzu::activation_type;--template <typename StateT, typename InputT>-VARIANTS(StateT, float, bfloat)-VARIANTS(InputT, float, bfloat)-PUBLIC KERNEL(Conv1dPack)(- device const StateT* state_in,- device const InputT* x,- device StateT* padded,- constant const uint& state_stride,- constant const uint& row_stride,- constant const uint& suffix_len,- constant const uint& num_channels,- const uint channel_idx AXIS(num_channels, 32),- const uint row_idx AXIS(state_stride + suffix_len, 1)-) {- const uint padded_index = row_idx * row_stride + channel_idx;- if (row_idx < state_stride) {- const uint state_index = channel_idx * state_stride + row_idx;- padded[padded_index] = state_in[state_index];- } else {- const uint token = row_idx - state_stride;- const uint x_index = token * row_stride + channel_idx;- padded[padded_index] = static_cast<StateT>(x[x_index]);- }-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(Conv1dDecode)(- device const T* x,- device const T* w,- device const T* b OPTIONAL(has_bias),- device const T* state OPTIONAL(!state_in_place),- device T* x_out,- device T* b_out,- device T* c_out,- device T* next_state,- constant const uint& kernel_size,- constant const uint& row_stride,- constant const uint& state_stride,- constant const uint& num_channels,- constant const uint& suffix_len,- constant const uint& inner_dim,- constant const uint& proj_dim,- constant const ActivationType& activation_type,- const bool has_bias SPECIALIZE,- const bool state_in_place SPECIALIZE,- const uint token_idx AXIS(suffix_len, 32),- const uint channel_idx AXIS(num_channels, 1)-) {- if (state_in_place) {- state = next_state;- }-- const uint x_idx = token_idx * row_stride + channel_idx;- const uint state_offset = channel_idx * state_stride;- const device T* w_row = w + channel_idx * kernel_size;-- float acc = 0.0f;- if (has_bias) {- acc = float(b[channel_idx]);- }-- const uint state_taps = max(kernel_size - 1, 0u);- for (uint tap = 0; tap < state_taps; ++tap) {- const uint state_index = state_offset + tap;- const float sample = float(state[state_index]);- acc += float(w_row[tap]) * sample;- }-- const float current = float(x[x_idx]);- const uint current_tap_index = state_taps;- if (current_tap_index < kernel_size) {- acc += float(w_row[current_tap_index]) * current;- }-- const T activated = activate(static_cast<T>(acc), activation_type);- if (channel_idx < inner_dim) {- const uint dst = token_idx * inner_dim + channel_idx;- x_out[dst] = activated;- } else if (channel_idx < inner_dim + proj_dim) {- const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);- b_out[dst] = activated;- } else if (channel_idx < inner_dim + 2 * proj_dim) {- const uint dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);- c_out[dst] = activated;- }-- if (state_taps == 0) {- return;- }-- for (uint tap = 0; tap < state_taps - 1; ++tap) {- const size_t dst_index = state_offset + size_t(tap);- const size_t src_index = state_offset + size_t(tap + 1);- next_state[dst_index] = state[src_index];- }-- const uint tail_index = state_offset + state_taps - 1;- next_state[tail_index] = static_cast<T>(current);-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(Conv1dScan)(- device const T* padded, // (prefix+suffix, channels)- device const T* w, // (channels, kernel)- device const T* b OPTIONAL(has_bias), // optional (channels)- device T* x_out, // (suffix, inner_dim)- device T* b_out, // (suffix, proj_dim)- device T* c_out, // (suffix, proj_dim)- device T* state_out, // (channels, kernel-1)- constant const uint& suffix_len,- constant const uint& kernel_size,- constant const uint& row_stride,- constant const uint& state_stride,- constant const uint& num_channels,- constant const uint& inner_dim,- constant const uint& proj_dim,- constant const ActivationType& activation_type,- const bool has_bias SPECIALIZE,- const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 32),- const uint channel_idx AXIS(num_channels, 1)-) {- const uint tap_count = max(kernel_size - 1, 0u);- const uint state_offset = channel_idx * state_stride;- const uint weight_offset = channel_idx * kernel_size;- const device T* w_row = w + weight_offset;-- if (token_idx < suffix_len) {- float acc = 0.0f;- if (has_bias) {- acc = float(b[channel_idx]);- }-- for (uint tap = 0; tap < kernel_size; ++tap) {- const uint padded_index = (token_idx + tap) * row_stride + channel_idx;- float sample = float(padded[padded_index]);- acc += float(w_row[tap]) * sample;- }-- const T activated = activate(static_cast<T>(acc), activation_type);- if (channel_idx < inner_dim) {- const uint dst = token_idx * inner_dim + channel_idx;- x_out[dst] = activated;- } else if (channel_idx < inner_dim + proj_dim) {- const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);- b_out[dst] = activated;- } else if (channel_idx < inner_dim + 2 * proj_dim) {- const uint dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);- c_out[dst] = activated;- }- } else if (tap_count > 0) {- const uint tap = token_idx - suffix_len;- if (tap >= tap_count) {- return;- }- const uint padded_index = token_idx * row_stride + channel_idx;- const float sample = float(padded[padded_index]);- state_out[state_offset + tap] = static_cast<T>(sample);- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu-engine/src/backends/metal/kernel/ssm/split_inproj.metaldeleted file mode 100644index 18bd8805b..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/ssm/split_inproj.metal+++ /dev/null@@ -1,32 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(SplitInProj)(- device const T* input,- device T* conv_out,- device T* z_out,- device T* dt_out,- device const T* z_bias,- constant const uint& suffix_length,- constant const uint& total_dim,- constant const uint& conv_dim,- constant const uint& inner_dim,- constant const uint& num_heads,- const uint row AXIS(suffix_length, 16),- const uint col AXIS(total_dim, 16)-) {- const uint input_idx = row * total_dim + col;- if (col < conv_dim) {- const uint dst = row * conv_dim + col;- conv_out[dst] = input[input_idx];- } else if (col < conv_dim + inner_dim) {- const uint dst = row * inner_dim + (col - conv_dim);- const uint bias_idx = col - conv_dim;- z_out[dst] = input[input_idx] + z_bias[bias_idx];- } else if (col < conv_dim + inner_dim + num_heads) {- const uint dst = row * num_heads + (col - conv_dim - inner_dim);- dt_out[dst] = input[input_idx];- }-}\ No newline at end of filediff --git a/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_prefill.metaldeleted file mode 100644index 35c065f3f..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ /dev/null@@ -1,203 +0,0 @@-#include <metal_stdlib>-#include <metal_simdgroup>-#include "../activation/activations.h"-#include "../common/dsl.h"-#include "../common/thread_context.h"--using namespace metal;--constant ushort SSM_PREFILL_MAX_STATE = 256;--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(SSDPrefill64)(- device const T* x, // (suffix, h, dh)- device const T* dt_raw, // (suffix, h) - raw dt values- device const T* b, // (suffix, g, n)- device const T* c, // (suffix, g, n)- device const T* d, // (h)- device const T* z, // (suffix, h, dh)- device T* state, // (h, dh, n)- device T* y, // (suffix, h, dh)- constant const uint& suffix_len,- constant const uint& group_size,- constant const uint& state_size,- constant const uint x_strides[3],- constant const uint dt_strides[2],- constant const uint cb_strides[3],- constant const uint state_strides[3],- constant const uint& num_heads,- constant const uint& head_dim,- const ThreadContext thread_context,- const uint pair_idx GROUPS(num_heads * head_dim),- const uint lane_idx THREADS(32)-) {- const uint h_idx = pair_idx / head_dim;- const uint dh_idx = pair_idx % head_dim;- const uint safe_group = max(group_size, 1u);- const uint group_idx = h_idx / safe_group;-- const uint x_token_stride = x_strides[0];- const uint x_head_stride = x_strides[1];- const uint x_dim_stride = x_strides[2];- const uint dt_token_stride = dt_strides[0];- const uint dt_head_stride = dt_strides[1];- const uint cb_token_stride = cb_strides[0];- const uint cb_group_stride = cb_strides[1];- const uint cb_state_stride = cb_strides[2];- const uint state_head_stride = state_strides[0];- const uint state_dim_stride = state_strides[1];- const uint state_inner_stride = state_strides[2];-- const uint x_base = h_idx * x_head_stride + dh_idx * x_dim_stride;- const uint dt_base = h_idx * dt_head_stride;- const uint state_base = h_idx * state_head_stride + dh_idx * state_dim_stride;- const uint cb_group_base = group_idx * cb_group_stride;- const float d_scalar = float(d[h_idx]);-- const uint idx0 = lane_idx;- const uint idx1 = lane_idx + thread_context.simdgroup_size;-- const uint state_idx0 = state_base + idx0 * state_inner_stride;- const uint state_idx1 = state_base + idx1 * state_inner_stride;- float state0 = float(state[state_idx0]);- float state1 = float(state[state_idx1]);-- uint cb_idx0 = cb_group_base + idx0 * cb_state_stride;- uint cb_idx1 = cb_group_base + idx1 * cb_state_stride;-- for (uint token = 0; token < suffix_len; ++token) {- const uint x_idx = token * x_token_stride + x_base;- const uint dt_idx = token * dt_token_stride + dt_base;-- const float x_val = float(x[x_idx]);- const float decay_val = fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));- const float gate = float(activate_silu(z[x_idx]));- const float skip = d_scalar * x_val;- const float dt_scaled_input = x_val;-- float contrib = 0.0f;- const float new_state0 = decay_val * state0 + dt_scaled_input * float(b[cb_idx0]);- state0 = new_state0;- contrib += new_state0 * float(c[cb_idx0]);- cb_idx0 += cb_token_stride;-- const float new_state1 = decay_val * state1 + dt_scaled_input * float(b[cb_idx1]);- state1 = new_state1;- contrib += new_state1 * float(c[cb_idx1]);- cb_idx1 += cb_token_stride;-- float dot = simd_sum(contrib);- if (lane_idx == 0) {- y[x_idx] = static_cast<T>((skip + dot) * gate);- }- }-- state[state_idx0] = static_cast<T>(state0);- state[state_idx1] = static_cast<T>(state1);-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(SSDPrefill)(- device const T* x, // (suffix, h, dh)- device const T* dt_raw, // (suffix, h) - raw dt values- device const T* b, // (suffix, g, n)- device const T* c, // (suffix, g, n)- device const T* d, // (h)- device const T* z, // (suffix, h, dh)- device T* state, // (h, dh, n)- device T* y, // (suffix, h, dh)- constant const uint& suffix_len,- constant const uint& group_size,- constant const uint& state_size,- constant const uint x_strides[3],- constant const uint dt_strides[2],- constant const uint cb_strides[3],- constant const uint state_strides[3],- constant const uint& num_heads,- constant const uint& head_dim,- const ThreadContext thread_context,- const uint pair_idx GROUPS(num_heads * head_dim),- const uint lane_idx THREADS(32)-) {- const int state_dim = state_size;- const uint h_idx = pair_idx / head_dim;- const uint dh_idx = pair_idx % head_dim;- const uint safe_group = max(group_size, 1u);- const uint group_idx = h_idx / safe_group;-- const uint x_token_stride = x_strides[0];- const uint x_head_stride = x_strides[1];- const uint x_dim_stride = x_strides[2];- const uint dt_token_stride = dt_strides[0];- const uint dt_head_stride = dt_strides[1];- const uint cb_token_stride = cb_strides[0];- const uint cb_group_stride = cb_strides[1];- const uint cb_state_stride = cb_strides[2];- const uint state_head_stride = state_strides[0];- const uint state_dim_stride = state_strides[1];- const uint state_inner_stride = state_strides[2];-- const uint x_base = h_idx * x_head_stride + dh_idx * x_dim_stride;- const uint dt_base = h_idx * dt_head_stride;- const uint state_base = h_idx * state_head_stride + dh_idx * state_dim_stride;- const uint cb_group_base = group_idx * cb_group_stride;- const float d_scalar = float(d[h_idx]);-- const int max_chunks = SSM_PREFILL_MAX_STATE / thread_context.simdgroup_size;- const int chunk_count = (state_dim + int(thread_context.simdgroup_size) - 1) / int(thread_context.simdgroup_size);- if (state_dim <= 0 || chunk_count > max_chunks) {- return;- }-- thread float lane_states[SSM_PREFILL_MAX_STATE / 32];--#pragma unroll- for (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);- if (idx < state_dim) {- const uint state_idx = state_base + idx * state_inner_stride;- lane_states[chunk] = float(state[state_idx]);- } else {- lane_states[chunk] = 0.0f;- }- }-- for (uint token = 0; token < suffix_len; ++token) {- const uint x_idx = token * x_token_stride + x_base;- const uint dt_idx = token * dt_token_stride + dt_base;-- const float x_val = float(x[x_idx]);- const float decay_val = fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));- const float gate = float(activate_silu(z[x_idx]));- const float skip = d_scalar * x_val;- const float dt_scaled_input = x_val;-- float contrib_sum = 0.0f;-#pragma unroll- for (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);- if (idx < state_dim) {- const uint cb_idx = cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;- const float new_state = decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);- lane_states[chunk] = new_state;- contrib_sum += new_state * float(c[cb_idx]);- }- }-- float dot = simd_sum(contrib_sum);- if (lane_idx == 0) {- y[x_idx] = static_cast<T>((skip + dot) * gate);- }- }--#pragma unroll- for (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);- if (idx < state_dim) {- const uint state_idx = state_base + uint(idx) * state_inner_stride;- state[state_idx] = static_cast<T>(lane_states[chunk]);- }- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_update.metaldeleted file mode 100644index 3e6156b6a..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_update.metal+++ /dev/null@@ -1,69 +0,0 @@-#include <metal_stdlib>-#include "../activation/activations.h"-#include "../common/dsl.h"--using namespace metal;--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(SSDUpdate)(- // Input- device const T* x, // (b, h, dh)- device const T* dt_raw, // (b, h)- device const T* b, // (b, g, n)- device const T* c, // (b, g, n)- device const T* d, // (h)- device const T* z, // (b, d)- device const T* state OPTIONAL(!state_in_place), // (b, h, dh, n)- device T* y,- device T* next_state,- // Parameters- constant const uint& group_size, // h = group_size * g- constant const uint& state_size,- // Strides- constant const uint x_strides[3],- constant const uint dt_strides[2],- constant const uint cb_strides[3],- constant const uint state_strides[4],- // Threads count- constant const uint& b_size,- constant const uint& h_size,- constant const uint& dh_size,- const bool state_in_place SPECIALIZE,- // Grid- const uint b_idx AXIS(b_size, 32),- const uint h_idx AXIS(h_size, 32),- const uint dh_idx AXIS(dh_size, 1)-) {- if (state_in_place) {- state = next_state;- }-- const uint cb_start_idx = b_idx * cb_strides[0] + (h_idx / group_size) * cb_strides[1];- const uint x_idx = b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];- const uint dt_idx = b_idx * dt_strides[0] + h_idx * dt_strides[1];- const uint state_start_idx = b_idx * state_strides[0] + h_idx * state_strides[1] + dh_idx * state_strides[2];-- // load data- T this_x = x[x_idx];- T dt_raw_val = dt_raw[dt_idx];- T this_dt = activate_softplus(dt_raw_val);- T this_decay = static_cast<T>(fast::exp(-float(this_dt)));- T this_d = d[h_idx];- T this_z = z[x_idx];- this_z = activate_silu(this_z);- T dt_scaled_input = this_x;-- T temp = T(0);-#pragma unroll- for (uint i = 0; i < state_size; ++i) {- uint cb_idx = cb_start_idx + i;- uint state_idx = state_start_idx + i;- T this_new_state = state[state_idx] * this_decay + b[cb_idx] * dt_scaled_input;- next_state[state_idx] = this_new_state;- temp = temp + this_new_state * c[cb_idx];- }- temp = temp + this_d * this_x;- temp = temp * this_z;- y[x_idx] = temp;-}\ No newline at end of filediff --git a/crates/uzu-engine/src/config/token_mixer/mamba2.rs b/crates/uzu-engine/src/config/token_mixer/mamba2.rsdeleted file mode 100644index e20d3558a..000000000--- a/crates/uzu-engine/src/config/token_mixer/mamba2.rs+++ /dev/null@@ -1,21 +0,0 @@-use uzu_engine_macros::uzu_config;--use crate::config::{- activation::AnyActivation, linear::LinearConfig, token_mixer::convolutions::SeparableCausalConvConfig,-};--#[uzu_config(super::TokenMixerConfig)]-pub struct Mamba2Config {- pub in_projection_config: LinearConfig,- pub out_projection_config: LinearConfig,- pub conv_config: SeparableCausalConvConfig,- pub activation: AnyActivation,-- pub kernel_size: u32,- pub num_heads: u32,- pub num_groups: u32,- pub head_dim: u32,- pub state_dim: u32,- pub has_in_biases: bool,- pub has_out_biases: bool,-}diff --git a/crates/uzu-engine/src/config/token_mixer/mod.rs b/crates/uzu-engine/src/config/token_mixer/mod.rsindex 9faa3c20e..fff58ebd6 100644--- a/crates/uzu-engine/src/config/token_mixer/mod.rs+++ b/crates/uzu-engine/src/config/token_mixer/mod.rs@@ -3,13 +3,7 @@ use uzu_engine_macros::uzu_config_abstract;pub mod attention;pub mod convolutions;pub mod delta_net;-pub mod mamba2;pub mod short_conv;-#[uzu_config_abstract(- attention::AttentionConfig,- delta_net::DeltaNetConfig,- mamba2::Mamba2Config,- short_conv::ShortConvConfig-)]+#[uzu_config_abstract(attention::AttentionConfig, delta_net::DeltaNetConfig, short_conv::ShortConvConfig)]pub struct TokenMixerConfig;diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rsdeleted file mode 100644index 8605e17d0..000000000--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs+++ /dev/null@@ -1,436 +0,0 @@-use derive_more::Debug;-use thiserror::Error;--use crate::{- backends::common::{- Backend, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::activation_type::ActivationType,- kernel::{- Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDPrefill64Kernel, SSDPrefillKernel,- SSDUpdateKernel, SplitInProjKernel,- },- },- config::token_mixer::mamba2::Mamba2Config,- data_type::DataType,- encodable_block::{- batch_topology::BatchTopology,- linear::{Linear, LinearBlockError},- mixer::{Mixer, MixerEncodeError, MixerState, attention::rope::PrecalculatedRoPE},- },- safetensors::{ParameterLoaderError, ParameterTree},- utils::{maybe_mut::MaybeMut, size_for_shape::size_for_shape},-};--const INNER_DATA_TYPE: DataType = DataType::F32;--pub struct Mamba2State<B: Backend> {- conv_state: B::GlobalBuffer,- ssm_state: B::GlobalBuffer,- suffix_length: Option<u32>,-}--impl<B: Backend> MixerState<B> for Mamba2State<B> {- fn prepare(- &mut self,- _context_length: u32,- _suffix_length: u32,- ) -> Result<(), B::Error> {- Ok(())- }-- fn encode_accept(- &mut self,- accepted_indices: &[u32],- _command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), <B as Backend>::Error> {- assert!(self.suffix_length.take() == Some(*accepted_indices.last().unwrap() + 1));- Ok(())- }-}--enum Mamba2SSDPrefillVariant<B: Backend> {- Universal(<B::Kernels as Kernels>::SSDPrefillKernel),- Special64(<B::Kernels as Kernels>::SSDPrefill64Kernel),-}--pub struct Mamba2<B: Backend> {- kernel_size: u32,- num_heads: u32,- num_groups: u32,- head_dim: u32,- state_dim: u32,- inner_dim: u32,- conv_dim: u32,- activation_type: ActivationType,- in_projection: Box<dyn Linear<B>>,- gate_bias: B::GlobalBuffer,- split_inproj: <B::Kernels as Kernels>::SplitInProjKernel,- conv_weight: B::GlobalBuffer,- conv_bias: Option<B::GlobalBuffer>,- conv_decode: <B::Kernels as Kernels>::Conv1dDecodeKernel,- conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,- conv_scan: <B::Kernels as Kernels>::Conv1dScanKernel,- skip_connection_weight: B::GlobalBuffer,- ssd_update: <B::Kernels as Kernels>::SSDUpdateKernel,- ssd_prefill: Mamba2SSDPrefillVariant<B>,- out_projection: Box<dyn Linear<B>>,-}--#[derive(Debug, Error)]-pub enum Mamba2NewError<B: Backend> {- #[error("Backend error: {0}")]- Backend(#[source] B::Error),- #[error("Parameter loader error: {0}")]- ParameterLoader(#[from] ParameterLoaderError<B>),- #[error("Linear error: {0}")]- Linear(#[from] LinearBlockError<B>),- #[error("Unsupported configuration: {0}")]- UnsupportedConfiguration(String),-}--impl<B: Backend> Mamba2<B> {- pub fn new(- hidden_dim: u32,- outer_data_type: DataType,- config: &Mamba2Config,- parameter_tree: &ParameterTree<B>,- context: &B::Context,- ) -> Result<(Self, Option<B::GlobalBuffer>), Mamba2NewError<B>> {- let kernel_size = config.kernel_size;- let num_heads = config.num_heads;- let num_groups = config.num_groups;- let head_dim = config.head_dim;- let state_dim = config.state_dim;-- let inner_dim = num_heads * head_dim;- let conv_dim = inner_dim + 2 * num_groups * state_dim;-- let activation_type = config.activation.act_type();-- if kernel_size <= 1 {- return Err(Mamba2NewError::UnsupportedConfiguration(format!("kernel_size = {kernel_size} (must be > 1)")));- }-- let (in_projection, in_projection_input_hadamard_factors) =- <dyn Linear<B>>::new_with_input_rht_mixed_precision(- hidden_dim,- conv_dim + inner_dim + num_heads,- outer_data_type,- outer_data_type,- INNER_DATA_TYPE,- config.has_in_biases,- ¶meter_tree.subtree("in_projection"),- context,- )?;-- let gate_bias = parameter_tree.read_buffer("gate_bias", &[inner_dim], INNER_DATA_TYPE)?;- let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, INNER_DATA_TYPE)- .map_err(Mamba2NewError::Backend)?;-- let conv_config = &config.conv_config;- let conv_tree = parameter_tree.subtree("conv");-- let conv_weight = conv_tree.read_buffer("weights", &[conv_dim, kernel_size], INNER_DATA_TYPE)?;- let conv_bias = if conv_config.has_biases {- Some(conv_tree.read_buffer("biases", &[conv_dim], INNER_DATA_TYPE)?)- } else {- None- };- let conv_decode =- <B::Kernels as Kernels>::Conv1dDecodeKernel::new(context, INNER_DATA_TYPE, conv_config.has_biases, true)- .map_err(Mamba2NewError::Backend)?;- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, INNER_DATA_TYPE, INNER_DATA_TYPE)- .map_err(Mamba2NewError::Backend)?;- let conv_scan =- <B::Kernels as Kernels>::Conv1dScanKernel::new(context, INNER_DATA_TYPE, conv_config.has_biases)- .map_err(Mamba2NewError::Backend)?;-- let skip_connection_weight =- parameter_tree.read_buffer("skip_connection_weight", &[num_heads], INNER_DATA_TYPE)?;- let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, INNER_DATA_TYPE, true)- .map_err(Mamba2NewError::Backend)?;-- let ssd_prefill = if state_dim == 64 {- Mamba2SSDPrefillVariant::Special64(- <B::Kernels as Kernels>::SSDPrefill64Kernel::new(context, INNER_DATA_TYPE)- .map_err(Mamba2NewError::Backend)?,- )- } else {- Mamba2SSDPrefillVariant::Universal(- <B::Kernels as Kernels>::SSDPrefillKernel::new(context, INNER_DATA_TYPE)- .map_err(Mamba2NewError::Backend)?,- )- };-- let out_projection = <dyn Linear<B>>::new_mixed_precision(- inner_dim,- hidden_dim,- outer_data_type,- INNER_DATA_TYPE,- outer_data_type,- config.has_out_biases,- ¶meter_tree.subtree("out_projection"),- context,- )?;-- Ok((- Self {- kernel_size,- num_heads,- num_groups,- head_dim,- state_dim,- inner_dim,- conv_dim,- activation_type,- in_projection,- gate_bias,- split_inproj,- conv_weight,- conv_bias,- conv_decode,- conv_pack,- conv_scan,- skip_connection_weight,- ssd_update,- ssd_prefill,- out_projection,- },- in_projection_input_hadamard_factors,- ))- }-}--impl<B: Backend> Mixer<B> for Mamba2<B> {- fn speculation_supported(&self) -> bool {- false- }-- fn max_context_length(&self) -> Option<u32> {- None- }-- fn create_empty_state(- &self,- _max_context_length: Option<u32>,- context: &B::Context,- ) -> Result<Box<dyn MixerState<B>>, B::Error> {- let mut conv_state =- context.create_buffer(size_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE))?;-- let mut ssm_state =- context.create_buffer(size_for_shape(&[self.num_heads, self.head_dim, self.state_dim], INNER_DATA_TYPE))?;-- let mut zero_command_buffer = context.create_command_buffer(None, None)?;- zero_command_buffer.encode_fill(&mut conv_state, 0);- zero_command_buffer.encode_fill(&mut ssm_state, 0);- zero_command_buffer.end_encoding().submit().wait_until_completed()?;-- Ok(Box::new(Mamba2State {- conv_state,- ssm_state,- suffix_length: None,- }))- }-- fn encode(- &self,- hidden: B::ScratchBuffer,- precalculated_rope: Option<&PrecalculatedRoPE<B>>,- batch_dim: &BatchTopology,- state: Option<MaybeMut<dyn MixerState<B>>>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {- let command_buffer = &mut *command_buffer.debug_group("mamba2");-- assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");-- if !batch_dim.full_accept() {- panic!("mamba2 doesn't support speculation");- }-- let state = state.expect("mamba2 requires state");- let state = state.downcast::<Mamba2State<B>>().expect("incorrect type of mamba2 state");- let MaybeMut::Mut(state) = state else {- panic!("mamba2 doesn't support immutable state");- };-- assert!(state.suffix_length.is_none(), "mamba2 called with state with unaccepted tokens");-- let in_projected = self.in_projection.encode(hidden, batch_dim.size(), command_buffer)?;-- let mut conv_inputs = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.conv_dim], INNER_DATA_TYPE)- .map_err(MixerEncodeError::Backend)?;- let mut gate = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)- .map_err(MixerEncodeError::Backend)?;- let mut time_step = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)- .map_err(MixerEncodeError::Backend)?;- self.split_inproj.encode(- &in_projected,- &mut conv_inputs,- &mut gate,- &mut time_step,- &self.gate_bias,- batch_dim.size(),- self.conv_dim + self.inner_dim + self.num_heads,- self.conv_dim,- self.inner_dim,- self.num_heads,- command_buffer,- );-- let mut conv_x = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)- .map_err(MixerEncodeError::Backend)?;- let mut state_b = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)- .map_err(MixerEncodeError::Backend)?;- let mut state_c = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)- .map_err(MixerEncodeError::Backend)?;- let state_stride = self.kernel_size - 1;- if batch_dim.size() == 1 {- self.conv_decode.encode(- &conv_inputs,- &self.conv_weight,- self.conv_bias.as_ref(),- None::<&B::GlobalBuffer>,- &mut conv_x,- &mut state_b,- &mut state_c,- &mut state.conv_state,- self.kernel_size,- self.conv_dim,- state_stride,- self.conv_dim,- batch_dim.size(),- self.inner_dim,- self.num_groups * self.state_dim,- self.activation_type,- command_buffer,- );- } else {- let mut padded = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size() + state_stride, self.conv_dim], INNER_DATA_TYPE)- .map_err(MixerEncodeError::Backend)?;- self.conv_pack.encode(- &state.conv_state,- &conv_inputs,- &mut padded,- state_stride,- self.conv_dim,- batch_dim.size(),- self.conv_dim,- command_buffer,- );- self.conv_scan.encode(- &padded,- &self.conv_weight,- self.conv_bias.as_ref(),- &mut conv_x,- &mut state_b,- &mut state_c,- &mut state.conv_state,- batch_dim.size(),- self.kernel_size,- self.conv_dim,- state_stride,- self.conv_dim,- self.inner_dim,- self.num_groups * self.state_dim,- self.activation_type,- command_buffer,- );- }-- let mut ssd_output = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.inner_dim], INNER_DATA_TYPE)- .map_err(MixerEncodeError::Backend)?;- let x_strides = [self.num_heads * self.head_dim, self.head_dim, 1];- let dt_strides = [self.num_heads, 1];- let cb_strides = [self.num_groups * self.state_dim, self.state_dim, 1];- let group_size = self.num_heads / self.num_groups;- if batch_dim.size() == 1 {- let state_strides =- [self.num_heads * self.head_dim * self.state_dim, self.head_dim * self.state_dim, self.state_dim, 1];- self.ssd_update.encode(- &conv_x,- &time_step,- &state_b,- &state_c,- &self.skip_connection_weight,- &gate,- None::<&B::GlobalBuffer>,- &mut ssd_output,- &mut state.ssm_state,- group_size,- self.state_dim,- &x_strides,- &dt_strides,- &cb_strides,- &state_strides,- batch_dim.size(),- self.num_heads,- self.head_dim,- command_buffer,- );- } else {- let state_strides = [self.head_dim * self.state_dim, self.state_dim, 1];- match &self.ssd_prefill {- Mamba2SSDPrefillVariant::Universal(ssd_prefill) => ssd_prefill.encode(- &conv_x,- &time_step,- &state_b,- &state_c,- &self.skip_connection_weight,- &gate,- &mut state.ssm_state,- &mut ssd_output,- batch_dim.size(),- group_size,- self.state_dim,- &x_strides,- &dt_strides,- &cb_strides,- &state_strides,- self.num_heads,- self.head_dim,- command_buffer,- ),- Mamba2SSDPrefillVariant::Special64(ssd_prefill) => ssd_prefill.encode(- &conv_x,- &time_step,- &state_b,- &state_c,- &self.skip_connection_weight,- &gate,- &mut state.ssm_state,- &mut ssd_output,- batch_dim.size(),- group_size,- self.state_dim,- &x_strides,- &dt_strides,- &cb_strides,- &state_strides,- self.num_heads,- self.head_dim,- command_buffer,- ),- }- }-- state.suffix_length = Some(batch_dim.size());-- let output = self.out_projection.encode(ssd_output, batch_dim.size(), command_buffer)?;-- Ok(output)- }-}--#[cfg(test)]-#[path = "../../../unit/encodable_block/mamba_mixer/ssd_prefill_test.rs"]-mod tests;diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rsindex dc1061e44..80b81775e 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs@@ -13,7 +13,6 @@ use crate::{mixer::{attention::{Attention, AttentionEncodeError, AttentionNewError, rope::PrecalculatedRoPE},delta_net::{DeltaNet, DeltaNetNewError},- mamba2::{Mamba2, Mamba2NewError},short_conv::{ShortConv, ShortConvNewError},},},@@ -23,7 +22,6 @@ use crate::{pub mod attention;pub mod delta_net;-pub mod mamba2;pub mod short_conv;pub trait MixerState<B: Backend>: Any + Send {@@ -86,8 +84,6 @@ pub enum MixerNewError<B: Backend> {Attention(#[from] AttentionNewError<B>),#[error("Delta net mixer error: {0}")]DeltaNet(#[from] DeltaNetNewError<B>),- #[error("Mamba2 mixer error: {0}")]- Mamba2(#[from] Mamba2NewError<B>),#[error("Short conv mixer error: {0}")]ShortConv(#[from] ShortConvNewError<B>),}@@ -116,14 +112,6 @@ impl<B: Backend> dyn Mixer<B> {Ok((Box::new(delta_net), in_projection_input_hadamard_factors))},- AnyTokenMixerConfig::Mamba2Config(config) => {- assert!(rope_config.is_none(), "unexpected rope for mamba2 mixer");-- let (mamba2, in_projection_input_hadamard_factors) =- Mamba2::new(hidden_dim, data_type, config, parameter_tree, context)?;-- Ok((Box::new(mamba2), in_projection_input_hadamard_factors))- },AnyTokenMixerConfig::ShortConvConfig(config) => {assert!(rope_config.is_none(), "unexpected rope for short conv mixer");diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rsindex 91cc482b1..e1865a8b0 100644--- a/crates/uzu-engine/src/engine/language_model/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/mod.rs@@ -137,7 +137,7 @@ impl<B: Backend> LanguageModel<B> {// Currenlty true for all mixers in uzu:// - full attention uses sparse if it's available to make max context free until it's actually used// - sliding window attention is bound, usually well below the recommended max context size on non-sparse (but can be made to use sparse if we care about it enough)- // - short conv/mamba2/delta net are constant state size+ // - short conv/delta net are constant state sizemax_context_length} else if let Some(max_context_length) = max_context_length {// If sparse buffers aren't supported and model has finite maximum context length we assume that kv cache is expensive enough that we should probably clamp it todiff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rssimilarity index 100%rename from crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rsrename to crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rsdiff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/mod.rsindex adcd93b69..25e93f66c 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/mod.rs@@ -1,2 +1,3 @@+mod conv_pack_test;mod delta_net_test;mod tree_verify;diff --git a/crates/uzu-engine/unit/backends/common/kernel/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/mod.rsindex 72471df67..a6841562c 100644--- a/crates/uzu-engine/unit/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/mod.rs@@ -11,7 +11,6 @@ mod pooling;mod radix_top_k_small_test;mod separable_causal_conv_test;mod short_conv;-mod ssm;mod tensor_add_bias_test;mod tensor_add_scale_test;mod trellis_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/mod.rsdeleted file mode 100644index f704533c4..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/mod.rs+++ /dev/null@@ -1,6 +0,0 @@-mod ssm_conv1d_decode_test;-mod ssm_conv1d_pack_test;-mod ssm_conv1d_scan_test;-mod ssm_prefill_test;-mod ssm_split_inproj_test;-mod ssm_update_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rsdeleted file mode 100644index 5406e67ab..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs+++ /dev/null@@ -1,338 +0,0 @@-use std::fmt::{Debug, Display};--use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::activation_type::ActivationType, kernel::Conv1dDecodeKernel,- },- cpu::Cpu,- },- data_type::{DataType, DataTypeRepr},- tests::{- assert::assert_eq_float,- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},- },-};--struct Input<T: DataTypeRepr + Float> {- x: Box<[T]>,- w: Box<[T]>,- b: Option<Box<[T]>>,- state: Box<[T]>,- kernel_size: u32,- row_stride: u32,- state_stride: u32,- num_channels: u32,- suffix_len: u32,- inner_dim: u32,- proj_dim: u32,- activation_type: ActivationType,- state_in_place: bool,-}--struct Output<T: DataTypeRepr + Float> {- x_out: Vec<T>,- b_out: Vec<T>,- c_out: Vec<T>,- next_state: Vec<T>,-}--fn get_input<T: DataTypeRepr + Float>(- kernel_size: u32,- num_channels: u32,- inner_dim: u32,- proj_dim: u32,- suffix_len: u32,- has_bias: bool,- state_in_place: bool,- activation_type: ActivationType,-) -> Input<T> {- let row_stride = num_channels;- let state_stride = kernel_size.saturating_sub(1);- let state_size = num_channels as usize * state_stride as usize;- let x_size = suffix_len as usize * row_stride as usize;- let w_size = num_channels as usize * kernel_size as usize;-- let x: Vec<T> = (0..x_size).map(|i| T::from(0.1 * ((i as f64 % 7.0) - 3.0)).unwrap()).collect();- let w: Vec<T> = (0..w_size).map(|i| T::from(0.05 * ((i as f64 % 5.0) - 2.0)).unwrap()).collect();- let b = if has_bias {- Some(- (0..num_channels as usize)- .map(|i| T::from(0.01 * (i as f64 + 1.0)).unwrap())- .collect::<Vec<T>>()- .into_boxed_slice(),- )- } else {- None- };- let state: Vec<T> = (0..state_size).map(|i| T::from(0.02 * ((i as f64 % 9.0) - 4.0)).unwrap()).collect();-- Input {- x: x.into_boxed_slice(),- w: w.into_boxed_slice(),- b,- state: state.into_boxed_slice(),- kernel_size,- row_stride,- state_stride,- num_channels,- suffix_len,- inner_dim,- proj_dim,- activation_type,- state_in_place,- }-}--fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {- let context = B::Context::new().expect("Failed to create Context");- let has_bias = input.b.is_some();- let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dDecodeKernel::new(- &context,- T::data_type(),- has_bias,- input.state_in_place,- )- .expect("Failed to create Conv1dDecodeKernel");-- let x_out_size = input.suffix_len as usize * input.inner_dim as usize;- let b_out_size = input.suffix_len as usize * input.proj_dim as usize;- let c_out_size = input.suffix_len as usize * input.proj_dim as usize;- let state_size = input.num_channels as usize * input.state_stride as usize;-- let x = create_buffer_with_data::<B, T>(&context, &input.x);- let w = create_buffer_with_data::<B, T>(&context, &input.w);- let b = input.b.as_ref().map(|b| create_buffer_with_data::<B, T>(&context, b));-- let mut x_out = create_buffer::<B, T>(&context, x_out_size);- let mut b_out = create_buffer::<B, T>(&context, b_out_size);- let mut c_out = create_buffer::<B, T>(&context, c_out_size);-- if input.state_in_place {- let mut next_state = create_buffer_with_data::<B, T>(&context, &input.state);-- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- kernel.encode(- &x,- &w,- b.as_ref(),- None::<&B::GlobalBuffer>,- &mut x_out,- &mut b_out,- &mut c_out,- &mut next_state,- input.kernel_size,- input.row_stride,- input.state_stride,- input.num_channels,- input.suffix_len,- input.inner_dim,- input.proj_dim,- input.activation_type,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- Output {- x_out: buffer_to_vec(&x_out),- b_out: buffer_to_vec(&b_out),- c_out: buffer_to_vec(&c_out),- next_state: buffer_to_vec(&next_state),- }- } else {- let state = create_buffer_with_data::<B, T>(&context, &input.state);- let mut next_state = create_buffer::<B, T>(&context, state_size);-- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- kernel.encode(- &x,- &w,- b.as_ref(),- Some(&state),- &mut x_out,- &mut b_out,- &mut c_out,- &mut next_state,- input.kernel_size,- input.row_stride,- input.state_stride,- input.num_channels,- input.suffix_len,- input.inner_dim,- input.proj_dim,- input.activation_type,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- Output {- x_out: buffer_to_vec(&x_out),- b_out: buffer_to_vec(&b_out),- c_out: buffer_to_vec(&c_out),- next_state: buffer_to_vec(&next_state),- }- }-}--fn get_test_data<T: DataTypeRepr + Float>(- kernel_size: u32,- num_channels: u32,- inner_dim: u32,- proj_dim: u32,- suffix_len: u32,- has_bias: bool,- state_in_place: bool,- activation_type: ActivationType,-) -> (Input<T>, Output<T>) {- let input = get_input::<T>(- kernel_size,- num_channels,- inner_dim,- proj_dim,- suffix_len,- has_bias,- state_in_place,- activation_type,- );- let expected = get_output::<Cpu, T>(&input);- (input, expected)-}--fn test_internal<T: DataTypeRepr + Float + Debug + Display>(- input: &Input<T>,- expected: &Output<T>,- label: &str,-) {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 1e-2f32- } else {- 1e-5- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);- let backend_name = std::any::type_name::<B>();- let type_name = std::any::type_name::<T>();-- assert_eq_float::<T>(- &expected.x_out,- &output.x_out,- eps,- &format!("Conv1dDecode x_out {backend_name} {label} (type={type_name})"),- );- assert_eq_float::<T>(- &expected.b_out,- &output.b_out,- eps,- &format!("Conv1dDecode b_out {backend_name} {label} (type={type_name})"),- );- assert_eq_float::<T>(- &expected.c_out,- &output.c_out,- eps,- &format!("Conv1dDecode c_out {backend_name} {label} (type={type_name})"),- );- assert_eq_float::<T>(- &expected.next_state,- &output.next_state,- eps,- &format!("Conv1dDecode next_state {backend_name} {label} (type={type_name})"),- );- });-}--// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4-fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {- for state_in_place in [true, false] {- for has_bias in [true, false] {- let label = format!("basic(bias={has_bias},in_place={state_in_place})");- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, has_bias, state_in_place, ActivationType::SILU);- test_internal(&input, &expected, &label);- }- }-}--// kernel_size=2 (state_stride=1, minimal state)-fn test_small_kernel<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 1, true, true, ActivationType::SILU);- test_internal(&input, &expected, "small_kernel");-}--// Larger dimensions closer to real usage-fn test_large<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 1, true, true, ActivationType::SILU);- test_internal(&input, &expected, "large");-}--fn test_identity_activation<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, false, true, ActivationType::IDENTITY);- test_internal(&input, &expected, "identity_activation");-}--fn test_gelu_activation<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::GELUApprox);- test_internal(&input, &expected, "gelu_activation");-}--// f32-#[uzu_test]-fn test_basic_f32() {- test_basic::<f32>();-}--#[uzu_test]-fn test_small_kernel_f32() {- test_small_kernel::<f32>();-}--#[uzu_test]-fn test_large_f32() {- test_large::<f32>();-}--#[uzu_test]-fn test_identity_activation_f32() {- test_identity_activation::<f32>();-}--#[uzu_test]-fn test_gelu_activation_f32() {- test_gelu_activation::<f32>();-}--// f16-#[uzu_test]-fn test_basic_f16() {- test_basic::<f16>();-}--#[uzu_test]-fn test_small_kernel_f16() {- test_small_kernel::<f16>();-}--#[uzu_test]-fn test_large_f16() {- test_large::<f16>();-}--// bf16-#[uzu_test]-fn test_basic_bf16() {- test_basic::<bf16>();-}--#[uzu_test]-fn test_small_kernel_bf16() {- test_small_kernel::<bf16>();-}--#[uzu_test]-fn test_large_bf16() {- test_large::<bf16>();-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rsdeleted file mode 100644index 22f221edc..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs+++ /dev/null@@ -1,321 +0,0 @@-use std::fmt::{Debug, Display};--use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::activation_type::ActivationType, kernel::Conv1dScanKernel,- },- cpu::Cpu,- },- data_type::{DataType, DataTypeRepr},- tests::{- assert::assert_eq_float,- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},- },-};--struct Input<T: DataTypeRepr + Float> {- padded: Box<[T]>,- w: Box<[T]>,- b: Option<Box<[T]>>,- suffix_len: u32,- kernel_size: u32,- row_stride: u32,- state_stride: u32,- num_channels: u32,- inner_dim: u32,- proj_dim: u32,- activation_type: ActivationType,-}--struct Output<T: DataTypeRepr + Float> {- x_out: Vec<T>,- b_out: Vec<T>,- c_out: Vec<T>,- state_out: Vec<T>,-}--fn get_input<T: DataTypeRepr + Float>(- kernel_size: u32,- num_channels: u32,- inner_dim: u32,- proj_dim: u32,- suffix_len: u32,- has_bias: bool,- activation_type: ActivationType,-) -> Input<T> {- let row_stride = num_channels;- let state_stride = kernel_size.saturating_sub(1);- let total_rows = (suffix_len + state_stride) as usize;- let padded_size = total_rows * row_stride as usize;- let w_size = num_channels as usize * kernel_size as usize;-- let padded: Vec<T> = (0..padded_size).map(|i| T::from(0.1 * ((i as f64 % 11.0) - 5.0)).unwrap()).collect();- let w: Vec<T> = (0..w_size).map(|i| T::from(0.05 * ((i as f64 % 5.0) - 2.0)).unwrap()).collect();- let b = if has_bias {- Some(- (0..num_channels as usize)- .map(|i| T::from(0.01 * (i as f64 + 1.0)).unwrap())- .collect::<Vec<T>>()- .into_boxed_slice(),- )- } else {- None- };-- Input {- padded: padded.into_boxed_slice(),- w: w.into_boxed_slice(),- b,- suffix_len,- kernel_size,- row_stride,- state_stride,- num_channels,- inner_dim,- proj_dim,- activation_type,- }-}--fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {- let context = B::Context::new().expect("Failed to create Context");- let has_bias = input.b.is_some();- let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&context, T::data_type(), has_bias)- .expect("Failed to create Conv1dScanKernel");-- let x_out_size = input.suffix_len as usize * input.inner_dim as usize;- let b_out_size = input.suffix_len as usize * input.proj_dim as usize;- let c_out_size = input.suffix_len as usize * input.proj_dim as usize;- let state_size = input.num_channels as usize * input.state_stride as usize;-- let padded = create_buffer_with_data::<B, T>(&context, &input.padded);- let w = create_buffer_with_data::<B, T>(&context, &input.w);- let b = input.b.as_ref().map(|b| create_buffer_with_data::<B, T>(&context, b));-- let mut x_out = create_buffer::<B, T>(&context, x_out_size);- let mut b_out = create_buffer::<B, T>(&context, b_out_size);- let mut c_out = create_buffer::<B, T>(&context, c_out_size);- let mut state_out = create_buffer::<B, T>(&context, state_size);-- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- kernel.encode(- &padded,- &w,- b.as_ref(),- &mut x_out,- &mut b_out,- &mut c_out,- &mut state_out,- input.suffix_len,- input.kernel_size,- input.row_stride,- input.state_stride,- input.num_channels,- input.inner_dim,- input.proj_dim,- input.activation_type,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- Output {- x_out: buffer_to_vec(&x_out),- b_out: buffer_to_vec(&b_out),- c_out: buffer_to_vec(&c_out),- state_out: buffer_to_vec(&state_out),- }-}--fn get_test_data<T: DataTypeRepr + Float>(- kernel_size: u32,- num_channels: u32,- inner_dim: u32,- proj_dim: u32,- suffix_len: u32,- has_bias: bool,- activation_type: ActivationType,-) -> (Input<T>, Output<T>) {- let input = get_input::<T>(kernel_size, num_channels, inner_dim, proj_dim, suffix_len, has_bias, activation_type);- let expected = get_output::<Cpu, T>(&input);- (input, expected)-}--fn test_internal<T: DataTypeRepr + Float + Debug + Display>(- input: &Input<T>,- expected: &Output<T>,- label: &str,-) {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 1e-2f32- } else {- 1e-5- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);- let backend_name = std::any::type_name::<B>();- let type_name = std::any::type_name::<T>();-- assert_eq_float::<T>(- &expected.x_out,- &output.x_out,- eps,- &format!("Conv1dScan x_out {backend_name} {label} (type={type_name})"),- );- assert_eq_float::<T>(- &expected.b_out,- &output.b_out,- eps,- &format!("Conv1dScan b_out {backend_name} {label} (type={type_name})"),- );- assert_eq_float::<T>(- &expected.c_out,- &output.c_out,- eps,- &format!("Conv1dScan c_out {backend_name} {label} (type={type_name})"),- );- assert_eq_float::<T>(- &expected.state_out,- &output.state_out,- eps,- &format!("Conv1dScan state_out {backend_name} {label} (type={type_name})"),- );- });-}--// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4, suffix_len=2-fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {- for has_bias in [true, false] {- let label = format!("basic(bias={has_bias})");- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, has_bias, ActivationType::SILU);- test_internal(&input, &expected, &label);- }-}--// kernel_size=2 (state_stride=1, minimal state)-fn test_small_kernel<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 3, true, ActivationType::SILU);- test_internal(&input, &expected, "small_kernel");-}--// Single token suffix-fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, ActivationType::SILU);- test_internal(&input, &expected, "single_token");-}--// Many tokens-fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 16, true, ActivationType::SILU);- test_internal(&input, &expected, "many_tokens");-}--// Larger dimensions closer to real usage-fn test_large<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 8, true, ActivationType::SILU);- test_internal(&input, &expected, "large");-}--fn test_identity_activation<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, false, ActivationType::IDENTITY);- test_internal(&input, &expected, "identity_activation");-}--fn test_gelu_activation<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::GELUApprox);- test_internal(&input, &expected, "gelu_activation");-}--// f32-#[uzu_test]-fn test_basic_f32() {- test_basic::<f32>();-}--#[uzu_test]-fn test_small_kernel_f32() {- test_small_kernel::<f32>();-}--#[uzu_test]-fn test_single_token_f32() {- test_single_token::<f32>();-}--#[uzu_test]-fn test_many_tokens_f32() {- test_many_tokens::<f32>();-}--#[uzu_test]-fn test_large_f32() {- test_large::<f32>();-}--#[uzu_test]-fn test_identity_activation_f32() {- test_identity_activation::<f32>();-}--#[uzu_test]-fn test_gelu_activation_f32() {- test_gelu_activation::<f32>();-}--// f16-#[uzu_test]-fn test_basic_f16() {- test_basic::<f16>();-}--#[uzu_test]-fn test_small_kernel_f16() {- test_small_kernel::<f16>();-}--#[uzu_test]-fn test_single_token_f16() {- test_single_token::<f16>();-}--#[uzu_test]-fn test_many_tokens_f16() {- test_many_tokens::<f16>();-}--#[uzu_test]-fn test_large_f16() {- test_large::<f16>();-}--// bf16-#[uzu_test]-fn test_basic_bf16() {- test_basic::<bf16>();-}--#[uzu_test]-fn test_small_kernel_bf16() {- test_small_kernel::<bf16>();-}--#[uzu_test]-fn test_single_token_bf16() {- test_single_token::<bf16>();-}--#[uzu_test]-fn test_many_tokens_bf16() {- test_many_tokens::<bf16>();-}--#[uzu_test]-fn test_large_bf16() {- test_large::<bf16>();-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rsdeleted file mode 100644index 205b5df42..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs+++ /dev/null@@ -1,261 +0,0 @@-use std::fmt::{Debug, Display};--use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::uzu_test;--use crate::{- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::activation_type::ActivationType, kernel::SSDPrefillKernel,- },- data_type::{DataType, DataTypeRepr},- tests::{- assert::assert_eq_float,- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},- },-};--struct Input<T: DataTypeRepr + Float> {- x: Box<[T]>,- dt: Box<[T]>,- b: Box<[T]>,- c: Box<[T]>,- d: Box<[T]>,- z: Box<[T]>,- state: Box<[T]>,- suffix_len: usize,- num_heads: usize,- head_dim: usize,- state_dim: usize,- group_size: u32,- x_strides: [usize; 3],- dt_strides: [usize; 2],- cb_strides: [usize; 3],- state_strides: [usize; 3],-}--struct Output<T: DataTypeRepr + Float> {- y: Vec<T>,- state: Vec<T>,-}--fn get_input<T: DataTypeRepr + Float>(- suffix_len: usize,- num_heads: usize,- head_dim: usize,- state_dim: usize,- group_size: u32,-) -> Input<T> {- let safe_group = group_size.max(1) as usize;- let group_count = num_heads / safe_group;- let total_x = suffix_len * num_heads * head_dim;- let total_dt = suffix_len * num_heads;- let total_cb = suffix_len * group_count * state_dim;- let total_state = num_heads * head_dim * state_dim;-- let x: Vec<T> = (0..total_x).map(|i| T::from(((i % 17) as f64) * 0.01 - 0.05).unwrap()).collect();- let dt: Vec<T> = (0..total_dt).map(|i| T::from(((i % 13) as f64) * 0.2 - 1.5).unwrap()).collect();- let b: Vec<T> = (0..total_cb).map(|i| T::from(((i % 11) as f64) * 0.02 - 0.05).unwrap()).collect();- let c: Vec<T> = (0..total_cb).map(|i| T::from(((i % 19) as f64) * 0.01 - 0.02).unwrap()).collect();- let d: Vec<T> = (0..num_heads).map(|i| T::from(((i % 3) as f64) * 0.05 - 0.05).unwrap()).collect();- let z: Vec<T> = (0..total_x).map(|i| T::from(((i % 23) as f64) * 0.02 - 0.1).unwrap()).collect();- let state: Vec<T> = (0..total_state).map(|i| T::from(((i % 29) as f64) * 0.03 - 0.4).unwrap()).collect();-- let x_strides = [num_heads * head_dim, head_dim, 1];- let dt_strides = [num_heads, 1];- let cb_strides = [group_count * state_dim, state_dim, 1];- let state_strides = [head_dim * state_dim, state_dim, 1];-- Input {- x: x.into_boxed_slice(),- dt: dt.into_boxed_slice(),- b: b.into_boxed_slice(),- c: c.into_boxed_slice(),- d: d.into_boxed_slice(),- z: z.into_boxed_slice(),- state: state.into_boxed_slice(),- suffix_len,- num_heads,- head_dim,- state_dim,- group_size,- x_strides,- dt_strides,- cb_strides,- state_strides,- }-}--fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {- let context = B::Context::new().expect("Failed to create Context");-- let total_x = input.suffix_len * input.num_heads * input.head_dim;-- let x = create_buffer_with_data::<B, T>(&context, &input.x);- let dt = create_buffer_with_data::<B, T>(&context, &input.dt);- let b = create_buffer_with_data::<B, T>(&context, &input.b);- let c = create_buffer_with_data::<B, T>(&context, &input.c);- let d = create_buffer_with_data::<B, T>(&context, &input.d);- let z = create_buffer_with_data::<B, T>(&context, &input.z);- let mut state = create_buffer_with_data::<B, T>(&context, &input.state);- let mut y = create_buffer::<B, T>(&context, total_x);-- let x_strides: [u32; 3] = input.x_strides.map(|s| s as u32);- let dt_strides: [u32; 2] = input.dt_strides.map(|s| s as u32);- let cb_strides: [u32; 3] = input.cb_strides.map(|s| s as u32);- let state_strides: [u32; 3] = input.state_strides.map(|s| s as u32);-- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- let kernel = <<B as Backend>::Kernels as Kernels>::SSDPrefillKernel::new(&context, T::data_type())- .expect("Failed to create SSDPrefillKernel");- kernel.encode(- &x,- &dt,- &b,- &c,- &d,- &z,- &mut state,- &mut y,- input.suffix_len as u32,- input.group_size,- input.state_dim as u32,- &x_strides,- &dt_strides,- &cb_strides,- &state_strides,- input.num_heads as u32,- input.head_dim as u32,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- Output {- y: buffer_to_vec(&y),- state: buffer_to_vec(&state),- }-}--fn reference_output<T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {- let safe_group = input.group_size.max(1) as usize;- let mut state = input.state.to_vec();- let mut y = vec![T::zero(); input.suffix_len * input.num_heads * input.head_dim];-- for h in 0..input.num_heads {- let group_idx = h / safe_group;- for dh in 0..input.head_dim {- let state_base = h * input.state_strides[0] + dh * input.state_strides[1];- for token in 0..input.suffix_len {- let x_idx = token * input.x_strides[0] + h * input.x_strides[1] + dh * input.x_strides[2];- let dt_idx = token * input.dt_strides[0] + h * input.dt_strides[1];- let cb_base = token * input.cb_strides[0] + group_idx * input.cb_strides[1];-- let x_val = input.x[x_idx];- let dt_val = ActivationType::SOFTPLUS.activate(input.dt[dt_idx]);- let decay_val = (-dt_val).exp();-- let mut acc = input.d[h] * x_val;- for s in 0..input.state_dim {- let state_idx = state_base + s * input.state_strides[2];- let cb_idx = cb_base + s * input.cb_strides[2];- let new_state = decay_val * state[state_idx] + x_val * input.b[cb_idx];- state[state_idx] = new_state;- acc = acc + new_state * input.c[cb_idx];- }-- let gate = ActivationType::SILU.activate(input.z[x_idx]);- y[x_idx] = acc * gate;- }- }- }-- Output {- y,- state,- }-}--fn test_internal<T: DataTypeRepr + Float + Debug + Display>(- input: &Input<T>,- expected: &Output<T>,- label: &str,-) {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 2e-2f32- } else {- 5e-5- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);- let backend_name = std::any::type_name::<B>();- let type_name = std::any::type_name::<T>();-- assert_eq_float::<T>(- &expected.y,- &output.y,- eps,- &format!("SSDPrefill y {backend_name} {label} (type={type_name})"),- );- assert_eq_float::<T>(- &expected.state,- &output.state,- eps,- &format!("SSDPrefill state {backend_name} {label} (type={type_name})"),- );- });-}--// --- test shapes -----fn test_shape(- suffix_len: usize,- num_heads: usize,- head_dim: usize,- state_dim: usize,- group_size: u32,- label: &str,-) {- fn run<T: DataTypeRepr + Float + Debug + Display>(- suffix_len: usize,- num_heads: usize,- head_dim: usize,- state_dim: usize,- group_size: u32,- label: &str,- ) {- let input = get_input::<T>(suffix_len, num_heads, head_dim, state_dim, group_size);- let expected = reference_output(&input);- test_internal(&input, &expected, label);- }- run::<f32>(suffix_len, num_heads, head_dim, state_dim, group_size, label);- run::<f16>(suffix_len, num_heads, head_dim, state_dim, group_size, label);- run::<bf16>(suffix_len, num_heads, head_dim, state_dim, group_size, label);-}--// --- Prefill ----#[uzu_test]-fn test_prefill_basic() {- test_shape(512, 32, 64, 64, 1, "prefill_basic");-}--#[uzu_test]-fn test_prefill_small() {- test_shape(4, 4, 4, 8, 1, "prefill_small");-}--#[uzu_test]-fn test_prefill_minimal() {- test_shape(1, 1, 1, 1, 1, "prefill_minimal");-}--#[uzu_test]-fn test_prefill_multi_group() {- test_shape(8, 8, 4, 16, 4, "prefill_multi_group");-}--#[uzu_test]-fn test_prefill_group_per_head() {- test_shape(8, 4, 4, 8, 1, "prefill_group_per_head");-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rsdeleted file mode 100644index 6cd0d5d63..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs+++ /dev/null@@ -1,244 +0,0 @@-use std::fmt::{Debug, Display};--use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::SplitInProjKernel,- },- cpu::Cpu,- },- data_type::{DataType, DataTypeRepr},- tests::{- assert::assert_eq_float,- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},- },-};--struct Input<T: DataTypeRepr + Float> {- input: Box<[T]>,- z_bias: Box<[T]>,- suffix_length: u32,- total_dim: u32,- conv_dim: u32,- inner_dim: u32,- num_heads: u32,-}--struct Output<T: DataTypeRepr + Float> {- conv_out: Vec<T>,- z_out: Vec<T>,- dt_out: Vec<T>,-}--fn get_input<T: DataTypeRepr + Float>(- suffix_length: u32,- conv_dim: u32,- inner_dim: u32,- num_heads: u32,-) -> Input<T> {- let total_dim = conv_dim + inner_dim + num_heads;- let input_size = suffix_length as usize * total_dim as usize;-- let input: Vec<T> = (0..input_size).map(|i| T::from(0.1 * ((i as f64 % 13.0) - 6.0)).unwrap()).collect();- let z_bias: Vec<T> = (0..inner_dim as usize).map(|i| T::from(0.05 * ((i as f64 % 7.0) - 3.0)).unwrap()).collect();-- Input {- input: input.into_boxed_slice(),- z_bias: z_bias.into_boxed_slice(),- suffix_length,- total_dim,- conv_dim,- inner_dim,- num_heads,- }-}--fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {- let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::SplitInProjKernel::new(&context, T::data_type())- .expect("Failed to create SplitInProjKernel");-- let conv_out_size = input.suffix_length as usize * input.conv_dim as usize;- let z_out_size = input.suffix_length as usize * input.inner_dim as usize;- let dt_out_size = input.suffix_length as usize * input.num_heads as usize;-- let input_buffer = create_buffer_with_data::<B, T>(&context, &input.input);- let z_bias = create_buffer_with_data::<B, T>(&context, &input.z_bias);-- let mut conv_out = create_buffer::<B, T>(&context, conv_out_size);- let mut z_out = create_buffer::<B, T>(&context, z_out_size);- let mut dt_out = create_buffer::<B, T>(&context, dt_out_size);-- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- kernel.encode(- &input_buffer,- &mut conv_out,- &mut z_out,- &mut dt_out,- &z_bias,- input.suffix_length,- input.total_dim,- input.conv_dim,- input.inner_dim,- input.num_heads,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- Output {- conv_out: buffer_to_vec(&conv_out),- z_out: buffer_to_vec(&z_out),- dt_out: buffer_to_vec(&dt_out),- }-}--fn get_test_data<T: DataTypeRepr + Float>(- suffix_length: u32,- conv_dim: u32,- inner_dim: u32,- num_heads: u32,-) -> (Input<T>, Output<T>) {- let input = get_input::<T>(suffix_length, conv_dim, inner_dim, num_heads);- let expected = get_output::<Cpu, T>(&input);- (input, expected)-}--fn test_internal<T: DataTypeRepr + Float + Debug + Display>(- input: &Input<T>,- expected: &Output<T>,- label: &str,-) {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 1e-2f32- } else {- 1e-5- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);- let backend_name = std::any::type_name::<B>();- let type_name = std::any::type_name::<T>();-- assert_eq_float::<T>(- &expected.conv_out,- &output.conv_out,- eps,- &format!("SplitInProj conv_out {backend_name} {label} (type={type_name})"),- );- assert_eq_float::<T>(- &expected.z_out,- &output.z_out,- eps,- &format!("SplitInProj z_out {backend_name} {label} (type={type_name})"),- );- assert_eq_float::<T>(- &expected.dt_out,- &output.dt_out,- eps,- &format!("SplitInProj dt_out {backend_name} {label} (type={type_name})"),- );- });-}--// conv_dim=8, inner_dim=4, num_heads=2, suffix_length=2-fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(2, 8, 4, 2);- test_internal(&input, &expected, "basic");-}--// Single token-fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(1, 8, 4, 2);- test_internal(&input, &expected, "single_token");-}--// Many tokens-fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(16, 8, 4, 2);- test_internal(&input, &expected, "many_tokens");-}--// Larger dimensions closer to real usage-fn test_large<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(8, 128, 64, 32);- test_internal(&input, &expected, "large");-}--// Minimal dimensions-fn test_minimal<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(1, 1, 1, 1);- test_internal(&input, &expected, "minimal");-}--// f32-#[uzu_test]-fn test_basic_f32() {- test_basic::<f32>();-}--#[uzu_test]-fn test_single_token_f32() {- test_single_token::<f32>();-}--#[uzu_test]-fn test_many_tokens_f32() {- test_many_tokens::<f32>();-}--#[uzu_test]-fn test_large_f32() {- test_large::<f32>();-}--#[uzu_test]-fn test_minimal_f32() {- test_minimal::<f32>();-}--// f16-#[uzu_test]-fn test_basic_f16() {- test_basic::<f16>();-}--#[uzu_test]-fn test_single_token_f16() {- test_single_token::<f16>();-}--#[uzu_test]-fn test_many_tokens_f16() {- test_many_tokens::<f16>();-}--#[uzu_test]-fn test_large_f16() {- test_large::<f16>();-}--// bf16-#[uzu_test]-fn test_basic_bf16() {- test_basic::<bf16>();-}--#[uzu_test]-fn test_single_token_bf16() {- test_single_token::<bf16>();-}--#[uzu_test]-fn test_many_tokens_bf16() {- test_many_tokens::<bf16>();-}--#[uzu_test]-fn test_large_bf16() {- test_large::<bf16>();-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rsdeleted file mode 100644index 508e1ddb5..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs+++ /dev/null@@ -1,329 +0,0 @@-use std::fmt::{Debug, Display};--use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::SSDUpdateKernel,- },- cpu::Cpu,- },- data_type::{DataType, DataTypeRepr},- tests::{- assert::assert_eq_float,- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},- },-};--struct Input<T: DataTypeRepr + Float> {- x: Box<[T]>,- dt: Box<[T]>,- b: Box<[T]>,- c: Box<[T]>,- d: Box<[T]>,- z: Box<[T]>,- state: Box<[T]>,- bsz: u32,- h: u32,- dh: u32,- g: u32,- n: u32,- state_in_place: bool,-}--struct Output<T: DataTypeRepr + Float> {- y: Vec<T>,- next_state: Vec<T>,-}--fn get_input<T: DataTypeRepr + Float>(- bsz: u32,- h: u32,- dh: u32,- g: u32,- n: u32,- state_in_place: bool,-) -> Input<T> {- let bsz_u = bsz as usize;- let h_u = h as usize;- let dh_u = dh as usize;- let g_u = g as usize;- let n_u = n as usize;-- let x: Vec<T> = (0..bsz_u * h_u * dh_u).map(|i| T::from(((i % 7) as f64) * 0.1 - 0.2).unwrap()).collect();- let z: Vec<T> = (0..bsz_u * h_u * dh_u).map(|i| T::from(((i % 5) as f64) * 0.1 - 0.1).unwrap()).collect();- let dt: Vec<T> = (0..bsz_u * h_u).map(|i| T::from(((i % 5) as f64) * 0.3 - 1.0).unwrap()).collect();- let b: Vec<T> = (0..bsz_u * g_u * n_u).map(|i| T::from(((i % 11) as f64) * 0.02 - 0.05).unwrap()).collect();- let c: Vec<T> = (0..bsz_u * g_u * n_u).map(|i| T::from(((i % 13) as f64) * 0.015).unwrap()).collect();- let d: Vec<T> = (0..h_u).map(|i| T::from(((i % 3) as f64) * 0.05).unwrap()).collect();- let state: Vec<T> =- (0..bsz_u * h_u * dh_u * n_u).map(|i| T::from(((i % 23) as f64) * 0.01 - 0.05).unwrap()).collect();-- Input {- x: x.into_boxed_slice(),- dt: dt.into_boxed_slice(),- b: b.into_boxed_slice(),- c: c.into_boxed_slice(),- d: d.into_boxed_slice(),- z: z.into_boxed_slice(),- state: state.into_boxed_slice(),- bsz,- h,- dh,- g,- n,- state_in_place,- }-}--fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {- let context = B::Context::new().expect("Failed to create Context");- let kernel =- <<B as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&context, T::data_type(), input.state_in_place)- .expect("Failed to create SSDUpdateKernel");-- let bsz = input.bsz as usize;- let h = input.h as usize;- let dh = input.dh as usize;- let g = input.g as usize;- let n = input.n as usize;-- let x_strides = [(h * dh) as u32, dh as u32, 1u32];- let dt_strides = [h as u32, 1u32];- let cb_strides = [(g * n) as u32, n as u32, 1u32];- let state_strides = [(h * dh * n) as u32, (dh * n) as u32, n as u32, 1u32];-- let y_size = bsz * h * dh;- let ns_size = bsz * h * dh * n;-- let x = create_buffer_with_data::<B, T>(&context, &input.x);- let dt = create_buffer_with_data::<B, T>(&context, &input.dt);- let b = create_buffer_with_data::<B, T>(&context, &input.b);- let c = create_buffer_with_data::<B, T>(&context, &input.c);- let d = create_buffer_with_data::<B, T>(&context, &input.d);- let z = create_buffer_with_data::<B, T>(&context, &input.z);-- let mut y = create_buffer::<B, T>(&context, y_size);-- if input.state_in_place {- let mut next_state = create_buffer_with_data::<B, T>(&context, &input.state);-- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- kernel.encode(- &x,- &dt,- &b,- &c,- &d,- &z,- None::<&B::GlobalBuffer>,- &mut y,- &mut next_state,- (h / g) as u32,- input.n,- &x_strides,- &dt_strides,- &cb_strides,- &state_strides,- input.bsz,- input.h,- input.dh,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- Output {- y: buffer_to_vec(&y),- next_state: buffer_to_vec(&next_state),- }- } else {- let state = create_buffer_with_data::<B, T>(&context, &input.state);- let mut next_state = create_buffer::<B, T>(&context, ns_size);-- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- kernel.encode(- &x,- &dt,- &b,- &c,- &d,- &z,- Some(&state),- &mut y,- &mut next_state,- (h / g) as u32,- input.n,- &x_strides,- &dt_strides,- &cb_strides,- &state_strides,- input.bsz,- input.h,- input.dh,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- Output {- y: buffer_to_vec(&y),- next_state: buffer_to_vec(&next_state),- }- }-}--fn get_test_data<T: DataTypeRepr + Float>(- bsz: u32,- h: u32,- dh: u32,- g: u32,- n: u32,- state_in_place: bool,-) -> (Input<T>, Output<T>) {- let input = get_input::<T>(bsz, h, dh, g, n, state_in_place);- let expected = get_output::<Cpu, T>(&input);- (input, expected)-}--fn test_internal<T: DataTypeRepr + Float + Debug + Display>(- input: &Input<T>,- expected: &Output<T>,- label: &str,-) {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 2e-2f32- } else {- 1e-5- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);- let backend_name = std::any::type_name::<B>();- let type_name = std::any::type_name::<T>();-- assert_eq_float::<T>(- &expected.y,- &output.y,- eps,- &format!("SSDUpdate y {backend_name} {label} (type={type_name})"),- );- assert_eq_float::<T>(- &expected.next_state,- &output.next_state,- eps,- &format!("SSDUpdate next_state {backend_name} {label} (type={type_name})"),- );- });-}--// bsz=1, h=4, dh=3, g=2, n=8 (matches existing Metal test)-fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, false);- test_internal(&input, &expected, "basic");-}--// bsz=2 multi-batch-fn test_multi_batch<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(2, 4, 3, 2, 8, false);- test_internal(&input, &expected, "multi_batch");-}--// state_in_place=true-fn test_state_in_place<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, true);- test_internal(&input, &expected, "state_in_place");-}--// Larger dimensions closer to real usage-fn test_large<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(1, 8, 4, 4, 16, false);- test_internal(&input, &expected, "large");-}--// Minimal: single head, single group, dh=1, n=1-fn test_minimal<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(1, 1, 1, 1, 1, false);- test_internal(&input, &expected, "minimal");-}--// g == h (each head is its own group)-fn test_group_per_head<T: DataTypeRepr + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(1, 4, 2, 4, 8, false);- test_internal(&input, &expected, "group_per_head");-}--// f32-#[uzu_test]-fn test_basic_f32() {- test_basic::<f32>();-}--#[uzu_test]-fn test_multi_batch_f32() {- test_multi_batch::<f32>();-}--#[uzu_test]-fn test_state_in_place_f32() {- test_state_in_place::<f32>();-}--#[uzu_test]-fn test_large_f32() {- test_large::<f32>();-}--#[uzu_test]-fn test_minimal_f32() {- test_minimal::<f32>();-}--#[uzu_test]-fn test_group_per_head_f32() {- test_group_per_head::<f32>();-}--// f16-#[uzu_test]-fn test_basic_f16() {- test_basic::<f16>();-}--#[uzu_test]-fn test_multi_batch_f16() {- test_multi_batch::<f16>();-}--#[uzu_test]-fn test_state_in_place_f16() {- test_state_in_place::<f16>();-}--#[uzu_test]-fn test_large_f16() {- test_large::<f16>();-}--// bf16-#[uzu_test]-fn test_basic_bf16() {- test_basic::<bf16>();-}--#[uzu_test]-fn test_multi_batch_bf16() {- test_multi_batch::<bf16>();-}--#[uzu_test]-fn test_state_in_place_bf16() {- test_state_in_place::<bf16>();-}--#[uzu_test]-fn test_large_bf16() {- test_large::<bf16>();-}diff --git a/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs b/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rsdeleted file mode 100644index 3702cf678..000000000--- a/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs+++ /dev/null@@ -1,470 +0,0 @@-use uzu_engine_macros::uzu_test;--use crate::{- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::activation_type::ActivationType,- kernel::{Conv1dScanKernel, SSDPrefill64Kernel, SSDPrefillKernel},- },- data_type::DataType,- tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},-};--#[derive(Debug, Clone, Copy)]-enum SSDPrefillMode {- Universal,- Special64,-}--fn ssd_prefill_cpu_reference(- suffix_len: usize,- num_heads: usize,- head_dim: usize,- state_dim: usize,- group_size: i32,- x_data: &[f32],- dt_raw_data: &[f32],- b_data: &[f32],- c_data: &[f32],- d_data: &[f32],- z_data: &[f32],- state_init: &[f32],- x_strides: [usize; 3],- dt_strides: [usize; 2],- cb_strides: [usize; 3],- state_strides: [usize; 3],-) -> (Vec<f32>, Vec<f32>) {- let total_pairs = suffix_len * num_heads * head_dim;- let mut y_out = vec![0.0f32; total_pairs];- let mut state = state_init.to_vec();- let safe_group = group_size.max(1) as usize;-- for h in 0..num_heads {- let group_idx = h / safe_group;- for dh in 0..head_dim {- let state_base = h * state_strides[0] + dh * state_strides[1];- for token in 0..suffix_len {- let x_idx = token * x_strides[0] + h * x_strides[1] + dh * x_strides[2];- let dt_idx = token * dt_strides[0] + h * dt_strides[1];- let cb_base = token * cb_strides[0] + group_idx * cb_strides[1];-- let x_val = x_data[x_idx];- let dt_raw = dt_raw_data[dt_idx];- let dt_val = ActivationType::SOFTPLUS.activate(dt_raw);- let decay_val = (-dt_val).exp();- let dt_scaled_input = x_val;- let gate = ActivationType::SILU.activate(z_data[x_idx]);- let mut acc = d_data[h] * x_val;-- for s in 0..state_dim {- let state_idx = state_base + s * state_strides[2];- let cb_idx = cb_base + s * cb_strides[2];- let b_coeff = b_data[cb_idx];- let c_coeff = c_data[cb_idx];- let new_state = decay_val * state[state_idx] + dt_scaled_input * b_coeff;- state[state_idx] = new_state;- acc += new_state * c_coeff;- }-- y_out[x_idx] = acc * gate;- }- }- }-- (y_out, state)-}--struct SSDPrefillFixture {- suffix_len: usize,- num_heads: usize,- head_dim: usize,- state_dim: usize,- group_size: i32,- x_strides: [usize; 3],- dt_strides: [usize; 2],- cb_strides: [usize; 3],- state_strides: [usize; 3],- x_data: Vec<f32>,- dt_data: Vec<f32>,- b_data: Vec<f32>,- c_data: Vec<f32>,- d_data: Vec<f32>,- z_data: Vec<f32>,- state_init: Vec<f32>,-}--impl SSDPrefillFixture {- fn new() -> Self {- let suffix_len = 512usize;- let num_heads = 32usize;- let head_dim = 64usize;- let state_dim = 64usize;- let group_size = 1i32;- let group_count = num_heads / (group_size as usize);- let total_pairs = num_heads * head_dim;- let total_x = suffix_len * total_pairs;- let total_dt = suffix_len * num_heads;- let total_cb = suffix_len * group_count * state_dim;- let total_state = num_heads * head_dim * state_dim;-- let x_data: Vec<f32> = (0..total_x).map(|i| ((i % 17) as f32) * 0.01 - 0.05).collect();- let dt_data: Vec<f32> = (0..total_dt).map(|i| ((i % 13) as f32) * 0.2 - 1.5).collect();- let b_data: Vec<f32> = (0..total_cb).map(|i| ((i % 11) as f32) * 0.02 - 0.05).collect();- let c_data: Vec<f32> = (0..total_cb).map(|i| ((i % 19) as f32) * 0.01 - 0.02).collect();- let d_data: Vec<f32> = (0..num_heads).map(|i| ((i % 3) as f32) * 0.05 - 0.05).collect();- let z_data: Vec<f32> = (0..total_x).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();- let state_init: Vec<f32> = (0..total_state).map(|i| ((i % 29) as f32) * 0.03 - 0.4).collect();-- let x_strides = [num_heads * head_dim, head_dim, 1usize];- let dt_strides = [num_heads, 1usize];- let cb_strides = [group_count * state_dim, state_dim, 1usize];- let state_strides = [head_dim * state_dim, state_dim, 1usize];-- Self {- suffix_len,- num_heads,- head_dim,- state_dim,- group_size,- x_strides,- dt_strides,- cb_strides,- state_strides,- x_data,- dt_data,- b_data,- c_data,- d_data,- z_data,- state_init,- }- }-}--fn run_prefill_kernel_mode<B: Backend>(- ctx: &B::Context,- fixture: &SSDPrefillFixture,- mode: SSDPrefillMode,-) -> (Vec<f32>, Vec<f32>) {- let x_buf = create_buffer_with_data::<B, _>(ctx, &fixture.x_data);- let dt_buf = create_buffer_with_data::<B, _>(ctx, &fixture.dt_data);- let b_buf = create_buffer_with_data::<B, _>(ctx, &fixture.b_data);- let c_buf = create_buffer_with_data::<B, _>(ctx, &fixture.c_data);- let d_buf = create_buffer_with_data::<B, _>(ctx, &fixture.d_data);- let z_buf = create_buffer_with_data::<B, _>(ctx, &fixture.z_data);- let mut state_buf = create_buffer_with_data::<B, _>(ctx, &fixture.state_init);- let mut y_buf = create_buffer::<B, f32>(ctx, fixture.suffix_len * fixture.num_heads * fixture.head_dim);-- let x_strides = fixture.x_strides.map(|stride| stride as u32);- let dt_strides = fixture.dt_strides.map(|stride| stride as u32);- let cb_strides = fixture.cb_strides.map(|stride| stride as u32);- let state_strides = fixture.state_strides.map(|stride| stride as u32);-- let mut command_buffer = ctx.create_command_buffer(None, None).unwrap();- match mode {- SSDPrefillMode::Universal => {- let kernel = <<B as Backend>::Kernels as Kernels>::SSDPrefillKernel::new(ctx, DataType::F32)- .expect("Failed to create SSD prefill kernel");- kernel.encode(- &x_buf,- &dt_buf,- &b_buf,- &c_buf,- &d_buf,- &z_buf,- &mut state_buf,- &mut y_buf,- fixture.suffix_len as u32,- fixture.group_size as u32,- fixture.state_dim as u32,- &x_strides,- &dt_strides,- &cb_strides,- &state_strides,- fixture.num_heads as u32,- fixture.head_dim as u32,- &mut command_buffer,- );- },- SSDPrefillMode::Special64 => {- assert_eq!(fixture.state_dim, 64);- let kernel = <<B as Backend>::Kernels as Kernels>::SSDPrefill64Kernel::new(ctx, DataType::F32)- .expect("Failed to create SSD prefill64 kernel");- kernel.encode(- &x_buf,- &dt_buf,- &b_buf,- &c_buf,- &d_buf,- &z_buf,- &mut state_buf,- &mut y_buf,- fixture.suffix_len as u32,- fixture.group_size as u32,- fixture.state_dim as u32,- &x_strides,- &dt_strides,- &cb_strides,- &state_strides,- fixture.num_heads as u32,- fixture.head_dim as u32,- &mut command_buffer,- );- },- }- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let y_vec = buffer_to_vec::<B, f32>(&y_buf);- let state_vec = buffer_to_vec::<B, f32>(&state_buf);- drop(y_buf);- drop(completed);- (y_vec, state_vec)-}--fn run_conv_scan_once<B: Backend>(- ctx: &<B as Backend>::Context,- kernel: &<<B as Backend>::Kernels as Kernels>::Conv1dScanKernel,- suffix_len: usize,- channels: usize,- kernel_size: i32,- tap_count: usize,- x_data: &[f32],- w_data: &[f32],- b_data: &[f32],- state_init: &[f32],- use_scratch: bool,- alias_io: bool,-) -> (Vec<f32>, Vec<f32>) {- let total_x = suffix_len * channels;- let _total_w = channels * kernel_size as usize;- let total_state = channels * tap_count;-- let mut y_buf = if alias_io {- create_buffer_with_data::<B, _>(ctx, x_data)- } else {- create_buffer::<B, f32>(ctx, total_x)- };- let mut b_out_buf = create_buffer::<B, f32>(ctx, total_x);- let mut c_out_buf = create_buffer::<B, f32>(ctx, total_x);- let w_buf = create_buffer_with_data::<B, _>(ctx, w_data);- let b_buf = create_buffer_with_data::<B, _>(ctx, b_data);- let mut state_buf = create_buffer_with_data::<B, _>(ctx, state_init);- let mut scratch_buf = create_buffer::<B, f32>(ctx, total_state);-- let padded_len = tap_count + suffix_len;- let mut padded_host = vec![0.0f32; padded_len * channels];- for tap in 0..tap_count {- for ch in 0..channels {- padded_host[tap * channels + ch] = state_init[ch * tap_count + tap];- }- }- for token in 0..suffix_len {- for ch in 0..channels {- padded_host[(tap_count + token) * channels + ch] = x_data[token * channels + ch];- }- }- let padded_buf = create_buffer_with_data::<B, _>(ctx, &padded_host);-- let mut command_buffer = ctx.create_command_buffer(None, None).unwrap();- if use_scratch && tap_count > 0 {- command_buffer.encode_fill(&mut scratch_buf, 0);- }- kernel.encode(- &padded_buf,- &w_buf,- Some(&b_buf),- &mut y_buf,- &mut b_out_buf,- &mut c_out_buf,- &mut state_buf,- suffix_len as u32,- kernel_size as u32,- channels as u32,- tap_count as u32,- channels as u32,- channels as u32,- 0u32,- ActivationType::SILU,- &mut command_buffer,- );-- if use_scratch && tap_count > 0 {- command_buffer.encode_copy(&scratch_buf, &mut state_buf);- }-- command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let y_vec = buffer_to_vec::<B, f32>(&y_buf);- let state_vec = buffer_to_vec::<B, f32>(&state_buf);- (y_vec, state_vec)-}--fn assert_deterministic_for_mode<B: Backend>(mode: SSDPrefillMode) {- let Some(ctx) = <B as Backend>::Context::new().ok() else {- eprintln!("Skipping SSD prefill determinism test: no Metal device");- return;- };- let fixture = SSDPrefillFixture::new();-- let (y_a, state_a) = run_prefill_kernel_mode::<B>(&ctx, &fixture, mode);- let (y_b, state_b) = run_prefill_kernel_mode::<B>(&ctx, &fixture, mode);-- assert_eq!(y_a, y_b, "Prefill outputs differ in {:?} mode", mode);- assert_eq!(state_a, state_b, "Prefill states differ in {:?} mode", mode);-}--fn assert_matches_cpu_reference<B: Backend>(mode: SSDPrefillMode) {- let Some(ctx) = <B as Backend>::Context::new().ok() else {- eprintln!("Skipping SSD prefill reference test: no Metal device");- return;- };- let fixture = SSDPrefillFixture::new();-- let (y_ref, state_ref) = ssd_prefill_cpu_reference(- fixture.suffix_len,- fixture.num_heads,- fixture.head_dim,- fixture.state_dim,- fixture.group_size,- &fixture.x_data,- &fixture.dt_data,- &fixture.b_data,- &fixture.c_data,- &fixture.d_data,- &fixture.z_data,- &fixture.state_init,- fixture.x_strides,- fixture.dt_strides,- fixture.cb_strides,- fixture.state_strides,- );-- let (y_gpu, state_gpu) = run_prefill_kernel_mode::<B>(&ctx, &fixture, mode);-- let tolerance = 5e-5f32;- let mut max_y_diff = 0.0f32;- let mut max_y_idx = 0usize;- for (idx, (&lhs, &rhs)) in y_gpu.iter().zip(&y_ref).enumerate() {- let diff = (lhs - rhs).abs();- if diff > max_y_diff {- max_y_diff = diff;- max_y_idx = idx;- }- }- assert!(- max_y_diff <= tolerance,- "Prefill outputs diverge in {:?} mode at idx {max_y_idx}: metal={} cpu={} (diff {max_y_diff})",- mode,- y_gpu[max_y_idx],- y_ref[max_y_idx]- );-- let mut max_state_diff = 0.0f32;- let mut max_state_idx = 0usize;- for (idx, (&lhs, &rhs)) in state_gpu.iter().zip(&state_ref).enumerate() {- let diff = (lhs - rhs).abs();- if diff > max_state_diff {- max_state_diff = diff;- max_state_idx = idx;- }- }- assert!(- max_state_diff <= tolerance,- "Prefill states diverge in {:?} mode at idx {max_state_idx}: metal={} cpu={} (diff {max_state_diff})",- mode,- state_gpu[max_state_idx],- state_ref[max_state_idx]- );-}--fn conv1d_scan_deterministic_internal<B: Backend>() {- let Some(ctx) = <B as Backend>::Context::new().ok() else {- eprintln!("Skipping conv1d scan determinism test: no Metal device");- return;- };- let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, true).unwrap();-- let suffix_len = 192usize;- let channels = 8usize;- let kernel_size = 5i32;- let tap_count = (kernel_size - 1) as usize;-- let total_x = suffix_len * channels;- let total_state = channels * tap_count;-- let x_data: Vec<f32> = (0..total_x).map(|i| ((i % 31) as f32) * 0.02 - 0.3).collect();- let w_data: Vec<f32> = (0..(channels * kernel_size as usize)).map(|i| ((i % 17) as f32) * 0.01 - 0.04).collect();- let b_data: Vec<f32> = (0..channels).map(|i| ((i % 5) as f32) * 0.03 - 0.07).collect();- let state_init: Vec<f32> = (0..total_state).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();-- let use_scratch = tap_count > 0 && suffix_len > 1;-- for &alias_io in &[false, true] {- let first = run_conv_scan_once::<B>(- &ctx,- &kernel,- suffix_len,- channels,- kernel_size,- tap_count,- &x_data,- &w_data,- &b_data,- &state_init,- use_scratch,- alias_io,- );- let second = run_conv_scan_once::<B>(- &ctx,- &kernel,- suffix_len,- channels,- kernel_size,- tap_count,- &x_data,- &w_data,- &b_data,- &state_init,- use_scratch,- alias_io,- );-- assert_eq!(first.0, second.0, "Conv outputs differ (alias_io={alias_io})");- assert_eq!(first.1, second.1, "Conv states differ (alias_io={alias_io})");- }-}--#[uzu_test]-fn ssd_prefill_universal_is_deterministic() {- for_each_non_cpu_backend!(|B| {- assert_deterministic_for_mode::<B>(SSDPrefillMode::Universal);- });-}--#[uzu_test]-fn ssd_prefill_special64_is_deterministic() {- for_each_non_cpu_backend!(|B| {- assert_deterministic_for_mode::<B>(SSDPrefillMode::Special64);- });-}--#[uzu_test]-fn ssd_prefill_universal_matches_cpu_reference() {- for_each_non_cpu_backend!(|B| {- assert_matches_cpu_reference::<B>(SSDPrefillMode::Universal);- });-}--#[uzu_test]-fn ssd_prefill_special64_matches_cpu_reference() {- for_each_non_cpu_backend!(|B| {- assert_matches_cpu_reference::<B>(SSDPrefillMode::Special64);- });-}--#[uzu_test]-fn conv1d_scan_is_deterministic() {- for_each_non_cpu_backend!(|B| {- conv1d_scan_deterministic_internal::<B>();- });-}