Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Small cleanup part 6 (#957)

uuuvn3602af9b09e5
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs
new file mode 100644
index 000000000..9f97cdadc
--- /dev/null
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs
@@ -0,0 +1,39 @@
+use half::bf16;
+use num_traits::Float;
+use uzu_engine_macros::kernel;
+
+use crate::data_type::DataTypeRepr;
+
+#[kernel(Conv1dPack)]
+#[variants(StateT, f32, bf16)]
+#[variants(InputT, f32, bf16)]
+pub fn conv1d_pack<StateT: DataTypeRepr + Float, InputT: DataTypeRepr + Float>(
+ state_in: *const StateT,
+ x: *const InputT,
+ padded: *mut StateT,
+ state_stride: u32,
+ row_stride: u32,
+ suffix_len: u32,
+ num_channels: u32,
+) {
+ let state_stride = state_stride as usize;
+ let row_stride = row_stride as usize;
+ let suffix_len = suffix_len as usize;
+ let num_channels = num_channels as usize;
+
+ unsafe {
+ for channel_idx in 0..num_channels {
+ for row_idx in 0..state_stride + suffix_len {
+ let padded_index = row_idx * row_stride + channel_idx;
+ if row_idx < state_stride {
+ let state_index = channel_idx * state_stride + row_idx;
+ *padded.add(padded_index) = *state_in.add(state_index);
+ } else {
+ let token = row_idx - state_stride;
+ let x_index = token * row_stride + channel_idx;
+ *padded.add(padded_index) = StateT::from((*x.add(x_index)).to_f32().unwrap()).unwrap();
+ }
+ }
+ }
+ }
+}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/mod.rs
index e0c4e7865..075767ac1 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/mod.rs
@@ -1,3 +1,4 @@
+pub mod conv_pack;
pub mod conv_scan;
pub mod conv_update;
pub mod norm_gate;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
index e96c27e1c..26f1d4619 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
@@ -18,7 +18,6 @@ mod radix_top_k_small;
mod sampling;
mod short_conv;
mod softmax;
-mod ssm;
mod tensor_add_bias;
mod tensor_add_scale;
mod weaver;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs
deleted file mode 100644
index f16d261c7..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs
+++ /dev/null
@@ -1,201 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::data_type::DataTypeRepr;
-
-#[kernel(Conv1dPack)]
-#[variants(StateT, f32, bf16)]
-#[variants(InputT, f32, bf16)]
-pub fn conv1d_pack<StateT: DataTypeRepr + Float, InputT: DataTypeRepr + Float>(
- state_in: *const StateT,
- x: *const InputT,
- padded: *mut StateT,
- state_stride: u32,
- row_stride: u32,
- suffix_len: u32,
- num_channels: u32,
-) {
- let state_stride = state_stride as usize;
- let row_stride = row_stride as usize;
- let suffix_len = suffix_len as usize;
- let num_channels = num_channels as usize;
-
- unsafe {
- for channel_idx in 0..num_channels {
- for row_idx in 0..state_stride + suffix_len {
- let padded_index = row_idx * row_stride + channel_idx;
- if row_idx < state_stride {
- let state_index = channel_idx * state_stride + row_idx;
- *padded.add(padded_index) = *state_in.add(state_index);
- } else {
- let token = row_idx - state_stride;
- let x_index = token * row_stride + channel_idx;
- *padded.add(padded_index) = StateT::from((*x.add(x_index)).to_f32().unwrap()).unwrap();
- }
- }
- }
- }
-}
-
-#[kernel(Conv1dDecode)]
-#[variants(T, f32, f16, bf16)]
-pub fn conv1d_decode<T: DataTypeRepr + Float>(
- x: *const T,
- w: *const T,
- #[optional(has_bias)] b: Option<*const T>,
- #[optional(!state_in_place)] state: Option<*const T>,
- x_out: *mut T,
- b_out: *mut T,
- c_out: *mut T,
- next_state: *mut T,
- kernel_size: u32,
- row_stride: u32,
- state_stride: u32,
- num_channels: u32,
- suffix_len: u32,
- inner_dim: u32,
- proj_dim: u32,
- activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,
- #[specialize] has_bias: bool,
- #[specialize] state_in_place: bool,
-) {
- let kernel_size = kernel_size as usize;
- let row_stride = row_stride as usize;
- let state_stride = state_stride as usize;
- let num_channels = num_channels as usize;
- let suffix_len = suffix_len as usize;
- let inner_dim = inner_dim as usize;
- let proj_dim = proj_dim as usize;
-
- let state = match state_in_place {
- true => next_state,
- false => state.unwrap(),
- };
-
- unsafe {
- for token_idx in 0..suffix_len {
- for channel_idx in 0..num_channels {
- let x_idx = token_idx * row_stride + channel_idx;
- let state_offset = channel_idx * state_stride;
- let w_row = w.add(channel_idx * kernel_size);
-
- let mut acc = 0.0f32;
- if has_bias {
- acc = (*b.unwrap().add(channel_idx)).to_f32().unwrap();
- }
-
- let state_taps = kernel_size.saturating_sub(1);
- for tap in 0..state_taps {
- let state_index = state_offset + tap;
- let sample = (*state.add(state_index)).to_f32().unwrap();
- acc += (*w_row.add(tap)).to_f32().unwrap() * sample;
- }
-
- let current = (*x.add(x_idx)).to_f32().unwrap();
- let current_tap_index = state_taps;
- if current_tap_index < kernel_size {
- acc += (*w_row.add(current_tap_index)).to_f32().unwrap() * current;
- }
-
- let activated = activation_type.activate(T::from(acc).unwrap());
- if channel_idx < inner_dim {
- let dst = token_idx * inner_dim + channel_idx;
- *x_out.add(dst) = activated;
- } else if channel_idx < inner_dim + proj_dim {
- let dst = token_idx * proj_dim + (channel_idx - inner_dim);
- *b_out.add(dst) = activated;
- } else if channel_idx < inner_dim + 2 * proj_dim {
- let dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
- *c_out.add(dst) = activated;
- }
- if state_taps == 0 {
- continue;
- }
-
- for tap in 0..state_taps - 1 {
- let src_index = state_offset + tap + 1;
- let dst_index = state_offset + tap;
- *next_state.add(dst_index) = *state.add(src_index);
- }
-
- let tail_index = state_offset + state_taps - 1;
- *next_state.add(tail_index) = T::from(current).unwrap();
- }
- }
- }
-}
-
-#[kernel(Conv1dScan)]
-#[variants(T, f32, f16, bf16)]
-pub fn conv1d_scan<T: DataTypeRepr + Float>(
- padded: *const T,
- w: *const T,
- #[optional(has_bias)] b: Option<*const T>,
- x_out: *mut T,
- b_out: *mut T,
- c_out: *mut T,
- state_out: *mut T,
- suffix_len: u32,
- kernel_size: u32,
- row_stride: u32,
- state_stride: u32,
- num_channels: u32,
- inner_dim: u32,
- proj_dim: u32,
- activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,
- #[specialize] has_bias: bool,
-) {
- let suffix_len = suffix_len as usize;
- let kernel_size = kernel_size as usize;
- let row_stride = row_stride as usize;
- let state_stride = state_stride as usize;
- let num_channels = num_channels as usize;
- let inner_dim = inner_dim as usize;
- let proj_dim = proj_dim as usize;
-
- unsafe {
- for token_idx in 0..suffix_len + kernel_size.saturating_sub(1) {
- for channel_idx in 0..num_channels {
- let tap_count = kernel_size.saturating_sub(1);
- let state_offset = channel_idx * state_stride;
- let weight_offset = channel_idx * kernel_size;
- let w_row = w.add(weight_offset);
-
- if token_idx < suffix_len {
- let mut acc = 0.0f32;
- if has_bias {
- acc = (*b.unwrap().add(channel_idx)).to_f32().unwrap();
- }
-
- for tap in 0..kernel_size {
- let padded_index = (token_idx + tap) * row_stride + channel_idx;
- let sample = (*padded.add(padded_index)).to_f32().unwrap();
- acc += (*w_row.add(tap)).to_f32().unwrap() * sample;
- }
-
- let activated = activation_type.activate(T::from(acc).unwrap());
- if channel_idx < inner_dim {
- let dst = token_idx * inner_dim + channel_idx;
- *x_out.add(dst) = activated;
- } else if channel_idx < inner_dim + proj_dim {
- let dst = token_idx * proj_dim + (channel_idx - inner_dim);
- *b_out.add(dst) = activated;
- } else if channel_idx < inner_dim + 2 * proj_dim {
- let dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
- *c_out.add(dst) = activated;
- }
- } else if tap_count > 0 {
- let tap = token_idx - suffix_len;
- if tap >= tap_count {
- return;
- }
-
- let padded_index = token_idx * row_stride + channel_idx;
- let sample = (*padded.add(padded_index)).to_f32().unwrap();
- *state_out.add(state_offset + tap) = T::from(sample).unwrap();
- }
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/mod.rs
deleted file mode 100644
index fb01e93a7..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/mod.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-pub mod conv1d;
-pub mod split_inproj;
-pub mod ssd_prefill;
-pub mod ssd_update;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs
deleted file mode 100644
index e6268c9a4..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs
+++ /dev/null
@@ -1,45 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::data_type::DataTypeRepr;
-
-#[kernel(SplitInProj)]
-#[variants(T, f32, f16, bf16)]
-pub fn split_in_proj<T: DataTypeRepr + Float>(
- input: *const T,
- conv_out: *mut T,
- z_out: *mut T,
- dt_out: *mut T,
- z_bias: *const T,
- suffix_length: u32,
- total_dim: u32,
- conv_dim: u32,
- inner_dim: u32,
- num_heads: u32,
-) {
- let suffix_length = suffix_length as usize;
- let total_dim = total_dim as usize;
- let conv_dim = conv_dim as usize;
- let inner_dim = inner_dim as usize;
- let num_heads = num_heads as usize;
-
- unsafe {
- for row in 0..suffix_length {
- for col in 0..total_dim {
- let input_idx = row * total_dim + col;
- if col < conv_dim {
- let dst = row * conv_dim + col;
- *conv_out.add(dst) = *input.add(input_idx);
- } else if col < conv_dim + inner_dim {
- let dst = row * inner_dim + (col - conv_dim);
- let bias_idx = col - conv_dim;
- *z_out.add(dst) = *input.add(input_idx) + *z_bias.add(bias_idx);
- } else if col < conv_dim + inner_dim + num_heads {
- let dst = row * num_heads + (col - conv_dim - inner_dim);
- *dt_out.add(dst) = *input.add(input_idx);
- }
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs
deleted file mode 100644
index 84ec1a5b2..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::data_type::DataTypeRepr;
-
-#[kernel(SSDPrefill64)]
-#[variants(T, f32, f16, bf16)]
-pub fn ssd_prefill64<T: DataTypeRepr + Float>(
- #[allow(unused)] x: *const T,
- #[allow(unused)] dt_raw: *const T,
- #[allow(unused)] b: *const T,
- #[allow(unused)] c: *const T,
- #[allow(unused)] d: *const T,
- #[allow(unused)] z: *const T,
- #[allow(unused)] state: *mut T,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] suffix_len: u32,
- #[allow(unused)] group_size: u32,
- #[allow(unused)] state_size: u32,
- #[allow(unused)] x_strides: &[u32; 3],
- #[allow(unused)] dt_strides: &[u32; 2],
- #[allow(unused)] cb_strides: &[u32; 3],
- #[allow(unused)] state_strides: &[u32; 3],
- #[allow(unused)] num_heads: u32,
- #[allow(unused)] head_dim: u32,
-) {
- todo!()
-}
-
-#[kernel(SSDPrefill)]
-#[variants(T, f32, f16, bf16)]
-pub fn ssd_prefill<T: DataTypeRepr + Float>(
- #[allow(unused)] x: *const T,
- #[allow(unused)] dt_raw: *const T,
- #[allow(unused)] b: *const T,
- #[allow(unused)] c: *const T,
- #[allow(unused)] d: *const T,
- #[allow(unused)] z: *const T,
- #[allow(unused)] state: *mut T,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] suffix_len: u32,
- #[allow(unused)] group_size: u32,
- #[allow(unused)] state_size: u32,
- #[allow(unused)] x_strides: &[u32; 3],
- #[allow(unused)] dt_strides: &[u32; 2],
- #[allow(unused)] cb_strides: &[u32; 3],
- #[allow(unused)] state_strides: &[u32; 3],
- #[allow(unused)] num_heads: u32,
- #[allow(unused)] head_dim: u32,
-) {
- todo!()
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs
deleted file mode 100644
index c7669cb04..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs
+++ /dev/null
@@ -1,75 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};
-
-#[kernel(SSDUpdate)]
-#[variants(T, f32, f16, bf16)]
-pub fn ssd_update<T: DataTypeRepr + Float>(
- x: *const T,
- dt_raw: *const T,
- b: *const T,
- c: *const T,
- d: *const T,
- z: *const T,
- #[optional(!state_in_place)] state: Option<*const T>,
- y: *mut T,
- next_state: *mut T,
- group_size: u32,
- state_size: u32,
- x_strides: &[u32; 3],
- dt_strides: &[u32; 2],
- cb_strides: &[u32; 3],
- state_strides: &[u32; 4],
- b_size: u32,
- h_size: u32,
- dh_size: u32,
- #[specialize] state_in_place: bool,
-) {
- let group_size = group_size as usize;
- let state_size = state_size as usize;
- let b_size = b_size as usize;
- let h_size = h_size as usize;
- let dh_size = dh_size as usize;
-
- let state = match state_in_place {
- true => next_state,
- false => state.unwrap(),
- };
-
- unsafe {
- for b_idx in 0..b_size {
- for h_idx in 0..h_size {
- for dh_idx in 0..dh_size {
- let cb_start_idx = b_idx * cb_strides[0] as usize + (h_idx / group_size) * cb_strides[1] as usize;
- let x_idx =
- b_idx * x_strides[0] as usize + h_idx * x_strides[1] as usize + dh_idx * x_strides[2] as usize;
- let dt_idx = b_idx * dt_strides[0] as usize + h_idx * dt_strides[1] as usize;
- let state_start_idx = b_idx * state_strides[0] as usize
- + h_idx * state_strides[1] as usize
- + dh_idx * state_strides[2] as usize;
-
- let this_x = *x.add(x_idx);
- let dt_raw_val = *dt_raw.add(dt_idx);
- let this_dt = ActivationType::SOFTPLUS.activate(dt_raw_val);
- let this_decay = T::from((-this_dt.to_f32().unwrap()).exp()).unwrap();
- let this_d = *d.add(h_idx);
- let this_z = ActivationType::SILU.activate(*z.add(x_idx));
-
- let mut temp = T::zero();
- for i in 0..state_size {
- let cb_idx = cb_start_idx + i;
- let state_idx = state_start_idx + i;
- let this_new_state = *state.add(state_idx) * this_decay + *b.add(cb_idx) * this_x;
- *next_state.add(state_idx) = this_new_state;
- temp = temp + this_new_state * *c.add(cb_idx);
- }
- temp = temp + this_d * this_x;
- temp = temp * this_z;
- *y.add(x_idx) = temp;
- }
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal
new file mode 100644
index 000000000..9e076f61f
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal
@@ -0,0 +1,27 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+template <typename StateT, typename InputT>
+VARIANTS(StateT, float, bfloat)
+VARIANTS(InputT, float, bfloat)
+PUBLIC KERNEL(Conv1dPack)(
+ device const StateT* state_in,
+ device const InputT* x,
+ device StateT* padded,
+ constant const uint& state_stride,
+ constant const uint& row_stride,
+ constant const uint& suffix_len,
+ constant const uint& num_channels,
+ const uint channel_idx AXIS(num_channels, 32),
+ const uint row_idx AXIS(state_stride + suffix_len, 1)
+) {
+ const uint padded_index = row_idx * row_stride + channel_idx;
+ if (row_idx < state_stride) {
+ const uint state_index = channel_idx * state_stride + row_idx;
+ padded[padded_index] = state_in[state_index];
+ } else {
+ const uint token = row_idx - state_stride;
+ const uint x_index = token * row_stride + channel_idx;
+ padded[padded_index] = static_cast<StateT>(x[x_index]);
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu-engine/src/backends/metal/kernel/ssm/conv1d.metal
deleted file mode 100644
index 9d997c001..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/ssm/conv1d.metal
+++ /dev/null
@@ -1,167 +0,0 @@
-#include <metal_stdlib>
-#include "../activation/activations.h"
-#include "../common/dsl.h"
-
-using namespace uzu::activation_type;
-
-template <typename StateT, typename InputT>
-VARIANTS(StateT, float, bfloat)
-VARIANTS(InputT, float, bfloat)
-PUBLIC KERNEL(Conv1dPack)(
- device const StateT* state_in,
- device const InputT* x,
- device StateT* padded,
- constant const uint& state_stride,
- constant const uint& row_stride,
- constant const uint& suffix_len,
- constant const uint& num_channels,
- const uint channel_idx AXIS(num_channels, 32),
- const uint row_idx AXIS(state_stride + suffix_len, 1)
-) {
- const uint padded_index = row_idx * row_stride + channel_idx;
- if (row_idx < state_stride) {
- const uint state_index = channel_idx * state_stride + row_idx;
- padded[padded_index] = state_in[state_index];
- } else {
- const uint token = row_idx - state_stride;
- const uint x_index = token * row_stride + channel_idx;
- padded[padded_index] = static_cast<StateT>(x[x_index]);
- }
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(Conv1dDecode)(
- device const T* x,
- device const T* w,
- device const T* b OPTIONAL(has_bias),
- device const T* state OPTIONAL(!state_in_place),
- device T* x_out,
- device T* b_out,
- device T* c_out,
- device T* next_state,
- constant const uint& kernel_size,
- constant const uint& row_stride,
- constant const uint& state_stride,
- constant const uint& num_channels,
- constant const uint& suffix_len,
- constant const uint& inner_dim,
- constant const uint& proj_dim,
- constant const ActivationType& activation_type,
- const bool has_bias SPECIALIZE,
- const bool state_in_place SPECIALIZE,
- const uint token_idx AXIS(suffix_len, 32),
- const uint channel_idx AXIS(num_channels, 1)
-) {
- if (state_in_place) {
- state = next_state;
- }
-
- const uint x_idx = token_idx * row_stride + channel_idx;
- const uint state_offset = channel_idx * state_stride;
- const device T* w_row = w + channel_idx * kernel_size;
-
- float acc = 0.0f;
- if (has_bias) {
- acc = float(b[channel_idx]);
- }
-
- const uint state_taps = max(kernel_size - 1, 0u);
- for (uint tap = 0; tap < state_taps; ++tap) {
- const uint state_index = state_offset + tap;
- const float sample = float(state[state_index]);
- acc += float(w_row[tap]) * sample;
- }
-
- const float current = float(x[x_idx]);
- const uint current_tap_index = state_taps;
- if (current_tap_index < kernel_size) {
- acc += float(w_row[current_tap_index]) * current;
- }
-
- const T activated = activate(static_cast<T>(acc), activation_type);
- if (channel_idx < inner_dim) {
- const uint dst = token_idx * inner_dim + channel_idx;
- x_out[dst] = activated;
- } else if (channel_idx < inner_dim + proj_dim) {
- const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);
- b_out[dst] = activated;
- } else if (channel_idx < inner_dim + 2 * proj_dim) {
- const uint dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
- c_out[dst] = activated;
- }
-
- if (state_taps == 0) {
- return;
- }
-
- for (uint tap = 0; tap < state_taps - 1; ++tap) {
- const size_t dst_index = state_offset + size_t(tap);
- const size_t src_index = state_offset + size_t(tap + 1);
- next_state[dst_index] = state[src_index];
- }
-
- const uint tail_index = state_offset + state_taps - 1;
- next_state[tail_index] = static_cast<T>(current);
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(Conv1dScan)(
- device const T* padded, // (prefix+suffix, channels)
- device const T* w, // (channels, kernel)
- device const T* b OPTIONAL(has_bias), // optional (channels)
- device T* x_out, // (suffix, inner_dim)
- device T* b_out, // (suffix, proj_dim)
- device T* c_out, // (suffix, proj_dim)
- device T* state_out, // (channels, kernel-1)
- constant const uint& suffix_len,
- constant const uint& kernel_size,
- constant const uint& row_stride,
- constant const uint& state_stride,
- constant const uint& num_channels,
- constant const uint& inner_dim,
- constant const uint& proj_dim,
- constant const ActivationType& activation_type,
- const bool has_bias SPECIALIZE,
- const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 32),
- const uint channel_idx AXIS(num_channels, 1)
-) {
- const uint tap_count = max(kernel_size - 1, 0u);
- const uint state_offset = channel_idx * state_stride;
- const uint weight_offset = channel_idx * kernel_size;
- const device T* w_row = w + weight_offset;
-
- if (token_idx < suffix_len) {
- float acc = 0.0f;
- if (has_bias) {
- acc = float(b[channel_idx]);
- }
-
- for (uint tap = 0; tap < kernel_size; ++tap) {
- const uint padded_index = (token_idx + tap) * row_stride + channel_idx;
- float sample = float(padded[padded_index]);
- acc += float(w_row[tap]) * sample;
- }
-
- const T activated = activate(static_cast<T>(acc), activation_type);
- if (channel_idx < inner_dim) {
- const uint dst = token_idx * inner_dim + channel_idx;
- x_out[dst] = activated;
- } else if (channel_idx < inner_dim + proj_dim) {
- const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);
- b_out[dst] = activated;
- } else if (channel_idx < inner_dim + 2 * proj_dim) {
- const uint dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
- c_out[dst] = activated;
- }
- } else if (tap_count > 0) {
- const uint tap = token_idx - suffix_len;
- if (tap >= tap_count) {
- return;
- }
- const uint padded_index = token_idx * row_stride + channel_idx;
- const float sample = float(padded[padded_index]);
- state_out[state_offset + tap] = static_cast<T>(sample);
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu-engine/src/backends/metal/kernel/ssm/split_inproj.metal
deleted file mode 100644
index 18bd8805b..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/ssm/split_inproj.metal
+++ /dev/null
@@ -1,32 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(SplitInProj)(
- device const T* input,
- device T* conv_out,
- device T* z_out,
- device T* dt_out,
- device const T* z_bias,
- constant const uint& suffix_length,
- constant const uint& total_dim,
- constant const uint& conv_dim,
- constant const uint& inner_dim,
- constant const uint& num_heads,
- const uint row AXIS(suffix_length, 16),
- const uint col AXIS(total_dim, 16)
-) {
- const uint input_idx = row * total_dim + col;
- if (col < conv_dim) {
- const uint dst = row * conv_dim + col;
- conv_out[dst] = input[input_idx];
- } else if (col < conv_dim + inner_dim) {
- const uint dst = row * inner_dim + (col - conv_dim);
- const uint bias_idx = col - conv_dim;
- z_out[dst] = input[input_idx] + z_bias[bias_idx];
- } else if (col < conv_dim + inner_dim + num_heads) {
- const uint dst = row * num_heads + (col - conv_dim - inner_dim);
- dt_out[dst] = input[input_idx];
- }
-}
\ No newline at end of file
diff --git a/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_prefill.metal
deleted file mode 100644
index 35c065f3f..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ /dev/null
@@ -1,203 +0,0 @@
-#include <metal_stdlib>
-#include <metal_simdgroup>
-#include "../activation/activations.h"
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-
-using namespace metal;
-
-constant ushort SSM_PREFILL_MAX_STATE = 256;
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(SSDPrefill64)(
- device const T* x, // (suffix, h, dh)
- device const T* dt_raw, // (suffix, h) - raw dt values
- device const T* b, // (suffix, g, n)
- device const T* c, // (suffix, g, n)
- device const T* d, // (h)
- device const T* z, // (suffix, h, dh)
- device T* state, // (h, dh, n)
- device T* y, // (suffix, h, dh)
- constant const uint& suffix_len,
- constant const uint& group_size,
- constant const uint& state_size,
- constant const uint x_strides[3],
- constant const uint dt_strides[2],
- constant const uint cb_strides[3],
- constant const uint state_strides[3],
- constant const uint& num_heads,
- constant const uint& head_dim,
- const ThreadContext thread_context,
- const uint pair_idx GROUPS(num_heads * head_dim),
- const uint lane_idx THREADS(32)
-) {
- const uint h_idx = pair_idx / head_dim;
- const uint dh_idx = pair_idx % head_dim;
- const uint safe_group = max(group_size, 1u);
- const uint group_idx = h_idx / safe_group;
-
- const uint x_token_stride = x_strides[0];
- const uint x_head_stride = x_strides[1];
- const uint x_dim_stride = x_strides[2];
- const uint dt_token_stride = dt_strides[0];
- const uint dt_head_stride = dt_strides[1];
- const uint cb_token_stride = cb_strides[0];
- const uint cb_group_stride = cb_strides[1];
- const uint cb_state_stride = cb_strides[2];
- const uint state_head_stride = state_strides[0];
- const uint state_dim_stride = state_strides[1];
- const uint state_inner_stride = state_strides[2];
-
- const uint x_base = h_idx * x_head_stride + dh_idx * x_dim_stride;
- const uint dt_base = h_idx * dt_head_stride;
- const uint state_base = h_idx * state_head_stride + dh_idx * state_dim_stride;
- const uint cb_group_base = group_idx * cb_group_stride;
- const float d_scalar = float(d[h_idx]);
-
- const uint idx0 = lane_idx;
- const uint idx1 = lane_idx + thread_context.simdgroup_size;
-
- const uint state_idx0 = state_base + idx0 * state_inner_stride;
- const uint state_idx1 = state_base + idx1 * state_inner_stride;
- float state0 = float(state[state_idx0]);
- float state1 = float(state[state_idx1]);
-
- uint cb_idx0 = cb_group_base + idx0 * cb_state_stride;
- uint cb_idx1 = cb_group_base + idx1 * cb_state_stride;
-
- for (uint token = 0; token < suffix_len; ++token) {
- const uint x_idx = token * x_token_stride + x_base;
- const uint dt_idx = token * dt_token_stride + dt_base;
-
- const float x_val = float(x[x_idx]);
- const float decay_val = fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));
- const float gate = float(activate_silu(z[x_idx]));
- const float skip = d_scalar * x_val;
- const float dt_scaled_input = x_val;
-
- float contrib = 0.0f;
- const float new_state0 = decay_val * state0 + dt_scaled_input * float(b[cb_idx0]);
- state0 = new_state0;
- contrib += new_state0 * float(c[cb_idx0]);
- cb_idx0 += cb_token_stride;
-
- const float new_state1 = decay_val * state1 + dt_scaled_input * float(b[cb_idx1]);
- state1 = new_state1;
- contrib += new_state1 * float(c[cb_idx1]);
- cb_idx1 += cb_token_stride;
-
- float dot = simd_sum(contrib);
- if (lane_idx == 0) {
- y[x_idx] = static_cast<T>((skip + dot) * gate);
- }
- }
-
- state[state_idx0] = static_cast<T>(state0);
- state[state_idx1] = static_cast<T>(state1);
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(SSDPrefill)(
- device const T* x, // (suffix, h, dh)
- device const T* dt_raw, // (suffix, h) - raw dt values
- device const T* b, // (suffix, g, n)
- device const T* c, // (suffix, g, n)
- device const T* d, // (h)
- device const T* z, // (suffix, h, dh)
- device T* state, // (h, dh, n)
- device T* y, // (suffix, h, dh)
- constant const uint& suffix_len,
- constant const uint& group_size,
- constant const uint& state_size,
- constant const uint x_strides[3],
- constant const uint dt_strides[2],
- constant const uint cb_strides[3],
- constant const uint state_strides[3],
- constant const uint& num_heads,
- constant const uint& head_dim,
- const ThreadContext thread_context,
- const uint pair_idx GROUPS(num_heads * head_dim),
- const uint lane_idx THREADS(32)
-) {
- const int state_dim = state_size;
- const uint h_idx = pair_idx / head_dim;
- const uint dh_idx = pair_idx % head_dim;
- const uint safe_group = max(group_size, 1u);
- const uint group_idx = h_idx / safe_group;
-
- const uint x_token_stride = x_strides[0];
- const uint x_head_stride = x_strides[1];
- const uint x_dim_stride = x_strides[2];
- const uint dt_token_stride = dt_strides[0];
- const uint dt_head_stride = dt_strides[1];
- const uint cb_token_stride = cb_strides[0];
- const uint cb_group_stride = cb_strides[1];
- const uint cb_state_stride = cb_strides[2];
- const uint state_head_stride = state_strides[0];
- const uint state_dim_stride = state_strides[1];
- const uint state_inner_stride = state_strides[2];
-
- const uint x_base = h_idx * x_head_stride + dh_idx * x_dim_stride;
- const uint dt_base = h_idx * dt_head_stride;
- const uint state_base = h_idx * state_head_stride + dh_idx * state_dim_stride;
- const uint cb_group_base = group_idx * cb_group_stride;
- const float d_scalar = float(d[h_idx]);
-
- const int max_chunks = SSM_PREFILL_MAX_STATE / thread_context.simdgroup_size;
- const int chunk_count = (state_dim + int(thread_context.simdgroup_size) - 1) / int(thread_context.simdgroup_size);
- if (state_dim <= 0 || chunk_count > max_chunks) {
- return;
- }
-
- thread float lane_states[SSM_PREFILL_MAX_STATE / 32];
-
-#pragma unroll
- for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);
- if (idx < state_dim) {
- const uint state_idx = state_base + idx * state_inner_stride;
- lane_states[chunk] = float(state[state_idx]);
- } else {
- lane_states[chunk] = 0.0f;
- }
- }
-
- for (uint token = 0; token < suffix_len; ++token) {
- const uint x_idx = token * x_token_stride + x_base;
- const uint dt_idx = token * dt_token_stride + dt_base;
-
- const float x_val = float(x[x_idx]);
- const float decay_val = fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));
- const float gate = float(activate_silu(z[x_idx]));
- const float skip = d_scalar * x_val;
- const float dt_scaled_input = x_val;
-
- float contrib_sum = 0.0f;
-#pragma unroll
- for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);
- if (idx < state_dim) {
- const uint cb_idx = cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;
- const float new_state = decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);
- lane_states[chunk] = new_state;
- contrib_sum += new_state * float(c[cb_idx]);
- }
- }
-
- float dot = simd_sum(contrib_sum);
- if (lane_idx == 0) {
- y[x_idx] = static_cast<T>((skip + dot) * gate);
- }
- }
-
-#pragma unroll
- for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);
- if (idx < state_dim) {
- const uint state_idx = state_base + uint(idx) * state_inner_stride;
- state[state_idx] = static_cast<T>(lane_states[chunk]);
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_update.metal
deleted file mode 100644
index 3e6156b6a..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/ssm/ssd_update.metal
+++ /dev/null
@@ -1,69 +0,0 @@
-#include <metal_stdlib>
-#include "../activation/activations.h"
-#include "../common/dsl.h"
-
-using namespace metal;
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(SSDUpdate)(
- // Input
- device const T* x, // (b, h, dh)
- device const T* dt_raw, // (b, h)
- device const T* b, // (b, g, n)
- device const T* c, // (b, g, n)
- device const T* d, // (h)
- device const T* z, // (b, d)
- device const T* state OPTIONAL(!state_in_place), // (b, h, dh, n)
- device T* y,
- device T* next_state,
- // Parameters
- constant const uint& group_size, // h = group_size * g
- constant const uint& state_size,
- // Strides
- constant const uint x_strides[3],
- constant const uint dt_strides[2],
- constant const uint cb_strides[3],
- constant const uint state_strides[4],
- // Threads count
- constant const uint& b_size,
- constant const uint& h_size,
- constant const uint& dh_size,
- const bool state_in_place SPECIALIZE,
- // Grid
- const uint b_idx AXIS(b_size, 32),
- const uint h_idx AXIS(h_size, 32),
- const uint dh_idx AXIS(dh_size, 1)
-) {
- if (state_in_place) {
- state = next_state;
- }
-
- const uint cb_start_idx = b_idx * cb_strides[0] + (h_idx / group_size) * cb_strides[1];
- const uint x_idx = b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];
- const uint dt_idx = b_idx * dt_strides[0] + h_idx * dt_strides[1];
- const uint state_start_idx = b_idx * state_strides[0] + h_idx * state_strides[1] + dh_idx * state_strides[2];
-
- // load data
- T this_x = x[x_idx];
- T dt_raw_val = dt_raw[dt_idx];
- T this_dt = activate_softplus(dt_raw_val);
- T this_decay = static_cast<T>(fast::exp(-float(this_dt)));
- T this_d = d[h_idx];
- T this_z = z[x_idx];
- this_z = activate_silu(this_z);
- T dt_scaled_input = this_x;
-
- T temp = T(0);
-#pragma unroll
- for (uint i = 0; i < state_size; ++i) {
- uint cb_idx = cb_start_idx + i;
- uint state_idx = state_start_idx + i;
- T this_new_state = state[state_idx] * this_decay + b[cb_idx] * dt_scaled_input;
- next_state[state_idx] = this_new_state;
- temp = temp + this_new_state * c[cb_idx];
- }
- temp = temp + this_d * this_x;
- temp = temp * this_z;
- y[x_idx] = temp;
-}
\ No newline at end of file
diff --git a/crates/uzu-engine/src/config/token_mixer/mamba2.rs b/crates/uzu-engine/src/config/token_mixer/mamba2.rs
deleted file mode 100644
index e20d3558a..000000000
--- a/crates/uzu-engine/src/config/token_mixer/mamba2.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-use uzu_engine_macros::uzu_config;
-
-use crate::config::{
- activation::AnyActivation, linear::LinearConfig, token_mixer::convolutions::SeparableCausalConvConfig,
-};
-
-#[uzu_config(super::TokenMixerConfig)]
-pub struct Mamba2Config {
- pub in_projection_config: LinearConfig,
- pub out_projection_config: LinearConfig,
- pub conv_config: SeparableCausalConvConfig,
- pub activation: AnyActivation,
-
- pub kernel_size: u32,
- pub num_heads: u32,
- pub num_groups: u32,
- pub head_dim: u32,
- pub state_dim: u32,
- pub has_in_biases: bool,
- pub has_out_biases: bool,
-}
diff --git a/crates/uzu-engine/src/config/token_mixer/mod.rs b/crates/uzu-engine/src/config/token_mixer/mod.rs
index 9faa3c20e..fff58ebd6 100644
--- a/crates/uzu-engine/src/config/token_mixer/mod.rs
+++ b/crates/uzu-engine/src/config/token_mixer/mod.rs
@@ -3,13 +3,7 @@ use uzu_engine_macros::uzu_config_abstract;
pub mod attention;
pub mod convolutions;
pub mod delta_net;
-pub mod mamba2;
pub mod short_conv;
-#[uzu_config_abstract(
- attention::AttentionConfig,
- delta_net::DeltaNetConfig,
- mamba2::Mamba2Config,
- short_conv::ShortConvConfig
-)]
+#[uzu_config_abstract(attention::AttentionConfig, delta_net::DeltaNetConfig, short_conv::ShortConvConfig)]
pub struct TokenMixerConfig;
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
deleted file mode 100644
index 8605e17d0..000000000
--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
+++ /dev/null
@@ -1,436 +0,0 @@
-use derive_more::Debug;
-use thiserror::Error;
-
-use crate::{
- backends::common::{
- Backend, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::activation_type::ActivationType,
- kernel::{
- Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDPrefill64Kernel, SSDPrefillKernel,
- SSDUpdateKernel, SplitInProjKernel,
- },
- },
- config::token_mixer::mamba2::Mamba2Config,
- data_type::DataType,
- encodable_block::{
- batch_topology::BatchTopology,
- linear::{Linear, LinearBlockError},
- mixer::{Mixer, MixerEncodeError, MixerState, attention::rope::PrecalculatedRoPE},
- },
- safetensors::{ParameterLoaderError, ParameterTree},
- utils::{maybe_mut::MaybeMut, size_for_shape::size_for_shape},
-};
-
-const INNER_DATA_TYPE: DataType = DataType::F32;
-
-pub struct Mamba2State<B: Backend> {
- conv_state: B::GlobalBuffer,
- ssm_state: B::GlobalBuffer,
- suffix_length: Option<u32>,
-}
-
-impl<B: Backend> MixerState<B> for Mamba2State<B> {
- fn prepare(
- &mut self,
- _context_length: u32,
- _suffix_length: u32,
- ) -> Result<(), B::Error> {
- Ok(())
- }
-
- fn encode_accept(
- &mut self,
- accepted_indices: &[u32],
- _command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), <B as Backend>::Error> {
- assert!(self.suffix_length.take() == Some(*accepted_indices.last().unwrap() + 1));
- Ok(())
- }
-}
-
-enum Mamba2SSDPrefillVariant<B: Backend> {
- Universal(<B::Kernels as Kernels>::SSDPrefillKernel),
- Special64(<B::Kernels as Kernels>::SSDPrefill64Kernel),
-}
-
-pub struct Mamba2<B: Backend> {
- kernel_size: u32,
- num_heads: u32,
- num_groups: u32,
- head_dim: u32,
- state_dim: u32,
- inner_dim: u32,
- conv_dim: u32,
- activation_type: ActivationType,
- in_projection: Box<dyn Linear<B>>,
- gate_bias: B::GlobalBuffer,
- split_inproj: <B::Kernels as Kernels>::SplitInProjKernel,
- conv_weight: B::GlobalBuffer,
- conv_bias: Option<B::GlobalBuffer>,
- conv_decode: <B::Kernels as Kernels>::Conv1dDecodeKernel,
- conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,
- conv_scan: <B::Kernels as Kernels>::Conv1dScanKernel,
- skip_connection_weight: B::GlobalBuffer,
- ssd_update: <B::Kernels as Kernels>::SSDUpdateKernel,
- ssd_prefill: Mamba2SSDPrefillVariant<B>,
- out_projection: Box<dyn Linear<B>>,
-}
-
-#[derive(Debug, Error)]
-pub enum Mamba2NewError<B: Backend> {
- #[error("Backend error: {0}")]
- Backend(#[source] B::Error),
- #[error("Parameter loader error: {0}")]
- ParameterLoader(#[from] ParameterLoaderError<B>),
- #[error("Linear error: {0}")]
- Linear(#[from] LinearBlockError<B>),
- #[error("Unsupported configuration: {0}")]
- UnsupportedConfiguration(String),
-}
-
-impl<B: Backend> Mamba2<B> {
- pub fn new(
- hidden_dim: u32,
- outer_data_type: DataType,
- config: &Mamba2Config,
- parameter_tree: &ParameterTree<B>,
- context: &B::Context,
- ) -> Result<(Self, Option<B::GlobalBuffer>), Mamba2NewError<B>> {
- let kernel_size = config.kernel_size;
- let num_heads = config.num_heads;
- let num_groups = config.num_groups;
- let head_dim = config.head_dim;
- let state_dim = config.state_dim;
-
- let inner_dim = num_heads * head_dim;
- let conv_dim = inner_dim + 2 * num_groups * state_dim;
-
- let activation_type = config.activation.act_type();
-
- if kernel_size <= 1 {
- return Err(Mamba2NewError::UnsupportedConfiguration(format!("kernel_size = {kernel_size} (must be > 1)")));
- }
-
- let (in_projection, in_projection_input_hadamard_factors) =
- <dyn Linear<B>>::new_with_input_rht_mixed_precision(
- hidden_dim,
- conv_dim + inner_dim + num_heads,
- outer_data_type,
- outer_data_type,
- INNER_DATA_TYPE,
- config.has_in_biases,
- &parameter_tree.subtree("in_projection"),
- context,
- )?;
-
- let gate_bias = parameter_tree.read_buffer("gate_bias", &[inner_dim], INNER_DATA_TYPE)?;
- let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, INNER_DATA_TYPE)
- .map_err(Mamba2NewError::Backend)?;
-
- let conv_config = &config.conv_config;
- let conv_tree = parameter_tree.subtree("conv");
-
- let conv_weight = conv_tree.read_buffer("weights", &[conv_dim, kernel_size], INNER_DATA_TYPE)?;
- let conv_bias = if conv_config.has_biases {
- Some(conv_tree.read_buffer("biases", &[conv_dim], INNER_DATA_TYPE)?)
- } else {
- None
- };
- let conv_decode =
- <B::Kernels as Kernels>::Conv1dDecodeKernel::new(context, INNER_DATA_TYPE, conv_config.has_biases, true)
- .map_err(Mamba2NewError::Backend)?;
- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, INNER_DATA_TYPE, INNER_DATA_TYPE)
- .map_err(Mamba2NewError::Backend)?;
- let conv_scan =
- <B::Kernels as Kernels>::Conv1dScanKernel::new(context, INNER_DATA_TYPE, conv_config.has_biases)
- .map_err(Mamba2NewError::Backend)?;
-
- let skip_connection_weight =
- parameter_tree.read_buffer("skip_connection_weight", &[num_heads], INNER_DATA_TYPE)?;
- let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, INNER_DATA_TYPE, true)
- .map_err(Mamba2NewError::Backend)?;
-
- let ssd_prefill = if state_dim == 64 {
- Mamba2SSDPrefillVariant::Special64(
- <B::Kernels as Kernels>::SSDPrefill64Kernel::new(context, INNER_DATA_TYPE)
- .map_err(Mamba2NewError::Backend)?,
- )
- } else {
- Mamba2SSDPrefillVariant::Universal(
- <B::Kernels as Kernels>::SSDPrefillKernel::new(context, INNER_DATA_TYPE)
- .map_err(Mamba2NewError::Backend)?,
- )
- };
-
- let out_projection = <dyn Linear<B>>::new_mixed_precision(
- inner_dim,
- hidden_dim,
- outer_data_type,
- INNER_DATA_TYPE,
- outer_data_type,
- config.has_out_biases,
- &parameter_tree.subtree("out_projection"),
- context,
- )?;
-
- Ok((
- Self {
- kernel_size,
- num_heads,
- num_groups,
- head_dim,
- state_dim,
- inner_dim,
- conv_dim,
- activation_type,
- in_projection,
- gate_bias,
- split_inproj,
- conv_weight,
- conv_bias,
- conv_decode,
- conv_pack,
- conv_scan,
- skip_connection_weight,
- ssd_update,
- ssd_prefill,
- out_projection,
- },
- in_projection_input_hadamard_factors,
- ))
- }
-}
-
-impl<B: Backend> Mixer<B> for Mamba2<B> {
- fn speculation_supported(&self) -> bool {
- false
- }
-
- fn max_context_length(&self) -> Option<u32> {
- None
- }
-
- fn create_empty_state(
- &self,
- _max_context_length: Option<u32>,
- context: &B::Context,
- ) -> Result<Box<dyn MixerState<B>>, B::Error> {
- let mut conv_state =
- context.create_buffer(size_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE))?;
-
- let mut ssm_state =
- context.create_buffer(size_for_shape(&[self.num_heads, self.head_dim, self.state_dim], INNER_DATA_TYPE))?;
-
- let mut zero_command_buffer = context.create_command_buffer(None, None)?;
- zero_command_buffer.encode_fill(&mut conv_state, 0);
- zero_command_buffer.encode_fill(&mut ssm_state, 0);
- zero_command_buffer.end_encoding().submit().wait_until_completed()?;
-
- Ok(Box::new(Mamba2State {
- conv_state,
- ssm_state,
- suffix_length: None,
- }))
- }
-
- fn encode(
- &self,
- hidden: B::ScratchBuffer,
- precalculated_rope: Option<&PrecalculatedRoPE<B>>,
- batch_dim: &BatchTopology,
- state: Option<MaybeMut<dyn MixerState<B>>>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {
- let command_buffer = &mut *command_buffer.debug_group("mamba2");
-
- assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");
-
- if !batch_dim.full_accept() {
- panic!("mamba2 doesn't support speculation");
- }
-
- let state = state.expect("mamba2 requires state");
- let state = state.downcast::<Mamba2State<B>>().expect("incorrect type of mamba2 state");
- let MaybeMut::Mut(state) = state else {
- panic!("mamba2 doesn't support immutable state");
- };
-
- assert!(state.suffix_length.is_none(), "mamba2 called with state with unaccepted tokens");
-
- let in_projected = self.in_projection.encode(hidden, batch_dim.size(), command_buffer)?;
-
- let mut conv_inputs = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.conv_dim], INNER_DATA_TYPE)
- .map_err(MixerEncodeError::Backend)?;
- let mut gate = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)
- .map_err(MixerEncodeError::Backend)?;
- let mut time_step = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)
- .map_err(MixerEncodeError::Backend)?;
- self.split_inproj.encode(
- &in_projected,
- &mut conv_inputs,
- &mut gate,
- &mut time_step,
- &self.gate_bias,
- batch_dim.size(),
- self.conv_dim + self.inner_dim + self.num_heads,
- self.conv_dim,
- self.inner_dim,
- self.num_heads,
- command_buffer,
- );
-
- let mut conv_x = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads, self.head_dim], INNER_DATA_TYPE)
- .map_err(MixerEncodeError::Backend)?;
- let mut state_b = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)
- .map_err(MixerEncodeError::Backend)?;
- let mut state_c = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_groups, self.state_dim], INNER_DATA_TYPE)
- .map_err(MixerEncodeError::Backend)?;
- let state_stride = self.kernel_size - 1;
- if batch_dim.size() == 1 {
- self.conv_decode.encode(
- &conv_inputs,
- &self.conv_weight,
- self.conv_bias.as_ref(),
- None::<&B::GlobalBuffer>,
- &mut conv_x,
- &mut state_b,
- &mut state_c,
- &mut state.conv_state,
- self.kernel_size,
- self.conv_dim,
- state_stride,
- self.conv_dim,
- batch_dim.size(),
- self.inner_dim,
- self.num_groups * self.state_dim,
- self.activation_type,
- command_buffer,
- );
- } else {
- let mut padded = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size() + state_stride, self.conv_dim], INNER_DATA_TYPE)
- .map_err(MixerEncodeError::Backend)?;
- self.conv_pack.encode(
- &state.conv_state,
- &conv_inputs,
- &mut padded,
- state_stride,
- self.conv_dim,
- batch_dim.size(),
- self.conv_dim,
- command_buffer,
- );
- self.conv_scan.encode(
- &padded,
- &self.conv_weight,
- self.conv_bias.as_ref(),
- &mut conv_x,
- &mut state_b,
- &mut state_c,
- &mut state.conv_state,
- batch_dim.size(),
- self.kernel_size,
- self.conv_dim,
- state_stride,
- self.conv_dim,
- self.inner_dim,
- self.num_groups * self.state_dim,
- self.activation_type,
- command_buffer,
- );
- }
-
- let mut ssd_output = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.inner_dim], INNER_DATA_TYPE)
- .map_err(MixerEncodeError::Backend)?;
- let x_strides = [self.num_heads * self.head_dim, self.head_dim, 1];
- let dt_strides = [self.num_heads, 1];
- let cb_strides = [self.num_groups * self.state_dim, self.state_dim, 1];
- let group_size = self.num_heads / self.num_groups;
- if batch_dim.size() == 1 {
- let state_strides =
- [self.num_heads * self.head_dim * self.state_dim, self.head_dim * self.state_dim, self.state_dim, 1];
- self.ssd_update.encode(
- &conv_x,
- &time_step,
- &state_b,
- &state_c,
- &self.skip_connection_weight,
- &gate,
- None::<&B::GlobalBuffer>,
- &mut ssd_output,
- &mut state.ssm_state,
- group_size,
- self.state_dim,
- &x_strides,
- &dt_strides,
- &cb_strides,
- &state_strides,
- batch_dim.size(),
- self.num_heads,
- self.head_dim,
- command_buffer,
- );
- } else {
- let state_strides = [self.head_dim * self.state_dim, self.state_dim, 1];
- match &self.ssd_prefill {
- Mamba2SSDPrefillVariant::Universal(ssd_prefill) => ssd_prefill.encode(
- &conv_x,
- &time_step,
- &state_b,
- &state_c,
- &self.skip_connection_weight,
- &gate,
- &mut state.ssm_state,
- &mut ssd_output,
- batch_dim.size(),
- group_size,
- self.state_dim,
- &x_strides,
- &dt_strides,
- &cb_strides,
- &state_strides,
- self.num_heads,
- self.head_dim,
- command_buffer,
- ),
- Mamba2SSDPrefillVariant::Special64(ssd_prefill) => ssd_prefill.encode(
- &conv_x,
- &time_step,
- &state_b,
- &state_c,
- &self.skip_connection_weight,
- &gate,
- &mut state.ssm_state,
- &mut ssd_output,
- batch_dim.size(),
- group_size,
- self.state_dim,
- &x_strides,
- &dt_strides,
- &cb_strides,
- &state_strides,
- self.num_heads,
- self.head_dim,
- command_buffer,
- ),
- }
- }
-
- state.suffix_length = Some(batch_dim.size());
-
- let output = self.out_projection.encode(ssd_output, batch_dim.size(), command_buffer)?;
-
- Ok(output)
- }
-}
-
-#[cfg(test)]
-#[path = "../../../unit/encodable_block/mamba_mixer/ssd_prefill_test.rs"]
-mod tests;
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
index dc1061e44..80b81775e 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
@@ -13,7 +13,6 @@ use crate::{
mixer::{
attention::{Attention, AttentionEncodeError, AttentionNewError, rope::PrecalculatedRoPE},
delta_net::{DeltaNet, DeltaNetNewError},
- mamba2::{Mamba2, Mamba2NewError},
short_conv::{ShortConv, ShortConvNewError},
},
},
@@ -23,7 +22,6 @@ use crate::{
pub mod attention;
pub mod delta_net;
-pub mod mamba2;
pub mod short_conv;
pub trait MixerState<B: Backend>: Any + Send {
@@ -86,8 +84,6 @@ pub enum MixerNewError<B: Backend> {
Attention(#[from] AttentionNewError<B>),
#[error("Delta net mixer error: {0}")]
DeltaNet(#[from] DeltaNetNewError<B>),
- #[error("Mamba2 mixer error: {0}")]
- Mamba2(#[from] Mamba2NewError<B>),
#[error("Short conv mixer error: {0}")]
ShortConv(#[from] ShortConvNewError<B>),
}
@@ -116,14 +112,6 @@ impl<B: Backend> dyn Mixer<B> {
Ok((Box::new(delta_net), in_projection_input_hadamard_factors))
},
- AnyTokenMixerConfig::Mamba2Config(config) => {
- assert!(rope_config.is_none(), "unexpected rope for mamba2 mixer");
-
- let (mamba2, in_projection_input_hadamard_factors) =
- Mamba2::new(hidden_dim, data_type, config, parameter_tree, context)?;
-
- Ok((Box::new(mamba2), in_projection_input_hadamard_factors))
- },
AnyTokenMixerConfig::ShortConvConfig(config) => {
assert!(rope_config.is_none(), "unexpected rope for short conv mixer");
diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rs
index 91cc482b1..e1865a8b0 100644
--- a/crates/uzu-engine/src/engine/language_model/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/mod.rs
@@ -137,7 +137,7 @@ impl<B: Backend> LanguageModel<B> {
// Currenlty true for all mixers in uzu:
// - full attention uses sparse if it's available to make max context free until it's actually used
// - sliding window attention is bound, usually well below the recommended max context size on non-sparse (but can be made to use sparse if we care about it enough)
- // - short conv/mamba2/delta net are constant state size
+ // - short conv/delta net are constant state size
max_context_length
} else if let Some(max_context_length) = max_context_length {
// If sparse buffers aren't supported and model has finite maximum context length we assume that kv cache is expensive enough that we should probably clamp it to
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rs
similarity index 100%
rename from crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rs
rename to crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rs
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/mod.rs
index adcd93b69..25e93f66c 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/mod.rs
@@ -1,2 +1,3 @@
+mod conv_pack_test;
mod delta_net_test;
mod tree_verify;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/mod.rs
index 72471df67..a6841562c 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/mod.rs
@@ -11,7 +11,6 @@ mod pooling;
mod radix_top_k_small_test;
mod separable_causal_conv_test;
mod short_conv;
-mod ssm;
mod tensor_add_bias_test;
mod tensor_add_scale_test;
mod trellis_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/mod.rs
deleted file mode 100644
index f704533c4..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/mod.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-mod ssm_conv1d_decode_test;
-mod ssm_conv1d_pack_test;
-mod ssm_conv1d_scan_test;
-mod ssm_prefill_test;
-mod ssm_split_inproj_test;
-mod ssm_update_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs
deleted file mode 100644
index 5406e67ab..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs
+++ /dev/null
@@ -1,338 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::activation_type::ActivationType, kernel::Conv1dDecodeKernel,
- },
- cpu::Cpu,
- },
- data_type::{DataType, DataTypeRepr},
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
- },
-};
-
-struct Input<T: DataTypeRepr + Float> {
- x: Box<[T]>,
- w: Box<[T]>,
- b: Option<Box<[T]>>,
- state: Box<[T]>,
- kernel_size: u32,
- row_stride: u32,
- state_stride: u32,
- num_channels: u32,
- suffix_len: u32,
- inner_dim: u32,
- proj_dim: u32,
- activation_type: ActivationType,
- state_in_place: bool,
-}
-
-struct Output<T: DataTypeRepr + Float> {
- x_out: Vec<T>,
- b_out: Vec<T>,
- c_out: Vec<T>,
- next_state: Vec<T>,
-}
-
-fn get_input<T: DataTypeRepr + Float>(
- kernel_size: u32,
- num_channels: u32,
- inner_dim: u32,
- proj_dim: u32,
- suffix_len: u32,
- has_bias: bool,
- state_in_place: bool,
- activation_type: ActivationType,
-) -> Input<T> {
- let row_stride = num_channels;
- let state_stride = kernel_size.saturating_sub(1);
- let state_size = num_channels as usize * state_stride as usize;
- let x_size = suffix_len as usize * row_stride as usize;
- let w_size = num_channels as usize * kernel_size as usize;
-
- let x: Vec<T> = (0..x_size).map(|i| T::from(0.1 * ((i as f64 % 7.0) - 3.0)).unwrap()).collect();
- let w: Vec<T> = (0..w_size).map(|i| T::from(0.05 * ((i as f64 % 5.0) - 2.0)).unwrap()).collect();
- let b = if has_bias {
- Some(
- (0..num_channels as usize)
- .map(|i| T::from(0.01 * (i as f64 + 1.0)).unwrap())
- .collect::<Vec<T>>()
- .into_boxed_slice(),
- )
- } else {
- None
- };
- let state: Vec<T> = (0..state_size).map(|i| T::from(0.02 * ((i as f64 % 9.0) - 4.0)).unwrap()).collect();
-
- Input {
- x: x.into_boxed_slice(),
- w: w.into_boxed_slice(),
- b,
- state: state.into_boxed_slice(),
- kernel_size,
- row_stride,
- state_stride,
- num_channels,
- suffix_len,
- inner_dim,
- proj_dim,
- activation_type,
- state_in_place,
- }
-}
-
-fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
- let context = B::Context::new().expect("Failed to create Context");
- let has_bias = input.b.is_some();
- let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dDecodeKernel::new(
- &context,
- T::data_type(),
- has_bias,
- input.state_in_place,
- )
- .expect("Failed to create Conv1dDecodeKernel");
-
- let x_out_size = input.suffix_len as usize * input.inner_dim as usize;
- let b_out_size = input.suffix_len as usize * input.proj_dim as usize;
- let c_out_size = input.suffix_len as usize * input.proj_dim as usize;
- let state_size = input.num_channels as usize * input.state_stride as usize;
-
- let x = create_buffer_with_data::<B, T>(&context, &input.x);
- let w = create_buffer_with_data::<B, T>(&context, &input.w);
- let b = input.b.as_ref().map(|b| create_buffer_with_data::<B, T>(&context, b));
-
- let mut x_out = create_buffer::<B, T>(&context, x_out_size);
- let mut b_out = create_buffer::<B, T>(&context, b_out_size);
- let mut c_out = create_buffer::<B, T>(&context, c_out_size);
-
- if input.state_in_place {
- let mut next_state = create_buffer_with_data::<B, T>(&context, &input.state);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- kernel.encode(
- &x,
- &w,
- b.as_ref(),
- None::<&B::GlobalBuffer>,
- &mut x_out,
- &mut b_out,
- &mut c_out,
- &mut next_state,
- input.kernel_size,
- input.row_stride,
- input.state_stride,
- input.num_channels,
- input.suffix_len,
- input.inner_dim,
- input.proj_dim,
- input.activation_type,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- Output {
- x_out: buffer_to_vec(&x_out),
- b_out: buffer_to_vec(&b_out),
- c_out: buffer_to_vec(&c_out),
- next_state: buffer_to_vec(&next_state),
- }
- } else {
- let state = create_buffer_with_data::<B, T>(&context, &input.state);
- let mut next_state = create_buffer::<B, T>(&context, state_size);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- kernel.encode(
- &x,
- &w,
- b.as_ref(),
- Some(&state),
- &mut x_out,
- &mut b_out,
- &mut c_out,
- &mut next_state,
- input.kernel_size,
- input.row_stride,
- input.state_stride,
- input.num_channels,
- input.suffix_len,
- input.inner_dim,
- input.proj_dim,
- input.activation_type,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- Output {
- x_out: buffer_to_vec(&x_out),
- b_out: buffer_to_vec(&b_out),
- c_out: buffer_to_vec(&c_out),
- next_state: buffer_to_vec(&next_state),
- }
- }
-}
-
-fn get_test_data<T: DataTypeRepr + Float>(
- kernel_size: u32,
- num_channels: u32,
- inner_dim: u32,
- proj_dim: u32,
- suffix_len: u32,
- has_bias: bool,
- state_in_place: bool,
- activation_type: ActivationType,
-) -> (Input<T>, Output<T>) {
- let input = get_input::<T>(
- kernel_size,
- num_channels,
- inner_dim,
- proj_dim,
- suffix_len,
- has_bias,
- state_in_place,
- activation_type,
- );
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
-}
-
-fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
- input: &Input<T>,
- expected: &Output<T>,
- label: &str,
-) {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 1e-2f32
- } else {
- 1e-5
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
- let backend_name = std::any::type_name::<B>();
- let type_name = std::any::type_name::<T>();
-
- assert_eq_float::<T>(
- &expected.x_out,
- &output.x_out,
- eps,
- &format!("Conv1dDecode x_out {backend_name} {label} (type={type_name})"),
- );
- assert_eq_float::<T>(
- &expected.b_out,
- &output.b_out,
- eps,
- &format!("Conv1dDecode b_out {backend_name} {label} (type={type_name})"),
- );
- assert_eq_float::<T>(
- &expected.c_out,
- &output.c_out,
- eps,
- &format!("Conv1dDecode c_out {backend_name} {label} (type={type_name})"),
- );
- assert_eq_float::<T>(
- &expected.next_state,
- &output.next_state,
- eps,
- &format!("Conv1dDecode next_state {backend_name} {label} (type={type_name})"),
- );
- });
-}
-
-// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4
-fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
- for state_in_place in [true, false] {
- for has_bias in [true, false] {
- let label = format!("basic(bias={has_bias},in_place={state_in_place})");
- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, has_bias, state_in_place, ActivationType::SILU);
- test_internal(&input, &expected, &label);
- }
- }
-}
-
-// kernel_size=2 (state_stride=1, minimal state)
-fn test_small_kernel<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 1, true, true, ActivationType::SILU);
- test_internal(&input, &expected, "small_kernel");
-}
-
-// Larger dimensions closer to real usage
-fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 1, true, true, ActivationType::SILU);
- test_internal(&input, &expected, "large");
-}
-
-fn test_identity_activation<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, false, true, ActivationType::IDENTITY);
- test_internal(&input, &expected, "identity_activation");
-}
-
-fn test_gelu_activation<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::GELUApprox);
- test_internal(&input, &expected, "gelu_activation");
-}
-
-// f32
-#[uzu_test]
-fn test_basic_f32() {
- test_basic::<f32>();
-}
-
-#[uzu_test]
-fn test_small_kernel_f32() {
- test_small_kernel::<f32>();
-}
-
-#[uzu_test]
-fn test_large_f32() {
- test_large::<f32>();
-}
-
-#[uzu_test]
-fn test_identity_activation_f32() {
- test_identity_activation::<f32>();
-}
-
-#[uzu_test]
-fn test_gelu_activation_f32() {
- test_gelu_activation::<f32>();
-}
-
-// f16
-#[uzu_test]
-fn test_basic_f16() {
- test_basic::<f16>();
-}
-
-#[uzu_test]
-fn test_small_kernel_f16() {
- test_small_kernel::<f16>();
-}
-
-#[uzu_test]
-fn test_large_f16() {
- test_large::<f16>();
-}
-
-// bf16
-#[uzu_test]
-fn test_basic_bf16() {
- test_basic::<bf16>();
-}
-
-#[uzu_test]
-fn test_small_kernel_bf16() {
- test_small_kernel::<bf16>();
-}
-
-#[uzu_test]
-fn test_large_bf16() {
- test_large::<bf16>();
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs
deleted file mode 100644
index 22f221edc..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs
+++ /dev/null
@@ -1,321 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::activation_type::ActivationType, kernel::Conv1dScanKernel,
- },
- cpu::Cpu,
- },
- data_type::{DataType, DataTypeRepr},
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
- },
-};
-
-struct Input<T: DataTypeRepr + Float> {
- padded: Box<[T]>,
- w: Box<[T]>,
- b: Option<Box<[T]>>,
- suffix_len: u32,
- kernel_size: u32,
- row_stride: u32,
- state_stride: u32,
- num_channels: u32,
- inner_dim: u32,
- proj_dim: u32,
- activation_type: ActivationType,
-}
-
-struct Output<T: DataTypeRepr + Float> {
- x_out: Vec<T>,
- b_out: Vec<T>,
- c_out: Vec<T>,
- state_out: Vec<T>,
-}
-
-fn get_input<T: DataTypeRepr + Float>(
- kernel_size: u32,
- num_channels: u32,
- inner_dim: u32,
- proj_dim: u32,
- suffix_len: u32,
- has_bias: bool,
- activation_type: ActivationType,
-) -> Input<T> {
- let row_stride = num_channels;
- let state_stride = kernel_size.saturating_sub(1);
- let total_rows = (suffix_len + state_stride) as usize;
- let padded_size = total_rows * row_stride as usize;
- let w_size = num_channels as usize * kernel_size as usize;
-
- let padded: Vec<T> = (0..padded_size).map(|i| T::from(0.1 * ((i as f64 % 11.0) - 5.0)).unwrap()).collect();
- let w: Vec<T> = (0..w_size).map(|i| T::from(0.05 * ((i as f64 % 5.0) - 2.0)).unwrap()).collect();
- let b = if has_bias {
- Some(
- (0..num_channels as usize)
- .map(|i| T::from(0.01 * (i as f64 + 1.0)).unwrap())
- .collect::<Vec<T>>()
- .into_boxed_slice(),
- )
- } else {
- None
- };
-
- Input {
- padded: padded.into_boxed_slice(),
- w: w.into_boxed_slice(),
- b,
- suffix_len,
- kernel_size,
- row_stride,
- state_stride,
- num_channels,
- inner_dim,
- proj_dim,
- activation_type,
- }
-}
-
-fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
- let context = B::Context::new().expect("Failed to create Context");
- let has_bias = input.b.is_some();
- let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&context, T::data_type(), has_bias)
- .expect("Failed to create Conv1dScanKernel");
-
- let x_out_size = input.suffix_len as usize * input.inner_dim as usize;
- let b_out_size = input.suffix_len as usize * input.proj_dim as usize;
- let c_out_size = input.suffix_len as usize * input.proj_dim as usize;
- let state_size = input.num_channels as usize * input.state_stride as usize;
-
- let padded = create_buffer_with_data::<B, T>(&context, &input.padded);
- let w = create_buffer_with_data::<B, T>(&context, &input.w);
- let b = input.b.as_ref().map(|b| create_buffer_with_data::<B, T>(&context, b));
-
- let mut x_out = create_buffer::<B, T>(&context, x_out_size);
- let mut b_out = create_buffer::<B, T>(&context, b_out_size);
- let mut c_out = create_buffer::<B, T>(&context, c_out_size);
- let mut state_out = create_buffer::<B, T>(&context, state_size);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- kernel.encode(
- &padded,
- &w,
- b.as_ref(),
- &mut x_out,
- &mut b_out,
- &mut c_out,
- &mut state_out,
- input.suffix_len,
- input.kernel_size,
- input.row_stride,
- input.state_stride,
- input.num_channels,
- input.inner_dim,
- input.proj_dim,
- input.activation_type,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- Output {
- x_out: buffer_to_vec(&x_out),
- b_out: buffer_to_vec(&b_out),
- c_out: buffer_to_vec(&c_out),
- state_out: buffer_to_vec(&state_out),
- }
-}
-
-fn get_test_data<T: DataTypeRepr + Float>(
- kernel_size: u32,
- num_channels: u32,
- inner_dim: u32,
- proj_dim: u32,
- suffix_len: u32,
- has_bias: bool,
- activation_type: ActivationType,
-) -> (Input<T>, Output<T>) {
- let input = get_input::<T>(kernel_size, num_channels, inner_dim, proj_dim, suffix_len, has_bias, activation_type);
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
-}
-
-fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
- input: &Input<T>,
- expected: &Output<T>,
- label: &str,
-) {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 1e-2f32
- } else {
- 1e-5
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
- let backend_name = std::any::type_name::<B>();
- let type_name = std::any::type_name::<T>();
-
- assert_eq_float::<T>(
- &expected.x_out,
- &output.x_out,
- eps,
- &format!("Conv1dScan x_out {backend_name} {label} (type={type_name})"),
- );
- assert_eq_float::<T>(
- &expected.b_out,
- &output.b_out,
- eps,
- &format!("Conv1dScan b_out {backend_name} {label} (type={type_name})"),
- );
- assert_eq_float::<T>(
- &expected.c_out,
- &output.c_out,
- eps,
- &format!("Conv1dScan c_out {backend_name} {label} (type={type_name})"),
- );
- assert_eq_float::<T>(
- &expected.state_out,
- &output.state_out,
- eps,
- &format!("Conv1dScan state_out {backend_name} {label} (type={type_name})"),
- );
- });
-}
-
-// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4, suffix_len=2
-fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
- for has_bias in [true, false] {
- let label = format!("basic(bias={has_bias})");
- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, has_bias, ActivationType::SILU);
- test_internal(&input, &expected, &label);
- }
-}
-
-// kernel_size=2 (state_stride=1, minimal state)
-fn test_small_kernel<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 3, true, ActivationType::SILU);
- test_internal(&input, &expected, "small_kernel");
-}
-
-// Single token suffix
-fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, ActivationType::SILU);
- test_internal(&input, &expected, "single_token");
-}
-
-// Many tokens
-fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 16, true, ActivationType::SILU);
- test_internal(&input, &expected, "many_tokens");
-}
-
-// Larger dimensions closer to real usage
-fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 8, true, ActivationType::SILU);
- test_internal(&input, &expected, "large");
-}
-
-fn test_identity_activation<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, false, ActivationType::IDENTITY);
- test_internal(&input, &expected, "identity_activation");
-}
-
-fn test_gelu_activation<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::GELUApprox);
- test_internal(&input, &expected, "gelu_activation");
-}
-
-// f32
-#[uzu_test]
-fn test_basic_f32() {
- test_basic::<f32>();
-}
-
-#[uzu_test]
-fn test_small_kernel_f32() {
- test_small_kernel::<f32>();
-}
-
-#[uzu_test]
-fn test_single_token_f32() {
- test_single_token::<f32>();
-}
-
-#[uzu_test]
-fn test_many_tokens_f32() {
- test_many_tokens::<f32>();
-}
-
-#[uzu_test]
-fn test_large_f32() {
- test_large::<f32>();
-}
-
-#[uzu_test]
-fn test_identity_activation_f32() {
- test_identity_activation::<f32>();
-}
-
-#[uzu_test]
-fn test_gelu_activation_f32() {
- test_gelu_activation::<f32>();
-}
-
-// f16
-#[uzu_test]
-fn test_basic_f16() {
- test_basic::<f16>();
-}
-
-#[uzu_test]
-fn test_small_kernel_f16() {
- test_small_kernel::<f16>();
-}
-
-#[uzu_test]
-fn test_single_token_f16() {
- test_single_token::<f16>();
-}
-
-#[uzu_test]
-fn test_many_tokens_f16() {
- test_many_tokens::<f16>();
-}
-
-#[uzu_test]
-fn test_large_f16() {
- test_large::<f16>();
-}
-
-// bf16
-#[uzu_test]
-fn test_basic_bf16() {
- test_basic::<bf16>();
-}
-
-#[uzu_test]
-fn test_small_kernel_bf16() {
- test_small_kernel::<bf16>();
-}
-
-#[uzu_test]
-fn test_single_token_bf16() {
- test_single_token::<bf16>();
-}
-
-#[uzu_test]
-fn test_many_tokens_bf16() {
- test_many_tokens::<bf16>();
-}
-
-#[uzu_test]
-fn test_large_bf16() {
- test_large::<bf16>();
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs
deleted file mode 100644
index 205b5df42..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs
+++ /dev/null
@@ -1,261 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::activation_type::ActivationType, kernel::SSDPrefillKernel,
- },
- data_type::{DataType, DataTypeRepr},
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
- },
-};
-
-struct Input<T: DataTypeRepr + Float> {
- x: Box<[T]>,
- dt: Box<[T]>,
- b: Box<[T]>,
- c: Box<[T]>,
- d: Box<[T]>,
- z: Box<[T]>,
- state: Box<[T]>,
- suffix_len: usize,
- num_heads: usize,
- head_dim: usize,
- state_dim: usize,
- group_size: u32,
- x_strides: [usize; 3],
- dt_strides: [usize; 2],
- cb_strides: [usize; 3],
- state_strides: [usize; 3],
-}
-
-struct Output<T: DataTypeRepr + Float> {
- y: Vec<T>,
- state: Vec<T>,
-}
-
-fn get_input<T: DataTypeRepr + Float>(
- suffix_len: usize,
- num_heads: usize,
- head_dim: usize,
- state_dim: usize,
- group_size: u32,
-) -> Input<T> {
- let safe_group = group_size.max(1) as usize;
- let group_count = num_heads / safe_group;
- let total_x = suffix_len * num_heads * head_dim;
- let total_dt = suffix_len * num_heads;
- let total_cb = suffix_len * group_count * state_dim;
- let total_state = num_heads * head_dim * state_dim;
-
- let x: Vec<T> = (0..total_x).map(|i| T::from(((i % 17) as f64) * 0.01 - 0.05).unwrap()).collect();
- let dt: Vec<T> = (0..total_dt).map(|i| T::from(((i % 13) as f64) * 0.2 - 1.5).unwrap()).collect();
- let b: Vec<T> = (0..total_cb).map(|i| T::from(((i % 11) as f64) * 0.02 - 0.05).unwrap()).collect();
- let c: Vec<T> = (0..total_cb).map(|i| T::from(((i % 19) as f64) * 0.01 - 0.02).unwrap()).collect();
- let d: Vec<T> = (0..num_heads).map(|i| T::from(((i % 3) as f64) * 0.05 - 0.05).unwrap()).collect();
- let z: Vec<T> = (0..total_x).map(|i| T::from(((i % 23) as f64) * 0.02 - 0.1).unwrap()).collect();
- let state: Vec<T> = (0..total_state).map(|i| T::from(((i % 29) as f64) * 0.03 - 0.4).unwrap()).collect();
-
- let x_strides = [num_heads * head_dim, head_dim, 1];
- let dt_strides = [num_heads, 1];
- let cb_strides = [group_count * state_dim, state_dim, 1];
- let state_strides = [head_dim * state_dim, state_dim, 1];
-
- Input {
- x: x.into_boxed_slice(),
- dt: dt.into_boxed_slice(),
- b: b.into_boxed_slice(),
- c: c.into_boxed_slice(),
- d: d.into_boxed_slice(),
- z: z.into_boxed_slice(),
- state: state.into_boxed_slice(),
- suffix_len,
- num_heads,
- head_dim,
- state_dim,
- group_size,
- x_strides,
- dt_strides,
- cb_strides,
- state_strides,
- }
-}
-
-fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
- let context = B::Context::new().expect("Failed to create Context");
-
- let total_x = input.suffix_len * input.num_heads * input.head_dim;
-
- let x = create_buffer_with_data::<B, T>(&context, &input.x);
- let dt = create_buffer_with_data::<B, T>(&context, &input.dt);
- let b = create_buffer_with_data::<B, T>(&context, &input.b);
- let c = create_buffer_with_data::<B, T>(&context, &input.c);
- let d = create_buffer_with_data::<B, T>(&context, &input.d);
- let z = create_buffer_with_data::<B, T>(&context, &input.z);
- let mut state = create_buffer_with_data::<B, T>(&context, &input.state);
- let mut y = create_buffer::<B, T>(&context, total_x);
-
- let x_strides: [u32; 3] = input.x_strides.map(|s| s as u32);
- let dt_strides: [u32; 2] = input.dt_strides.map(|s| s as u32);
- let cb_strides: [u32; 3] = input.cb_strides.map(|s| s as u32);
- let state_strides: [u32; 3] = input.state_strides.map(|s| s as u32);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- let kernel = <<B as Backend>::Kernels as Kernels>::SSDPrefillKernel::new(&context, T::data_type())
- .expect("Failed to create SSDPrefillKernel");
- kernel.encode(
- &x,
- &dt,
- &b,
- &c,
- &d,
- &z,
- &mut state,
- &mut y,
- input.suffix_len as u32,
- input.group_size,
- input.state_dim as u32,
- &x_strides,
- &dt_strides,
- &cb_strides,
- &state_strides,
- input.num_heads as u32,
- input.head_dim as u32,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- Output {
- y: buffer_to_vec(&y),
- state: buffer_to_vec(&state),
- }
-}
-
-fn reference_output<T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
- let safe_group = input.group_size.max(1) as usize;
- let mut state = input.state.to_vec();
- let mut y = vec![T::zero(); input.suffix_len * input.num_heads * input.head_dim];
-
- for h in 0..input.num_heads {
- let group_idx = h / safe_group;
- for dh in 0..input.head_dim {
- let state_base = h * input.state_strides[0] + dh * input.state_strides[1];
- for token in 0..input.suffix_len {
- let x_idx = token * input.x_strides[0] + h * input.x_strides[1] + dh * input.x_strides[2];
- let dt_idx = token * input.dt_strides[0] + h * input.dt_strides[1];
- let cb_base = token * input.cb_strides[0] + group_idx * input.cb_strides[1];
-
- let x_val = input.x[x_idx];
- let dt_val = ActivationType::SOFTPLUS.activate(input.dt[dt_idx]);
- let decay_val = (-dt_val).exp();
-
- let mut acc = input.d[h] * x_val;
- for s in 0..input.state_dim {
- let state_idx = state_base + s * input.state_strides[2];
- let cb_idx = cb_base + s * input.cb_strides[2];
- let new_state = decay_val * state[state_idx] + x_val * input.b[cb_idx];
- state[state_idx] = new_state;
- acc = acc + new_state * input.c[cb_idx];
- }
-
- let gate = ActivationType::SILU.activate(input.z[x_idx]);
- y[x_idx] = acc * gate;
- }
- }
- }
-
- Output {
- y,
- state,
- }
-}
-
-fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
- input: &Input<T>,
- expected: &Output<T>,
- label: &str,
-) {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 2e-2f32
- } else {
- 5e-5
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
- let backend_name = std::any::type_name::<B>();
- let type_name = std::any::type_name::<T>();
-
- assert_eq_float::<T>(
- &expected.y,
- &output.y,
- eps,
- &format!("SSDPrefill y {backend_name} {label} (type={type_name})"),
- );
- assert_eq_float::<T>(
- &expected.state,
- &output.state,
- eps,
- &format!("SSDPrefill state {backend_name} {label} (type={type_name})"),
- );
- });
-}
-
-// --- test shapes ---
-
-fn test_shape(
- suffix_len: usize,
- num_heads: usize,
- head_dim: usize,
- state_dim: usize,
- group_size: u32,
- label: &str,
-) {
- fn run<T: DataTypeRepr + Float + Debug + Display>(
- suffix_len: usize,
- num_heads: usize,
- head_dim: usize,
- state_dim: usize,
- group_size: u32,
- label: &str,
- ) {
- let input = get_input::<T>(suffix_len, num_heads, head_dim, state_dim, group_size);
- let expected = reference_output(&input);
- test_internal(&input, &expected, label);
- }
- run::<f32>(suffix_len, num_heads, head_dim, state_dim, group_size, label);
- run::<f16>(suffix_len, num_heads, head_dim, state_dim, group_size, label);
- run::<bf16>(suffix_len, num_heads, head_dim, state_dim, group_size, label);
-}
-
-// --- Prefill ---
-#[uzu_test]
-fn test_prefill_basic() {
- test_shape(512, 32, 64, 64, 1, "prefill_basic");
-}
-
-#[uzu_test]
-fn test_prefill_small() {
- test_shape(4, 4, 4, 8, 1, "prefill_small");
-}
-
-#[uzu_test]
-fn test_prefill_minimal() {
- test_shape(1, 1, 1, 1, 1, "prefill_minimal");
-}
-
-#[uzu_test]
-fn test_prefill_multi_group() {
- test_shape(8, 8, 4, 16, 4, "prefill_multi_group");
-}
-
-#[uzu_test]
-fn test_prefill_group_per_head() {
- test_shape(8, 4, 4, 8, 1, "prefill_group_per_head");
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs
deleted file mode 100644
index 6cd0d5d63..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs
+++ /dev/null
@@ -1,244 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::SplitInProjKernel,
- },
- cpu::Cpu,
- },
- data_type::{DataType, DataTypeRepr},
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
- },
-};
-
-struct Input<T: DataTypeRepr + Float> {
- input: Box<[T]>,
- z_bias: Box<[T]>,
- suffix_length: u32,
- total_dim: u32,
- conv_dim: u32,
- inner_dim: u32,
- num_heads: u32,
-}
-
-struct Output<T: DataTypeRepr + Float> {
- conv_out: Vec<T>,
- z_out: Vec<T>,
- dt_out: Vec<T>,
-}
-
-fn get_input<T: DataTypeRepr + Float>(
- suffix_length: u32,
- conv_dim: u32,
- inner_dim: u32,
- num_heads: u32,
-) -> Input<T> {
- let total_dim = conv_dim + inner_dim + num_heads;
- let input_size = suffix_length as usize * total_dim as usize;
-
- let input: Vec<T> = (0..input_size).map(|i| T::from(0.1 * ((i as f64 % 13.0) - 6.0)).unwrap()).collect();
- let z_bias: Vec<T> = (0..inner_dim as usize).map(|i| T::from(0.05 * ((i as f64 % 7.0) - 3.0)).unwrap()).collect();
-
- Input {
- input: input.into_boxed_slice(),
- z_bias: z_bias.into_boxed_slice(),
- suffix_length,
- total_dim,
- conv_dim,
- inner_dim,
- num_heads,
- }
-}
-
-fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
- let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::SplitInProjKernel::new(&context, T::data_type())
- .expect("Failed to create SplitInProjKernel");
-
- let conv_out_size = input.suffix_length as usize * input.conv_dim as usize;
- let z_out_size = input.suffix_length as usize * input.inner_dim as usize;
- let dt_out_size = input.suffix_length as usize * input.num_heads as usize;
-
- let input_buffer = create_buffer_with_data::<B, T>(&context, &input.input);
- let z_bias = create_buffer_with_data::<B, T>(&context, &input.z_bias);
-
- let mut conv_out = create_buffer::<B, T>(&context, conv_out_size);
- let mut z_out = create_buffer::<B, T>(&context, z_out_size);
- let mut dt_out = create_buffer::<B, T>(&context, dt_out_size);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- kernel.encode(
- &input_buffer,
- &mut conv_out,
- &mut z_out,
- &mut dt_out,
- &z_bias,
- input.suffix_length,
- input.total_dim,
- input.conv_dim,
- input.inner_dim,
- input.num_heads,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- Output {
- conv_out: buffer_to_vec(&conv_out),
- z_out: buffer_to_vec(&z_out),
- dt_out: buffer_to_vec(&dt_out),
- }
-}
-
-fn get_test_data<T: DataTypeRepr + Float>(
- suffix_length: u32,
- conv_dim: u32,
- inner_dim: u32,
- num_heads: u32,
-) -> (Input<T>, Output<T>) {
- let input = get_input::<T>(suffix_length, conv_dim, inner_dim, num_heads);
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
-}
-
-fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
- input: &Input<T>,
- expected: &Output<T>,
- label: &str,
-) {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 1e-2f32
- } else {
- 1e-5
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
- let backend_name = std::any::type_name::<B>();
- let type_name = std::any::type_name::<T>();
-
- assert_eq_float::<T>(
- &expected.conv_out,
- &output.conv_out,
- eps,
- &format!("SplitInProj conv_out {backend_name} {label} (type={type_name})"),
- );
- assert_eq_float::<T>(
- &expected.z_out,
- &output.z_out,
- eps,
- &format!("SplitInProj z_out {backend_name} {label} (type={type_name})"),
- );
- assert_eq_float::<T>(
- &expected.dt_out,
- &output.dt_out,
- eps,
- &format!("SplitInProj dt_out {backend_name} {label} (type={type_name})"),
- );
- });
-}
-
-// conv_dim=8, inner_dim=4, num_heads=2, suffix_length=2
-fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(2, 8, 4, 2);
- test_internal(&input, &expected, "basic");
-}
-
-// Single token
-fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(1, 8, 4, 2);
- test_internal(&input, &expected, "single_token");
-}
-
-// Many tokens
-fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(16, 8, 4, 2);
- test_internal(&input, &expected, "many_tokens");
-}
-
-// Larger dimensions closer to real usage
-fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(8, 128, 64, 32);
- test_internal(&input, &expected, "large");
-}
-
-// Minimal dimensions
-fn test_minimal<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(1, 1, 1, 1);
- test_internal(&input, &expected, "minimal");
-}
-
-// f32
-#[uzu_test]
-fn test_basic_f32() {
- test_basic::<f32>();
-}
-
-#[uzu_test]
-fn test_single_token_f32() {
- test_single_token::<f32>();
-}
-
-#[uzu_test]
-fn test_many_tokens_f32() {
- test_many_tokens::<f32>();
-}
-
-#[uzu_test]
-fn test_large_f32() {
- test_large::<f32>();
-}
-
-#[uzu_test]
-fn test_minimal_f32() {
- test_minimal::<f32>();
-}
-
-// f16
-#[uzu_test]
-fn test_basic_f16() {
- test_basic::<f16>();
-}
-
-#[uzu_test]
-fn test_single_token_f16() {
- test_single_token::<f16>();
-}
-
-#[uzu_test]
-fn test_many_tokens_f16() {
- test_many_tokens::<f16>();
-}
-
-#[uzu_test]
-fn test_large_f16() {
- test_large::<f16>();
-}
-
-// bf16
-#[uzu_test]
-fn test_basic_bf16() {
- test_basic::<bf16>();
-}
-
-#[uzu_test]
-fn test_single_token_bf16() {
- test_single_token::<bf16>();
-}
-
-#[uzu_test]
-fn test_many_tokens_bf16() {
- test_many_tokens::<bf16>();
-}
-
-#[uzu_test]
-fn test_large_bf16() {
- test_large::<bf16>();
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs
deleted file mode 100644
index 508e1ddb5..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs
+++ /dev/null
@@ -1,329 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::SSDUpdateKernel,
- },
- cpu::Cpu,
- },
- data_type::{DataType, DataTypeRepr},
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
- },
-};
-
-struct Input<T: DataTypeRepr + Float> {
- x: Box<[T]>,
- dt: Box<[T]>,
- b: Box<[T]>,
- c: Box<[T]>,
- d: Box<[T]>,
- z: Box<[T]>,
- state: Box<[T]>,
- bsz: u32,
- h: u32,
- dh: u32,
- g: u32,
- n: u32,
- state_in_place: bool,
-}
-
-struct Output<T: DataTypeRepr + Float> {
- y: Vec<T>,
- next_state: Vec<T>,
-}
-
-fn get_input<T: DataTypeRepr + Float>(
- bsz: u32,
- h: u32,
- dh: u32,
- g: u32,
- n: u32,
- state_in_place: bool,
-) -> Input<T> {
- let bsz_u = bsz as usize;
- let h_u = h as usize;
- let dh_u = dh as usize;
- let g_u = g as usize;
- let n_u = n as usize;
-
- let x: Vec<T> = (0..bsz_u * h_u * dh_u).map(|i| T::from(((i % 7) as f64) * 0.1 - 0.2).unwrap()).collect();
- let z: Vec<T> = (0..bsz_u * h_u * dh_u).map(|i| T::from(((i % 5) as f64) * 0.1 - 0.1).unwrap()).collect();
- let dt: Vec<T> = (0..bsz_u * h_u).map(|i| T::from(((i % 5) as f64) * 0.3 - 1.0).unwrap()).collect();
- let b: Vec<T> = (0..bsz_u * g_u * n_u).map(|i| T::from(((i % 11) as f64) * 0.02 - 0.05).unwrap()).collect();
- let c: Vec<T> = (0..bsz_u * g_u * n_u).map(|i| T::from(((i % 13) as f64) * 0.015).unwrap()).collect();
- let d: Vec<T> = (0..h_u).map(|i| T::from(((i % 3) as f64) * 0.05).unwrap()).collect();
- let state: Vec<T> =
- (0..bsz_u * h_u * dh_u * n_u).map(|i| T::from(((i % 23) as f64) * 0.01 - 0.05).unwrap()).collect();
-
- Input {
- x: x.into_boxed_slice(),
- dt: dt.into_boxed_slice(),
- b: b.into_boxed_slice(),
- c: c.into_boxed_slice(),
- d: d.into_boxed_slice(),
- z: z.into_boxed_slice(),
- state: state.into_boxed_slice(),
- bsz,
- h,
- dh,
- g,
- n,
- state_in_place,
- }
-}
-
-fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
- let context = B::Context::new().expect("Failed to create Context");
- let kernel =
- <<B as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&context, T::data_type(), input.state_in_place)
- .expect("Failed to create SSDUpdateKernel");
-
- let bsz = input.bsz as usize;
- let h = input.h as usize;
- let dh = input.dh as usize;
- let g = input.g as usize;
- let n = input.n as usize;
-
- let x_strides = [(h * dh) as u32, dh as u32, 1u32];
- let dt_strides = [h as u32, 1u32];
- let cb_strides = [(g * n) as u32, n as u32, 1u32];
- let state_strides = [(h * dh * n) as u32, (dh * n) as u32, n as u32, 1u32];
-
- let y_size = bsz * h * dh;
- let ns_size = bsz * h * dh * n;
-
- let x = create_buffer_with_data::<B, T>(&context, &input.x);
- let dt = create_buffer_with_data::<B, T>(&context, &input.dt);
- let b = create_buffer_with_data::<B, T>(&context, &input.b);
- let c = create_buffer_with_data::<B, T>(&context, &input.c);
- let d = create_buffer_with_data::<B, T>(&context, &input.d);
- let z = create_buffer_with_data::<B, T>(&context, &input.z);
-
- let mut y = create_buffer::<B, T>(&context, y_size);
-
- if input.state_in_place {
- let mut next_state = create_buffer_with_data::<B, T>(&context, &input.state);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- kernel.encode(
- &x,
- &dt,
- &b,
- &c,
- &d,
- &z,
- None::<&B::GlobalBuffer>,
- &mut y,
- &mut next_state,
- (h / g) as u32,
- input.n,
- &x_strides,
- &dt_strides,
- &cb_strides,
- &state_strides,
- input.bsz,
- input.h,
- input.dh,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- Output {
- y: buffer_to_vec(&y),
- next_state: buffer_to_vec(&next_state),
- }
- } else {
- let state = create_buffer_with_data::<B, T>(&context, &input.state);
- let mut next_state = create_buffer::<B, T>(&context, ns_size);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- kernel.encode(
- &x,
- &dt,
- &b,
- &c,
- &d,
- &z,
- Some(&state),
- &mut y,
- &mut next_state,
- (h / g) as u32,
- input.n,
- &x_strides,
- &dt_strides,
- &cb_strides,
- &state_strides,
- input.bsz,
- input.h,
- input.dh,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- Output {
- y: buffer_to_vec(&y),
- next_state: buffer_to_vec(&next_state),
- }
- }
-}
-
-fn get_test_data<T: DataTypeRepr + Float>(
- bsz: u32,
- h: u32,
- dh: u32,
- g: u32,
- n: u32,
- state_in_place: bool,
-) -> (Input<T>, Output<T>) {
- let input = get_input::<T>(bsz, h, dh, g, n, state_in_place);
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
-}
-
-fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
- input: &Input<T>,
- expected: &Output<T>,
- label: &str,
-) {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 2e-2f32
- } else {
- 1e-5
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
- let backend_name = std::any::type_name::<B>();
- let type_name = std::any::type_name::<T>();
-
- assert_eq_float::<T>(
- &expected.y,
- &output.y,
- eps,
- &format!("SSDUpdate y {backend_name} {label} (type={type_name})"),
- );
- assert_eq_float::<T>(
- &expected.next_state,
- &output.next_state,
- eps,
- &format!("SSDUpdate next_state {backend_name} {label} (type={type_name})"),
- );
- });
-}
-
-// bsz=1, h=4, dh=3, g=2, n=8 (matches existing Metal test)
-fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, false);
- test_internal(&input, &expected, "basic");
-}
-
-// bsz=2 multi-batch
-fn test_multi_batch<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(2, 4, 3, 2, 8, false);
- test_internal(&input, &expected, "multi_batch");
-}
-
-// state_in_place=true
-fn test_state_in_place<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, true);
- test_internal(&input, &expected, "state_in_place");
-}
-
-// Larger dimensions closer to real usage
-fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(1, 8, 4, 4, 16, false);
- test_internal(&input, &expected, "large");
-}
-
-// Minimal: single head, single group, dh=1, n=1
-fn test_minimal<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(1, 1, 1, 1, 1, false);
- test_internal(&input, &expected, "minimal");
-}
-
-// g == h (each head is its own group)
-fn test_group_per_head<T: DataTypeRepr + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(1, 4, 2, 4, 8, false);
- test_internal(&input, &expected, "group_per_head");
-}
-
-// f32
-#[uzu_test]
-fn test_basic_f32() {
- test_basic::<f32>();
-}
-
-#[uzu_test]
-fn test_multi_batch_f32() {
- test_multi_batch::<f32>();
-}
-
-#[uzu_test]
-fn test_state_in_place_f32() {
- test_state_in_place::<f32>();
-}
-
-#[uzu_test]
-fn test_large_f32() {
- test_large::<f32>();
-}
-
-#[uzu_test]
-fn test_minimal_f32() {
- test_minimal::<f32>();
-}
-
-#[uzu_test]
-fn test_group_per_head_f32() {
- test_group_per_head::<f32>();
-}
-
-// f16
-#[uzu_test]
-fn test_basic_f16() {
- test_basic::<f16>();
-}
-
-#[uzu_test]
-fn test_multi_batch_f16() {
- test_multi_batch::<f16>();
-}
-
-#[uzu_test]
-fn test_state_in_place_f16() {
- test_state_in_place::<f16>();
-}
-
-#[uzu_test]
-fn test_large_f16() {
- test_large::<f16>();
-}
-
-// bf16
-#[uzu_test]
-fn test_basic_bf16() {
- test_basic::<bf16>();
-}
-
-#[uzu_test]
-fn test_multi_batch_bf16() {
- test_multi_batch::<bf16>();
-}
-
-#[uzu_test]
-fn test_state_in_place_bf16() {
- test_state_in_place::<bf16>();
-}
-
-#[uzu_test]
-fn test_large_bf16() {
- test_large::<bf16>();
-}
diff --git a/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs b/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs
deleted file mode 100644
index 3702cf678..000000000
--- a/crates/uzu-engine/unit/encodable_block/mamba_mixer/ssd_prefill_test.rs
+++ /dev/null
@@ -1,470 +0,0 @@
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::activation_type::ActivationType,
- kernel::{Conv1dScanKernel, SSDPrefill64Kernel, SSDPrefillKernel},
- },
- data_type::DataType,
- tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
-};
-
-#[derive(Debug, Clone, Copy)]
-enum SSDPrefillMode {
- Universal,
- Special64,
-}
-
-fn ssd_prefill_cpu_reference(
- suffix_len: usize,
- num_heads: usize,
- head_dim: usize,
- state_dim: usize,
- group_size: i32,
- x_data: &[f32],
- dt_raw_data: &[f32],
- b_data: &[f32],
- c_data: &[f32],
- d_data: &[f32],
- z_data: &[f32],
- state_init: &[f32],
- x_strides: [usize; 3],
- dt_strides: [usize; 2],
- cb_strides: [usize; 3],
- state_strides: [usize; 3],
-) -> (Vec<f32>, Vec<f32>) {
- let total_pairs = suffix_len * num_heads * head_dim;
- let mut y_out = vec![0.0f32; total_pairs];
- let mut state = state_init.to_vec();
- let safe_group = group_size.max(1) as usize;
-
- for h in 0..num_heads {
- let group_idx = h / safe_group;
- for dh in 0..head_dim {
- let state_base = h * state_strides[0] + dh * state_strides[1];
- for token in 0..suffix_len {
- let x_idx = token * x_strides[0] + h * x_strides[1] + dh * x_strides[2];
- let dt_idx = token * dt_strides[0] + h * dt_strides[1];
- let cb_base = token * cb_strides[0] + group_idx * cb_strides[1];
-
- let x_val = x_data[x_idx];
- let dt_raw = dt_raw_data[dt_idx];
- let dt_val = ActivationType::SOFTPLUS.activate(dt_raw);
- let decay_val = (-dt_val).exp();
- let dt_scaled_input = x_val;
- let gate = ActivationType::SILU.activate(z_data[x_idx]);
- let mut acc = d_data[h] * x_val;
-
- for s in 0..state_dim {
- let state_idx = state_base + s * state_strides[2];
- let cb_idx = cb_base + s * cb_strides[2];
- let b_coeff = b_data[cb_idx];
- let c_coeff = c_data[cb_idx];
- let new_state = decay_val * state[state_idx] + dt_scaled_input * b_coeff;
- state[state_idx] = new_state;
- acc += new_state * c_coeff;
- }
-
- y_out[x_idx] = acc * gate;
- }
- }
- }
-
- (y_out, state)
-}
-
-struct SSDPrefillFixture {
- suffix_len: usize,
- num_heads: usize,
- head_dim: usize,
- state_dim: usize,
- group_size: i32,
- x_strides: [usize; 3],
- dt_strides: [usize; 2],
- cb_strides: [usize; 3],
- state_strides: [usize; 3],
- x_data: Vec<f32>,
- dt_data: Vec<f32>,
- b_data: Vec<f32>,
- c_data: Vec<f32>,
- d_data: Vec<f32>,
- z_data: Vec<f32>,
- state_init: Vec<f32>,
-}
-
-impl SSDPrefillFixture {
- fn new() -> Self {
- let suffix_len = 512usize;
- let num_heads = 32usize;
- let head_dim = 64usize;
- let state_dim = 64usize;
- let group_size = 1i32;
- let group_count = num_heads / (group_size as usize);
- let total_pairs = num_heads * head_dim;
- let total_x = suffix_len * total_pairs;
- let total_dt = suffix_len * num_heads;
- let total_cb = suffix_len * group_count * state_dim;
- let total_state = num_heads * head_dim * state_dim;
-
- let x_data: Vec<f32> = (0..total_x).map(|i| ((i % 17) as f32) * 0.01 - 0.05).collect();
- let dt_data: Vec<f32> = (0..total_dt).map(|i| ((i % 13) as f32) * 0.2 - 1.5).collect();
- let b_data: Vec<f32> = (0..total_cb).map(|i| ((i % 11) as f32) * 0.02 - 0.05).collect();
- let c_data: Vec<f32> = (0..total_cb).map(|i| ((i % 19) as f32) * 0.01 - 0.02).collect();
- let d_data: Vec<f32> = (0..num_heads).map(|i| ((i % 3) as f32) * 0.05 - 0.05).collect();
- let z_data: Vec<f32> = (0..total_x).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();
- let state_init: Vec<f32> = (0..total_state).map(|i| ((i % 29) as f32) * 0.03 - 0.4).collect();
-
- let x_strides = [num_heads * head_dim, head_dim, 1usize];
- let dt_strides = [num_heads, 1usize];
- let cb_strides = [group_count * state_dim, state_dim, 1usize];
- let state_strides = [head_dim * state_dim, state_dim, 1usize];
-
- Self {
- suffix_len,
- num_heads,
- head_dim,
- state_dim,
- group_size,
- x_strides,
- dt_strides,
- cb_strides,
- state_strides,
- x_data,
- dt_data,
- b_data,
- c_data,
- d_data,
- z_data,
- state_init,
- }
- }
-}
-
-fn run_prefill_kernel_mode<B: Backend>(
- ctx: &B::Context,
- fixture: &SSDPrefillFixture,
- mode: SSDPrefillMode,
-) -> (Vec<f32>, Vec<f32>) {
- let x_buf = create_buffer_with_data::<B, _>(ctx, &fixture.x_data);
- let dt_buf = create_buffer_with_data::<B, _>(ctx, &fixture.dt_data);
- let b_buf = create_buffer_with_data::<B, _>(ctx, &fixture.b_data);
- let c_buf = create_buffer_with_data::<B, _>(ctx, &fixture.c_data);
- let d_buf = create_buffer_with_data::<B, _>(ctx, &fixture.d_data);
- let z_buf = create_buffer_with_data::<B, _>(ctx, &fixture.z_data);
- let mut state_buf = create_buffer_with_data::<B, _>(ctx, &fixture.state_init);
- let mut y_buf = create_buffer::<B, f32>(ctx, fixture.suffix_len * fixture.num_heads * fixture.head_dim);
-
- let x_strides = fixture.x_strides.map(|stride| stride as u32);
- let dt_strides = fixture.dt_strides.map(|stride| stride as u32);
- let cb_strides = fixture.cb_strides.map(|stride| stride as u32);
- let state_strides = fixture.state_strides.map(|stride| stride as u32);
-
- let mut command_buffer = ctx.create_command_buffer(None, None).unwrap();
- match mode {
- SSDPrefillMode::Universal => {
- let kernel = <<B as Backend>::Kernels as Kernels>::SSDPrefillKernel::new(ctx, DataType::F32)
- .expect("Failed to create SSD prefill kernel");
- kernel.encode(
- &x_buf,
- &dt_buf,
- &b_buf,
- &c_buf,
- &d_buf,
- &z_buf,
- &mut state_buf,
- &mut y_buf,
- fixture.suffix_len as u32,
- fixture.group_size as u32,
- fixture.state_dim as u32,
- &x_strides,
- &dt_strides,
- &cb_strides,
- &state_strides,
- fixture.num_heads as u32,
- fixture.head_dim as u32,
- &mut command_buffer,
- );
- },
- SSDPrefillMode::Special64 => {
- assert_eq!(fixture.state_dim, 64);
- let kernel = <<B as Backend>::Kernels as Kernels>::SSDPrefill64Kernel::new(ctx, DataType::F32)
- .expect("Failed to create SSD prefill64 kernel");
- kernel.encode(
- &x_buf,
- &dt_buf,
- &b_buf,
- &c_buf,
- &d_buf,
- &z_buf,
- &mut state_buf,
- &mut y_buf,
- fixture.suffix_len as u32,
- fixture.group_size as u32,
- fixture.state_dim as u32,
- &x_strides,
- &dt_strides,
- &cb_strides,
- &state_strides,
- fixture.num_heads as u32,
- fixture.head_dim as u32,
- &mut command_buffer,
- );
- },
- }
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let y_vec = buffer_to_vec::<B, f32>(&y_buf);
- let state_vec = buffer_to_vec::<B, f32>(&state_buf);
- drop(y_buf);
- drop(completed);
- (y_vec, state_vec)
-}
-
-fn run_conv_scan_once<B: Backend>(
- ctx: &<B as Backend>::Context,
- kernel: &<<B as Backend>::Kernels as Kernels>::Conv1dScanKernel,
- suffix_len: usize,
- channels: usize,
- kernel_size: i32,
- tap_count: usize,
- x_data: &[f32],
- w_data: &[f32],
- b_data: &[f32],
- state_init: &[f32],
- use_scratch: bool,
- alias_io: bool,
-) -> (Vec<f32>, Vec<f32>) {
- let total_x = suffix_len * channels;
- let _total_w = channels * kernel_size as usize;
- let total_state = channels * tap_count;
-
- let mut y_buf = if alias_io {
- create_buffer_with_data::<B, _>(ctx, x_data)
- } else {
- create_buffer::<B, f32>(ctx, total_x)
- };
- let mut b_out_buf = create_buffer::<B, f32>(ctx, total_x);
- let mut c_out_buf = create_buffer::<B, f32>(ctx, total_x);
- let w_buf = create_buffer_with_data::<B, _>(ctx, w_data);
- let b_buf = create_buffer_with_data::<B, _>(ctx, b_data);
- let mut state_buf = create_buffer_with_data::<B, _>(ctx, state_init);
- let mut scratch_buf = create_buffer::<B, f32>(ctx, total_state);
-
- let padded_len = tap_count + suffix_len;
- let mut padded_host = vec![0.0f32; padded_len * channels];
- for tap in 0..tap_count {
- for ch in 0..channels {
- padded_host[tap * channels + ch] = state_init[ch * tap_count + tap];
- }
- }
- for token in 0..suffix_len {
- for ch in 0..channels {
- padded_host[(tap_count + token) * channels + ch] = x_data[token * channels + ch];
- }
- }
- let padded_buf = create_buffer_with_data::<B, _>(ctx, &padded_host);
-
- let mut command_buffer = ctx.create_command_buffer(None, None).unwrap();
- if use_scratch && tap_count > 0 {
- command_buffer.encode_fill(&mut scratch_buf, 0);
- }
- kernel.encode(
- &padded_buf,
- &w_buf,
- Some(&b_buf),
- &mut y_buf,
- &mut b_out_buf,
- &mut c_out_buf,
- &mut state_buf,
- suffix_len as u32,
- kernel_size as u32,
- channels as u32,
- tap_count as u32,
- channels as u32,
- channels as u32,
- 0u32,
- ActivationType::SILU,
- &mut command_buffer,
- );
-
- if use_scratch && tap_count > 0 {
- command_buffer.encode_copy(&scratch_buf, &mut state_buf);
- }
-
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let y_vec = buffer_to_vec::<B, f32>(&y_buf);
- let state_vec = buffer_to_vec::<B, f32>(&state_buf);
- (y_vec, state_vec)
-}
-
-fn assert_deterministic_for_mode<B: Backend>(mode: SSDPrefillMode) {
- let Some(ctx) = <B as Backend>::Context::new().ok() else {
- eprintln!("Skipping SSD prefill determinism test: no Metal device");
- return;
- };
- let fixture = SSDPrefillFixture::new();
-
- let (y_a, state_a) = run_prefill_kernel_mode::<B>(&ctx, &fixture, mode);
- let (y_b, state_b) = run_prefill_kernel_mode::<B>(&ctx, &fixture, mode);
-
- assert_eq!(y_a, y_b, "Prefill outputs differ in {:?} mode", mode);
- assert_eq!(state_a, state_b, "Prefill states differ in {:?} mode", mode);
-}
-
-fn assert_matches_cpu_reference<B: Backend>(mode: SSDPrefillMode) {
- let Some(ctx) = <B as Backend>::Context::new().ok() else {
- eprintln!("Skipping SSD prefill reference test: no Metal device");
- return;
- };
- let fixture = SSDPrefillFixture::new();
-
- let (y_ref, state_ref) = ssd_prefill_cpu_reference(
- fixture.suffix_len,
- fixture.num_heads,
- fixture.head_dim,
- fixture.state_dim,
- fixture.group_size,
- &fixture.x_data,
- &fixture.dt_data,
- &fixture.b_data,
- &fixture.c_data,
- &fixture.d_data,
- &fixture.z_data,
- &fixture.state_init,
- fixture.x_strides,
- fixture.dt_strides,
- fixture.cb_strides,
- fixture.state_strides,
- );
-
- let (y_gpu, state_gpu) = run_prefill_kernel_mode::<B>(&ctx, &fixture, mode);
-
- let tolerance = 5e-5f32;
- let mut max_y_diff = 0.0f32;
- let mut max_y_idx = 0usize;
- for (idx, (&lhs, &rhs)) in y_gpu.iter().zip(&y_ref).enumerate() {
- let diff = (lhs - rhs).abs();
- if diff > max_y_diff {
- max_y_diff = diff;
- max_y_idx = idx;
- }
- }
- assert!(
- max_y_diff <= tolerance,
- "Prefill outputs diverge in {:?} mode at idx {max_y_idx}: metal={} cpu={} (diff {max_y_diff})",
- mode,
- y_gpu[max_y_idx],
- y_ref[max_y_idx]
- );
-
- let mut max_state_diff = 0.0f32;
- let mut max_state_idx = 0usize;
- for (idx, (&lhs, &rhs)) in state_gpu.iter().zip(&state_ref).enumerate() {
- let diff = (lhs - rhs).abs();
- if diff > max_state_diff {
- max_state_diff = diff;
- max_state_idx = idx;
- }
- }
- assert!(
- max_state_diff <= tolerance,
- "Prefill states diverge in {:?} mode at idx {max_state_idx}: metal={} cpu={} (diff {max_state_diff})",
- mode,
- state_gpu[max_state_idx],
- state_ref[max_state_idx]
- );
-}
-
-fn conv1d_scan_deterministic_internal<B: Backend>() {
- let Some(ctx) = <B as Backend>::Context::new().ok() else {
- eprintln!("Skipping conv1d scan determinism test: no Metal device");
- return;
- };
- let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, true).unwrap();
-
- let suffix_len = 192usize;
- let channels = 8usize;
- let kernel_size = 5i32;
- let tap_count = (kernel_size - 1) as usize;
-
- let total_x = suffix_len * channels;
- let total_state = channels * tap_count;
-
- let x_data: Vec<f32> = (0..total_x).map(|i| ((i % 31) as f32) * 0.02 - 0.3).collect();
- let w_data: Vec<f32> = (0..(channels * kernel_size as usize)).map(|i| ((i % 17) as f32) * 0.01 - 0.04).collect();
- let b_data: Vec<f32> = (0..channels).map(|i| ((i % 5) as f32) * 0.03 - 0.07).collect();
- let state_init: Vec<f32> = (0..total_state).map(|i| ((i % 23) as f32) * 0.02 - 0.1).collect();
-
- let use_scratch = tap_count > 0 && suffix_len > 1;
-
- for &alias_io in &[false, true] {
- let first = run_conv_scan_once::<B>(
- &ctx,
- &kernel,
- suffix_len,
- channels,
- kernel_size,
- tap_count,
- &x_data,
- &w_data,
- &b_data,
- &state_init,
- use_scratch,
- alias_io,
- );
- let second = run_conv_scan_once::<B>(
- &ctx,
- &kernel,
- suffix_len,
- channels,
- kernel_size,
- tap_count,
- &x_data,
- &w_data,
- &b_data,
- &state_init,
- use_scratch,
- alias_io,
- );
-
- assert_eq!(first.0, second.0, "Conv outputs differ (alias_io={alias_io})");
- assert_eq!(first.1, second.1, "Conv states differ (alias_io={alias_io})");
- }
-}
-
-#[uzu_test]
-fn ssd_prefill_universal_is_deterministic() {
- for_each_non_cpu_backend!(|B| {
- assert_deterministic_for_mode::<B>(SSDPrefillMode::Universal);
- });
-}
-
-#[uzu_test]
-fn ssd_prefill_special64_is_deterministic() {
- for_each_non_cpu_backend!(|B| {
- assert_deterministic_for_mode::<B>(SSDPrefillMode::Special64);
- });
-}
-
-#[uzu_test]
-fn ssd_prefill_universal_matches_cpu_reference() {
- for_each_non_cpu_backend!(|B| {
- assert_matches_cpu_reference::<B>(SSDPrefillMode::Universal);
- });
-}
-
-#[uzu_test]
-fn ssd_prefill_special64_matches_cpu_reference() {
- for_each_non_cpu_backend!(|B| {
- assert_matches_cpu_reference::<B>(SSDPrefillMode::Special64);
- });
-}
-
-#[uzu_test]
-fn conv1d_scan_is_deterministic() {
- for_each_non_cpu_backend!(|B| {
- conv1d_scan_deterministic_internal::<B>();
- });
-}