A high-performance inference engine for AI models
dslify-conv1d (#184)
Aleksandr Golokozc83fe37638af
diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rsindex 0023390ad..290f34405 100644--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs@@ -2,21 +2,22 @@use std::env;+use objc2::Message;+use super::{EncodableBlock, Metal, transformer_layer};use crate::{- DataType,+ Activation, DataType,backends::{- common::kernel::{SSDUpdateKernel, SplitInProjKernel},+ common::kernel::{Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel},metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,Retained,kernel::{- dsl::{SSDUpdateMetalKernel, SplitInProjMetalKernel},- ssm::{- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,- conv1d_scan::Conv1dDecodeArguments,- ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},+ dsl::{+ Conv1dDecodeMetalKernel, Conv1dPackMetalKernel, Conv1dScanMetalKernel, SSDUpdateMetalKernel,+ SplitInProjMetalKernel,},+ ssm::ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},},},@@ -32,7 +33,9 @@ pub(crate) struct MambaMixer {in_projection: Box<dyn EncodableBlock<Metal>>,out_projection: Box<dyn EncodableBlock<Metal>>,split_inproj: SplitInProjMetalKernel,- conv_scan: Conv1dScanKernel,+ conv_decode: Conv1dDecodeMetalKernel,+ conv_pack: Conv1dPackMetalKernel,+ conv_scan: Conv1dScanMetalKernel,ssd_prefill: SSDPrefillKernels,ssd_update: SSDUpdateMetalKernel,conv_weight: MetalArray,@@ -99,8 +102,21 @@ impl MambaMixer {let split_inproj =SplitInProjMetalKernel::new(mtl_context, data_type).expect("Failed to create split in-projection kernel");- let conv_scan = Conv1dScanKernel::new(mtl_context, data_type, &mamba_config.activation)- .expect("Failed to create conv scan kernel");+ let conv_scan = Conv1dScanMetalKernel::new(+ mtl_context,+ data_type,+ Self::activation_to_int(&mamba_config.activation),+ mamba_config.conv_config.has_biases,+ )+ .expect("Failed to create conv scan kernel");+ let conv_decode = Conv1dDecodeMetalKernel::new(+ mtl_context,+ data_type,+ Self::activation_to_int(&mamba_config.activation),+ mamba_config.conv_config.has_biases,+ )+ .expect("Failed to create conv decode kernel");+ let conv_pack = Conv1dPackMetalKernel::new(mtl_context, data_type).expect("Failed to create conv pack kernel");let ssd_prefill = SSDPrefillKernels::new(mtl_context, data_type).expect("Failed to create SSD prefill kernel");let ssd_update = SSDUpdateMetalKernel::new(mtl_context, data_type).expect("Failed to create SSD decode kernel");let prefill_mode = resolve_prefill_mode_from_env();@@ -111,6 +127,8 @@ impl MambaMixer {in_projection,out_projection,split_inproj,+ conv_decode,+ conv_pack,conv_scan,ssd_prefill,ssd_update,@@ -217,10 +235,7 @@ impl MambaMixer {let weight_storage = self.conv_weight.clone();let weight_buf = weight_storage.buffer().clone();- let bias_buf = self.conv_bias.as_ref().map(|arr| {- let storage = arr.clone();- storage.buffer().clone()- });+ let bias_buf = self.conv_bias.as_ref().map(|arr| arr.buffer());let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();@@ -229,28 +244,26 @@ impl MambaMixer {drop(conv_state);if suffix_length == 1 {- self.conv_scan- .encode_decode(- encoder,- Conv1dDecodeArguments {- x: &input_buf,- w: &weight_buf,- b: bias_buf.as_deref(),- state: &state_buf,- x_out: &x_buf,- b_out: &b_buf,- c_out: &c_buf,- next_state: &state_buf,- suffix_len: suffix_length,- kernel_size: self.config.kernel_size as i32,- row_stride: conv_dim,- state_stride,- channels: conv_dim,- inner_dim,- proj_dim,- },- )- .expect("Failed to encode conv decode kernel");+ if conv_dim > 0 && self.config.kernel_size > 0 {+ self.conv_decode.encode(+ &input_buf,+ &weight_buf,+ bias_buf,+ &state_buf,+ &x_buf,+ &b_buf,+ &c_buf,+ &state_buf,+ self.config.kernel_size as u32,+ conv_dim as u32,+ state_stride as u32,+ conv_dim as u32,+ suffix_length as u32,+ inner_dim as u32,+ proj_dim as u32,+ &encoder,+ );+ }} else {let padded_buf = if state_stride > 0 {let array = state.conv_padded_buffer().expect("Missing conv padded buffer");@@ -258,49 +271,42 @@ impl MambaMixer {let buf = borrow.buffer().clone();drop(borrow);- self.conv_scan- .encode_pack(- encoder,- Conv1dPackArguments {- state_in: &state_buf,- x: &input_buf,- padded: &buf,- state_stride,- row_stride: conv_dim,- suffix_len: suffix_length,- channels: conv_dim,- },- )- .expect("Failed to encode conv pack kernel");+ self.conv_pack.encode(+ &state_buf,+ &input_buf,+ &buf,+ state_stride as u32,+ conv_dim as u32,+ suffix_length as u32,+ conv_dim as u32,+ &encoder,+ );Some(buf)} else {None};- let conv_source = padded_buf.as_deref().unwrap_or(&input_buf);-- self.conv_scan- .encode(- encoder,- Conv1dScanArguments {- padded: conv_source,- w: &weight_buf,- b: bias_buf.as_deref(),- x_out: &x_buf,- b_out: &b_buf,- c_out: &c_buf,- state_out: &state_buf,- suffix_len: suffix_length,- kernel_size: self.config.kernel_size as i32,- row_stride: conv_dim,- state_stride,- channels: conv_dim,- inner_dim,- proj_dim,- },+ if conv_dim > 0 && self.config.kernel_size > 0 {+ let conv_source = padded_buf.as_deref().unwrap_or(&input_buf).retain();+ self.conv_scan.encode(+ &conv_source,+ &weight_buf,+ bias_buf,+ &x_buf,+ &b_buf,+ &c_buf,+ &state_buf,+ suffix_length as u32,+ self.config.kernel_size as u32,+ conv_dim as u32,+ state_stride as u32,+ conv_dim as u32,+ inner_dim as u32,+ proj_dim as u32,+ &encoder,)- .expect("Failed to encode conv scan kernel");+ }}}@@ -441,6 +447,16 @@ impl MambaMixer {encoder,);}++ fn activation_to_int(activation: &Activation) -> u32 {+ match activation {+ Activation::Identity => 0,+ Activation::SiLU {+ ..+ } => 1,+ Activation::Gelu => 2,+ }+ }}fn resolve_prefill_mode_from_env() -> SSDPrefillMode {diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 78e0e4d27..edf2b6d24 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -27,9 +27,7 @@ pub use moe::{MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,MoeScatterWithMapArguments,};-pub use ssm::{- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, SSDPrefillArguments, SSDPrefillMode, SSMKernelError,-};+pub use ssm::{SSDPrefillArguments, SSDPrefillMode};impl MatmulKernels for dsl::MetalKernels {type FullPrecisionMatmulKernel = MatmulKernel;diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metalnew file mode 100644index 000000000..304da7f7c--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal@@ -0,0 +1,164 @@+#include <metal_stdlib>+#include "../definitions.metal"+#include "ssm_common.h"++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(Conv1dPack)(+ device const T* state_in,+ device const T* x,+ device T* padded,+ constant const uint& state_stride,+ constant const uint& row_stride,+ constant const uint& suffix_len,+ constant const uint& num_channels,+ const uint channel_idx AXIS(num_channels, 32),+ const uint row_idx AXIS(state_stride + suffix_len, 1)+) {+ const uint padded_index = row_idx * row_stride + channel_idx;+ if (row_idx < state_stride) {+ const uint state_index = channel_idx * state_stride + row_idx;+ padded[padded_index] = state_in[state_index];+ } else {+ const uint token = row_idx - state_stride;+ const uint x_index = token * row_stride + channel_idx;+ padded[padded_index] = x[x_index];+ }+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(Conv1dDecode)(+ device const T* x,+ device const T* w,+ device const T* b OPTIONAL(has_bias),+ device const T* state,+ device T* x_out,+ device T* b_out,+ device T* c_out,+ device T* next_state,+ constant const uint& kernel_size,+ constant const uint& row_stride,+ constant const uint& state_stride,+ constant const uint& num_channels,+ constant const uint& suffix_len,+ constant const uint& inner_dim,+ constant const uint& proj_dim,+ const uint activation_type SPECIALIZE,+ const bool has_bias SPECIALIZE,+ const uint token_idx AXIS(suffix_len, 32),+ const uint channel_idx AXIS(num_channels, 1)+) {+ const uint x_idx = token_idx * row_stride + channel_idx;+ const uint state_offset = channel_idx * state_stride;+ const device T* w_row = w + channel_idx * kernel_size;++ float acc = 0.0f;+ if (has_bias) {+ acc = float(b[channel_idx]);+ }++ const uint state_taps = max(kernel_size - 1, 0u);+ for (uint tap = 0; tap < state_taps; ++tap) {+ const uint state_index = state_offset + tap;+ const float sample = float(state[state_index]);+ acc += float(w_row[tap]) * sample;+ }++ const float current = float(x[x_idx]);+ const uint current_tap_index = state_taps;+ if (current_tap_index < kernel_size) {+ acc += float(w_row[current_tap_index]) * current;+ }++ const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);+ if (channel_idx < inner_dim) {+ const uint dst = token_idx * inner_dim + channel_idx;+ x_out[dst] = activated;+ } else if (channel_idx < inner_dim + proj_dim) {+ const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);+ b_out[dst] = activated;+ } else if (channel_idx < inner_dim + 2 * proj_dim) {+ const uint dst =+ token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);+ c_out[dst] = activated;+ }++ if (state_taps == 0) {+ return;+ }++ for (uint tap = 0; tap < state_taps - 1; ++tap) {+ const size_t dst_index = state_offset + size_t(tap);+ const size_t src_index = state_offset + size_t(tap + 1);+ next_state[dst_index] = state[src_index];+ }++ const uint tail_index = state_offset + state_taps - 1;+ next_state[tail_index] = static_cast<T>(current);+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(Conv1dScan)(+ device const T* padded, // (prefix+suffix, channels)+ device const T* w, // (channels, kernel)+ device const T* b OPTIONAL(has_bias), // optional (channels)+ device T* x_out, // (suffix, inner_dim)+ device T* b_out, // (suffix, proj_dim)+ device T* c_out, // (suffix, proj_dim)+ device T* state_out, // (channels, kernel-1)+ constant const uint& suffix_len,+ constant const uint& kernel_size,+ constant const uint& row_stride,+ constant const uint& state_stride,+ constant const uint& num_channels,+ constant const uint& inner_dim,+ constant const uint& proj_dim,+ const uint activation_type SPECIALIZE,+ const bool has_bias SPECIALIZE,+ const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 32),+ const uint channel_idx AXIS(num_channels, 1)+) {+ const uint tap_count = max(kernel_size - 1, 0u);+ const uint state_offset = channel_idx * state_stride;+ const uint weight_offset = channel_idx * kernel_size;+ const device T* w_row = w + weight_offset;++ if (token_idx < suffix_len) {+ float acc = 0.0f;+ if (has_bias) {+ acc = float(b[channel_idx]);+ }+ for (uint tap = 0; tap < kernel_size; ++tap) {+ const uint padded_idx = size_t(token_idx) + size_t(tap);+ float sample = 0.0f;+ const uint padded_index = size_t(padded_idx) * row_stride + channel_idx;+ sample = float(padded[padded_index]);+ acc += float(w_row[tap]) * sample;+ }++ const T activated =+ apply_activation_fn(static_cast<T>(acc), activation_type);+ if (channel_idx < inner_dim) {+ const size_t dst = size_t(token_idx) * inner_dim + channel_idx;+ x_out[dst] = activated;+ } else if (channel_idx < inner_dim + proj_dim) {+ const size_t dst =+ size_t(token_idx) * proj_dim + (channel_idx - inner_dim);+ b_out[dst] = activated;+ } else if (channel_idx < inner_dim + 2 * proj_dim) {+ const size_t dst =+ size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);+ c_out[dst] = activated;+ }+ } else if (tap_count > 0) {+ const size_t tap = size_t(token_idx - suffix_len);+ if (tap >= size_t(tap_count)) {+ return;+ }+ const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;+ const float sample = float(padded[padded_index]);+ state_out[state_offset + tap] = static_cast<T>(sample);+ }+}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metaldeleted file mode 100644index 0b318ed91..000000000--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal+++ /dev/null@@ -1,289 +0,0 @@-#include <metal_stdlib>-#include "../definitions.metal"-#include "ssm_common.h"--using namespace metal;--constant int activation_type [[function_constant(0)]];-constant bool has_bias [[function_constant(1)]];--constant int ACTIVATION_IDENTITY = 0;-constant int ACTIVATION_SILU = 1;-constant int ACTIVATION_GELU = 2;--constant uint CONV_SCAN_THREADS = 32u;--template <typename T>-inline T apply_gelu(T x) {- float xf = float(x);- return static_cast<T>(- 0.5f * xf *- (1.0f + fast::tanh(0.797885f * (xf + 0.044715f * xf * xf * xf)))- );-}--template <typename T>-inline T apply_activation_fn(T x, int activation_type) {- if (activation_type == ACTIVATION_SILU) {- return apply_silu(x);- } else if (activation_type == ACTIVATION_GELU) {- return apply_gelu(x);- } else {- return x; // Identity- }-}--template <typename T>-[[kernel, max_total_threads_per_threadgroup(32)]]-void conv1d_pack_prefix_kernel(- device const T* state_in [[buffer(0)]],- device const T* x [[buffer(1)]],- device T* padded [[buffer(2)]],- constant const size_t& state_stride [[buffer(3)]],- constant const size_t& row_stride [[buffer(4)]],- constant const size_t& suffix_len [[buffer(5)]],- constant const uint& num_channels [[buffer(6)]],- uint2 grid_idx [[thread_position_in_grid]]-) {- const uint channel_idx = grid_idx.x;- const uint row_idx = grid_idx.y;- const size_t padded_rows = state_stride + suffix_len;- if (channel_idx >= num_channels || row_idx >= padded_rows) {- return;- }- const size_t padded_index =- size_t(row_idx) * row_stride + size_t(channel_idx);- if (row_idx < state_stride) {- const size_t state_index =- size_t(channel_idx) * state_stride + size_t(row_idx);- padded[padded_index] = state_in[state_index];- } else {- const size_t token = row_idx - state_stride;- const size_t x_index = size_t(token) * row_stride + size_t(channel_idx);- padded[padded_index] = x[x_index];- }-}--template <typename T>-[[kernel, max_total_threads_per_threadgroup(32)]]-void conv1d_scan_kernel(- device const T* padded [[buffer(0)]], // (prefix+suffix, channels)- device const T* w [[buffer(1)]], // (channels, kernel)- device const T* b- [[buffer(2), function_constant(has_bias)]], // optional (channels)- device T* x_out [[buffer(3)]], // (suffix, inner_dim)- device T* b_out [[buffer(4)]], // (suffix, proj_dim)- device T* c_out [[buffer(5)]], // (suffix, proj_dim)- device T* state_out [[buffer(6)]], // (channels, kernel-1)- constant const size_t& suffix_len [[buffer(7)]],- constant const int& kernel_size [[buffer(8)]],- constant const size_t& row_stride [[buffer(9)]],- constant const size_t& state_stride [[buffer(10)]],- constant const uint& num_channels [[buffer(11)]],- constant const uint& inner_dim [[buffer(12)]],- constant const uint& proj_dim [[buffer(13)]],- uint3 grid_idx [[thread_position_in_grid]]-) {- const int kernel_value = kernel_size;- if (kernel_value <= 0) {- return;- }-- const int tap_count = max(kernel_value - 1, 0);- const size_t work_len = suffix_len + size_t(tap_count);-- const uint token_idx = grid_idx.x;- const uint channel_idx = grid_idx.y;- if (channel_idx >= num_channels || token_idx >= work_len) {- return;- }-- const size_t state_offset = size_t(channel_idx) * state_stride;- const size_t weight_offset = size_t(channel_idx) * size_t(kernel_value);- const device T* w_row = w + weight_offset;-- if (token_idx < suffix_len) {- float acc = 0.0f;- if (has_bias) {- acc = float(b[channel_idx]);- }- for (int tap = 0; tap < kernel_value; ++tap) {- const size_t padded_idx = size_t(token_idx) + size_t(tap);- float sample = 0.0f;- const size_t padded_index = size_t(padded_idx) * row_stride + channel_idx;- sample = float(padded[padded_index]);- acc += float(w_row[tap]) * sample;- }-- const T activated =- apply_activation_fn(static_cast<T>(acc), activation_type);- if (channel_idx < inner_dim) {- const size_t dst = size_t(token_idx) * inner_dim + channel_idx;- x_out[dst] = activated;- } else if (channel_idx < inner_dim + proj_dim) {- const size_t dst =- size_t(token_idx) * proj_dim + (channel_idx - inner_dim);- b_out[dst] = activated;- } else if (channel_idx < inner_dim + 2 * proj_dim) {- const size_t dst =- size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);- c_out[dst] = activated;- }- } else if (tap_count > 0) {- const size_t tap = size_t(token_idx - suffix_len);- if (tap >= size_t(tap_count)) {- return;- }- const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;- const float sample = float(padded[padded_index]);- state_out[state_offset + tap] = static_cast<T>(sample);- }-}--template <typename T>-[[kernel, max_total_threads_per_threadgroup(32)]]-void conv1d_decode_kernel(- device const T* x [[buffer(0)]],- device const T* w [[buffer(1)]],- device const T* b [[buffer(2), function_constant(has_bias)]],- device const T* state [[buffer(3)]],- device T* x_out [[buffer(4)]],- device T* b_out [[buffer(5)]],- device T* c_out [[buffer(6)]],- device T* next_state [[buffer(7)]],- constant const int& kernel_size [[buffer(8)]],- constant const size_t& row_stride [[buffer(9)]],- constant const size_t& state_stride [[buffer(10)]],- constant const uint& num_channels [[buffer(11)]],- constant const size_t& suffix_len [[buffer(12)]],- constant const uint& inner_dim [[buffer(13)]],- constant const uint& proj_dim [[buffer(14)]],- uint2 grid_idx [[thread_position_in_grid]]-) {- const int kernel_value = kernel_size;- if (kernel_value <= 0) {- return;- }-- const uint token_idx = grid_idx.x;- const uint channel_idx = grid_idx.y;- if (channel_idx >= num_channels || token_idx >= suffix_len) {- return;- }-- const size_t x_idx = size_t(token_idx) * row_stride + size_t(channel_idx);- const size_t state_offset = size_t(channel_idx) * state_stride;- const device T* w_row = w + size_t(channel_idx) * size_t(kernel_value);-- float acc = 0.0f;- if (has_bias) {- acc = float(b[channel_idx]);- }- const int state_taps = max(kernel_value - 1, 0);- for (int tap = 0; tap < state_taps; ++tap) {- const size_t state_index = state_offset + size_t(tap);- const float sample = float(state[state_index]);- acc += float(w_row[tap]) * sample;- }-- const float current = float(x[x_idx]);- const int current_tap_index = state_taps;- if (current_tap_index < kernel_value) {- acc += float(w_row[current_tap_index]) * current;- }-- const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);- if (channel_idx < inner_dim) {- const size_t dst = size_t(token_idx) * inner_dim + channel_idx;- x_out[dst] = activated;- } else if (channel_idx < inner_dim + proj_dim) {- const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim);- b_out[dst] = activated;- } else if (channel_idx < inner_dim + 2 * proj_dim) {- const size_t dst =- size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);- c_out[dst] = activated;- }-- if (state_taps == 0) {- return;- }-- for (int tap = 0; tap < state_taps - 1; ++tap) {- const size_t dst_index = state_offset + size_t(tap);- const size_t src_index = state_offset + size_t(tap + 1);- next_state[dst_index] = state[src_index];- }-- const size_t tail_index = state_offset + size_t(state_taps - 1);- next_state[tail_index] = static_cast<T>(current);-}--#define instantiate_conv1d_pack_prefix_kernel(type_name, type) \- template [[host_name("conv1d_pack_prefix_kernel_" #type_name)]] \- kernel void conv1d_pack_prefix_kernel<type>( \- device const type* state_in [[buffer(0)]], \- device const type* x [[buffer(1)]], \- device type* padded [[buffer(2)]], \- constant const size_t& state_stride [[buffer(3)]], \- constant const size_t& row_stride [[buffer(4)]], \- constant const size_t& suffix_len [[buffer(5)]], \- constant const uint& num_channels [[buffer(6)]], \- uint2 grid_idx [[thread_position_in_grid]] \- );--#define instantiate_conv1d_scan_kernel(type_name, type) \- template [[host_name("conv1d_scan_kernel_" #type_name)]] \- kernel void conv1d_scan_kernel<type>( \- device const type* padded [[buffer(0)]], \- device const type* w [[buffer(1)]], \- device const type* b [[buffer(2), function_constant(has_bias)]], \- device type* x_out [[buffer(3)]], \- device type* b_out [[buffer(4)]], \- device type* c_out [[buffer(5)]], \- device type* state_out [[buffer(6)]], \- constant const size_t& suffix_len [[buffer(7)]], \- constant const int& kernel_size [[buffer(8)]], \- constant const size_t& row_stride [[buffer(9)]], \- constant const size_t& state_stride [[buffer(10)]], \- constant const uint& num_channels [[buffer(11)]], \- constant const uint& inner_dim [[buffer(12)]], \- constant const uint& proj_dim [[buffer(13)]], \- uint3 grid_idx [[thread_position_in_grid]] \- );--instantiate_conv1d_pack_prefix_kernel(float, float);-instantiate_conv1d_pack_prefix_kernel(bfloat, bfloat);-instantiate_conv1d_pack_prefix_kernel(half, half);--instantiate_conv1d_scan_kernel(float, float);-instantiate_conv1d_scan_kernel(bfloat, bfloat);-instantiate_conv1d_scan_kernel(half, half);--#define instantiate_conv1d_decode_kernel(type_name, type) \- template [[host_name("conv1d_decode_kernel_" #type_name)]] \- kernel void conv1d_decode_kernel<type>( \- device const type* x [[buffer(0)]], \- device const type* w [[buffer(1)]], \- device const type* b [[buffer(2), function_constant(has_bias)]], \- device const type* state [[buffer(3)]], \- device type* x_out [[buffer(4)]], \- device type* b_out [[buffer(5)]], \- device type* c_out [[buffer(6)]], \- device type* next_state [[buffer(7)]], \- constant const int& kernel_size [[buffer(8)]], \- constant const size_t& row_stride [[buffer(9)]], \- constant const size_t& state_stride [[buffer(10)]], \- constant const uint& num_channels [[buffer(11)]], \- constant const size_t& suffix_len [[buffer(12)]], \- constant const uint& inner_dim [[buffer(13)]], \- constant const uint& proj_dim [[buffer(14)]], \- uint2 grid_idx [[thread_position_in_grid]] \- );--#undef instantiate_conv1d_pack_prefix_kernel-#undef instantiate_conv1d_scan_kernel-instantiate_conv1d_decode_kernel(float, float);-instantiate_conv1d_decode_kernel(bfloat, bfloat);-instantiate_conv1d_decode_kernel(half, half);-#undef instantiate_conv1d_decode_kerneldiff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rsdeleted file mode 100644index 6056644ba..000000000--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs+++ /dev/null@@ -1,282 +0,0 @@-use super::SSMKernelError;-use crate::{- DataType,- backends::metal::{- FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,- MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained, data_type::MetalDataTypeExt,- metal_extensions::ComputeEncoderSetValue,- },- config::Activation,-};--const CONV1D_SCAN_THREADGROUP_SIZE: usize = 32;--const ACTIVATION_IDENTITY: i32 = 0;-const ACTIVATION_SILU: i32 = 1;-const ACTIVATION_GELU: i32 = 2;--fn activation_to_int(activation: &Activation) -> i32 {- match activation {- Activation::Identity => ACTIVATION_IDENTITY,- Activation::SiLU {- ..- } => ACTIVATION_SILU,- Activation::Gelu => ACTIVATION_GELU,- }-}--fn make_function_constants(- activation: &Activation,- has_bias: bool,-) -> Retained<MTLFunctionConstantValues> {- let function_constants = MTLFunctionConstantValues::new();- let activation_type = activation_to_int(activation);-- function_constants.set_value(&activation_type, 0);- function_constants.set_value(&has_bias, 1);-- function_constants-}--pub struct Conv1dScanKernel {- pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- pack_pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- decode_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- decode_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,-}--pub struct Conv1dScanArguments<'a> {- pub padded: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0) [prefix+suffix, channels]- pub w: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1) [channels, kernel]- pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(2) [channels]- pub x_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) [suffix, inner_dim]- pub b_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) [suffix, proj_dim]- pub c_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) [suffix, proj_dim]- pub state_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(6) [channels, kernel-1]- pub suffix_len: usize,- pub kernel_size: i32,- pub row_stride: usize,- pub state_stride: usize,- pub channels: usize,- pub inner_dim: usize,- pub proj_dim: usize,-}--pub struct Conv1dPackArguments<'a> {- pub state_in: &'a ProtocolObject<dyn MTLBuffer>,- pub x: &'a ProtocolObject<dyn MTLBuffer>,- pub padded: &'a ProtocolObject<dyn MTLBuffer>,- pub state_stride: usize,- pub row_stride: usize,- pub suffix_len: usize,- pub channels: usize,-}--pub struct Conv1dDecodeArguments<'a> {- pub x: &'a ProtocolObject<dyn MTLBuffer>,- pub w: &'a ProtocolObject<dyn MTLBuffer>,- pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>,- pub state: &'a ProtocolObject<dyn MTLBuffer>,- pub x_out: &'a ProtocolObject<dyn MTLBuffer>,- pub b_out: &'a ProtocolObject<dyn MTLBuffer>,- pub c_out: &'a ProtocolObject<dyn MTLBuffer>,- pub next_state: &'a ProtocolObject<dyn MTLBuffer>,- pub suffix_len: usize,- pub kernel_size: i32,- pub row_stride: usize,- pub state_stride: usize,- pub channels: usize,- pub inner_dim: usize,- pub proj_dim: usize,-}--impl Conv1dScanKernel {- pub fn new(- context: &MTLContext,- data_type: DataType,- activation: &Activation,- ) -> Result<Self, SSMKernelError> {- let fn_name = format!("conv1d_scan_kernel_{}", data_type.metal_type());- let pack_name = format!("conv1d_pack_prefix_kernel_{}", data_type.metal_type());- let decode_name = format!("conv1d_decode_kernel_{}", data_type.metal_type());- let activation_id = activation_to_int(activation);- let pipeline_no_bias = {- let function_constants = make_function_constants(activation, false);- let cache_key = format!("{}_act_{}_bias_0", fn_name, activation_id);- context- .compute_pipeline_state_cached(&cache_key, &fn_name, Some(&function_constants))- .map_err(SSMKernelError::MetalError)?- };- let pipeline_with_bias = {- let function_constants = make_function_constants(activation, true);- let cache_key = format!("{}_act_{}_bias_1", fn_name, activation_id);- context- .compute_pipeline_state_cached(&cache_key, &fn_name, Some(&function_constants))- .map_err(SSMKernelError::MetalError)?- };- let pack_pipeline = context.compute_pipeline_state(&pack_name, None).map_err(SSMKernelError::MetalError)?;- let decode_pipeline_no_bias = {- let function_constants = make_function_constants(activation, false);- let cache_key = format!("{}_act_{}_bias_0", decode_name, activation_id);- context- .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))- .map_err(SSMKernelError::MetalError)?- };- let decode_pipeline_with_bias = {- let function_constants = make_function_constants(activation, true);- let cache_key = format!("{}_act_{}_bias_1", decode_name, activation_id);- context- .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))- .map_err(SSMKernelError::MetalError)?- };- Ok(Self {- pipeline_no_bias,- pipeline_with_bias,- pack_pipeline,- decode_pipeline_no_bias,- decode_pipeline_with_bias,- })- }-- pub fn encode_pack(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: Conv1dPackArguments,- ) -> Result<(), SSMKernelError> {- if args.channels == 0 || args.state_stride == 0 {- return Ok(());- }-- compute_encoder.set_compute_pipeline_state(&self.pack_pipeline);- compute_encoder.set_buffer(Some(args.state_in), 0, 0);- compute_encoder.set_buffer(Some(args.x), 0, 1);- compute_encoder.set_buffer(Some(args.padded), 0, 2);- compute_encoder.set_value(&args.state_stride, 3);- compute_encoder.set_value(&args.row_stride, 4);- compute_encoder.set_value(&args.suffix_len, 5);- let channels_u32 = args.channels as u32;- compute_encoder.set_value(&channels_u32, 6);-- let threads_per_threadgroup = MTLSize {- width: CONV1D_SCAN_THREADGROUP_SIZE,- height: 1,- depth: 1,- };- let padded_rows = args.state_stride + args.suffix_len;- let total_threads = MTLSize {- width: args.channels,- height: padded_rows,- depth: 1,- };- compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);- Ok(())- }-- pub fn encode_decode(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: Conv1dDecodeArguments,- ) -> Result<(), SSMKernelError> {- if args.channels == 0 || args.suffix_len == 0 || args.kernel_size <= 0 {- return Ok(());- }-- let has_bias = args.b.is_some();- let pipeline = if has_bias {- &self.decode_pipeline_with_bias- } else {- &self.decode_pipeline_no_bias- };- compute_encoder.set_compute_pipeline_state(pipeline);- compute_encoder.set_buffer(Some(args.x), 0, 0);- compute_encoder.set_buffer(Some(args.w), 0, 1);- if has_bias {- compute_encoder.set_buffer(args.b, 0, 2);- }- compute_encoder.set_buffer(Some(args.state), 0, 3);- compute_encoder.set_buffer(Some(args.x_out), 0, 4);- compute_encoder.set_buffer(Some(args.b_out), 0, 5);- compute_encoder.set_buffer(Some(args.c_out), 0, 6);- compute_encoder.set_buffer(Some(args.next_state), 0, 7);- compute_encoder.set_value(&args.kernel_size, 8);- compute_encoder.set_value(&args.row_stride, 9);- compute_encoder.set_value(&args.state_stride, 10);- let channels_u32 = args.channels as u32;- compute_encoder.set_value(&channels_u32, 11);- compute_encoder.set_value(&args.suffix_len, 12);- let inner_dim_u32 = args.inner_dim as u32;- compute_encoder.set_value(&inner_dim_u32, 13);- let proj_dim_u32 = args.proj_dim as u32;- compute_encoder.set_value(&proj_dim_u32, 14);-- let threads_per_threadgroup = MTLSize {- width: CONV1D_SCAN_THREADGROUP_SIZE,- height: 1,- depth: 1,- };- let total_threads = MTLSize {- width: args.suffix_len,- height: args.channels,- depth: 1,- };- compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);- Ok(())- }-- pub fn encode(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: Conv1dScanArguments,- ) -> Result<(), SSMKernelError> {- if args.channels == 0 || args.suffix_len == 0 || args.kernel_size <= 0 {- return Ok(());- }-- let has_bias = args.b.is_some();- let pipeline = if has_bias {- &self.pipeline_with_bias- } else {- &self.pipeline_no_bias- };- compute_encoder.set_compute_pipeline_state(pipeline);-- compute_encoder.set_buffer(Some(args.padded), 0, 0);- compute_encoder.set_buffer(Some(args.w), 0, 1);- if has_bias {- compute_encoder.set_buffer(args.b, 0, 2);- }- compute_encoder.set_buffer(Some(args.x_out), 0, 3);- compute_encoder.set_buffer(Some(args.b_out), 0, 4);- compute_encoder.set_buffer(Some(args.c_out), 0, 5);- compute_encoder.set_buffer(Some(args.state_out), 0, 6);-- compute_encoder.set_value(&args.suffix_len, 7);- compute_encoder.set_value(&args.kernel_size, 8);- compute_encoder.set_value(&args.row_stride, 9);- compute_encoder.set_value(&args.state_stride, 10);- let channels_u32 = args.channels as u32;- compute_encoder.set_value(&channels_u32, 11);- let inner_dim_u32 = args.inner_dim as u32;- compute_encoder.set_value(&inner_dim_u32, 12);- let proj_dim_u32 = args.proj_dim as u32;- compute_encoder.set_value(&proj_dim_u32, 13);-- let suffix = args.suffix_len;- let tap_count = args.kernel_size.saturating_sub(1) as usize;- let work_len = suffix + tap_count;- let channels = args.channels;- let threads_per_threadgroup = MTLSize {- width: CONV1D_SCAN_THREADGROUP_SIZE,- height: 1,- depth: 1,- };- let total_threads = MTLSize {- width: work_len,- height: channels,- depth: 1,- };- compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rsindex c64f8af63..a87e87362 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs@@ -1,15 +1,2 @@-use crate::backends::metal::MTLError;--#[derive(Debug, thiserror::Error)]-pub enum SSMKernelError {- #[error("Metal error: {0}")]- MetalError(#[from] MTLError),- #[error("Function not found: {0}")]- FunctionNotFound(String),-}--pub mod conv1d_scan;pub mod ssd_prefill;--pub use conv1d_scan::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel};pub use ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode};diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.hindex d96d207a9..1e105a86e 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h@@ -1,3 +1,7 @@+#define ACTIVATION_IDENTITY 0+#define ACTIVATION_SILU 1+#define ACTIVATION_GELU 2+template <typename T>inline T apply_silu(T x) {float xf = float(x);@@ -13,4 +17,24 @@ inline T softplus(T x) {return x;}return static_cast<T>(log(1.0f + fast::exp(xf)));+}++template <typename T>+inline T apply_gelu(T x) {+ float xf = float(x);+ return static_cast<T>(+ 0.5f * xf *+ (1.0f + fast::tanh(0.797885f * (xf + 0.044715f * xf * xf * xf)))+ );+}++template <typename T>+inline T apply_activation_fn(T x, int activation_type) {+ if (activation_type == ACTIVATION_SILU) {+ return apply_silu(x);+ } else if (activation_type == ACTIVATION_GELU) {+ return apply_gelu(x);+ } else {+ return x; // Identity+ }}\ No newline at end of filediff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rsindex a4bd00717..aa150d51b 100644--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs@@ -9,16 +9,15 @@ use metal::{use uzu::{DataType,backends::{- common::Context,+ common::{Context, kernel::Conv1dScanKernel},metal::{MTLBuffer, MTLContext, ProtocolObject,- kernel::ssm::{- SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode,- conv1d_scan::{Conv1dScanArguments, Conv1dScanKernel},+ kernel::{+ dsl::Conv1dScanMetalKernel,+ ssm::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},},},- config::Activation,};const STORAGE_MODE: MTLResourceOptions = MTLResourceOptions::STORAGE_MODE_SHARED;@@ -261,7 +260,7 @@ fn run_prefill_kernel_mode(fn run_conv_scan_once(ctx: &MTLContext,- kernel: &Conv1dScanKernel,+ kernel: &Conv1dScanMetalKernel,suffix_len: usize,channels: usize,kernel_size: i32,@@ -319,26 +318,25 @@ fn run_conv_scan_once(write_buffer(&padded_buf, &host);}- let args = Conv1dScanArguments {- padded: &padded_buf,- w: &w_buf,- b: Some(&b_buf),- x_out: &y_buf,- b_out: &y_buf,- c_out: &y_buf,- state_out: scratch_buf.as_ref().unwrap_or(&state_buf),- suffix_len,- kernel_size,- row_stride: channels,- state_stride: tap_count,- channels,- inner_dim: channels,- proj_dim: 0,- };-let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&encoder, args).unwrap();+ kernel.encode(+ &padded_buf,+ &w_buf,+ Some(&b_buf),+ &y_buf,+ &y_buf,+ &y_buf,+ scratch_buf.as_ref().unwrap_or(&state_buf),+ suffix_len as u32,+ kernel_size as u32,+ channels as u32,+ tap_count as u32,+ channels as u32,+ channels as u32,+ 0u32,+ &encoder,+ );encoder.end_encoding();if let Some(ref scratch) = scratch_buf {@@ -464,8 +462,7 @@ fn conv1d_scan_is_deterministic() {eprintln!("Skipping conv1d scan determinism test: no Metal device");return;};- let activation = Activation::Identity;- let kernel = Conv1dScanKernel::new(&ctx, DataType::F32, &activation).unwrap();+ let kernel = Conv1dScanMetalKernel::new(&ctx, DataType::F32, 0u32, true).unwrap();let suffix_len = 192usize;let channels = 8usize;