Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dslify-conv1d (#184)

Aleksandr Golokozc83fe37638af
diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
index 0023390ad..290f34405 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
@@ -2,21 +2,22 @@
use std::env;
+use objc2::Message;
+
use super::{EncodableBlock, Metal, transformer_layer};
use crate::{
- DataType,
+ Activation, DataType,
backends::{
- common::kernel::{SSDUpdateKernel, SplitInProjKernel},
+ common::kernel::{Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel},
metal::{
MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,
Retained,
kernel::{
- dsl::{SSDUpdateMetalKernel, SplitInProjMetalKernel},
- ssm::{
- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
- conv1d_scan::Conv1dDecodeArguments,
- ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
+ dsl::{
+ Conv1dDecodeMetalKernel, Conv1dPackMetalKernel, Conv1dScanMetalKernel, SSDUpdateMetalKernel,
+ SplitInProjMetalKernel,
},
+ ssm::ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
},
},
},
@@ -32,7 +33,9 @@ pub(crate) struct MambaMixer {
in_projection: Box<dyn EncodableBlock<Metal>>,
out_projection: Box<dyn EncodableBlock<Metal>>,
split_inproj: SplitInProjMetalKernel,
- conv_scan: Conv1dScanKernel,
+ conv_decode: Conv1dDecodeMetalKernel,
+ conv_pack: Conv1dPackMetalKernel,
+ conv_scan: Conv1dScanMetalKernel,
ssd_prefill: SSDPrefillKernels,
ssd_update: SSDUpdateMetalKernel,
conv_weight: MetalArray,
@@ -99,8 +102,21 @@ impl MambaMixer {
let split_inproj =
SplitInProjMetalKernel::new(mtl_context, data_type).expect("Failed to create split in-projection kernel");
- let conv_scan = Conv1dScanKernel::new(mtl_context, data_type, &mamba_config.activation)
- .expect("Failed to create conv scan kernel");
+ let conv_scan = Conv1dScanMetalKernel::new(
+ mtl_context,
+ data_type,
+ Self::activation_to_int(&mamba_config.activation),
+ mamba_config.conv_config.has_biases,
+ )
+ .expect("Failed to create conv scan kernel");
+ let conv_decode = Conv1dDecodeMetalKernel::new(
+ mtl_context,
+ data_type,
+ Self::activation_to_int(&mamba_config.activation),
+ mamba_config.conv_config.has_biases,
+ )
+ .expect("Failed to create conv decode kernel");
+ let conv_pack = Conv1dPackMetalKernel::new(mtl_context, data_type).expect("Failed to create conv pack kernel");
let ssd_prefill = SSDPrefillKernels::new(mtl_context, data_type).expect("Failed to create SSD prefill kernel");
let ssd_update = SSDUpdateMetalKernel::new(mtl_context, data_type).expect("Failed to create SSD decode kernel");
let prefill_mode = resolve_prefill_mode_from_env();
@@ -111,6 +127,8 @@ impl MambaMixer {
in_projection,
out_projection,
split_inproj,
+ conv_decode,
+ conv_pack,
conv_scan,
ssd_prefill,
ssd_update,
@@ -217,10 +235,7 @@ impl MambaMixer {
let weight_storage = self.conv_weight.clone();
let weight_buf = weight_storage.buffer().clone();
- let bias_buf = self.conv_bias.as_ref().map(|arr| {
- let storage = arr.clone();
- storage.buffer().clone()
- });
+ let bias_buf = self.conv_bias.as_ref().map(|arr| arr.buffer());
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
@@ -229,28 +244,26 @@ impl MambaMixer {
drop(conv_state);
if suffix_length == 1 {
- self.conv_scan
- .encode_decode(
- encoder,
- Conv1dDecodeArguments {
- x: &input_buf,
- w: &weight_buf,
- b: bias_buf.as_deref(),
- state: &state_buf,
- x_out: &x_buf,
- b_out: &b_buf,
- c_out: &c_buf,
- next_state: &state_buf,
- suffix_len: suffix_length,
- kernel_size: self.config.kernel_size as i32,
- row_stride: conv_dim,
- state_stride,
- channels: conv_dim,
- inner_dim,
- proj_dim,
- },
- )
- .expect("Failed to encode conv decode kernel");
+ if conv_dim > 0 && self.config.kernel_size > 0 {
+ self.conv_decode.encode(
+ &input_buf,
+ &weight_buf,
+ bias_buf,
+ &state_buf,
+ &x_buf,
+ &b_buf,
+ &c_buf,
+ &state_buf,
+ self.config.kernel_size as u32,
+ conv_dim as u32,
+ state_stride as u32,
+ conv_dim as u32,
+ suffix_length as u32,
+ inner_dim as u32,
+ proj_dim as u32,
+ &encoder,
+ );
+ }
} else {
let padded_buf = if state_stride > 0 {
let array = state.conv_padded_buffer().expect("Missing conv padded buffer");
@@ -258,49 +271,42 @@ impl MambaMixer {
let buf = borrow.buffer().clone();
drop(borrow);
- self.conv_scan
- .encode_pack(
- encoder,
- Conv1dPackArguments {
- state_in: &state_buf,
- x: &input_buf,
- padded: &buf,
- state_stride,
- row_stride: conv_dim,
- suffix_len: suffix_length,
- channels: conv_dim,
- },
- )
- .expect("Failed to encode conv pack kernel");
+ self.conv_pack.encode(
+ &state_buf,
+ &input_buf,
+ &buf,
+ state_stride as u32,
+ conv_dim as u32,
+ suffix_length as u32,
+ conv_dim as u32,
+ &encoder,
+ );
Some(buf)
} else {
None
};
- let conv_source = padded_buf.as_deref().unwrap_or(&input_buf);
-
- self.conv_scan
- .encode(
- encoder,
- Conv1dScanArguments {
- padded: conv_source,
- w: &weight_buf,
- b: bias_buf.as_deref(),
- x_out: &x_buf,
- b_out: &b_buf,
- c_out: &c_buf,
- state_out: &state_buf,
- suffix_len: suffix_length,
- kernel_size: self.config.kernel_size as i32,
- row_stride: conv_dim,
- state_stride,
- channels: conv_dim,
- inner_dim,
- proj_dim,
- },
+ if conv_dim > 0 && self.config.kernel_size > 0 {
+ let conv_source = padded_buf.as_deref().unwrap_or(&input_buf).retain();
+ self.conv_scan.encode(
+ &conv_source,
+ &weight_buf,
+ bias_buf,
+ &x_buf,
+ &b_buf,
+ &c_buf,
+ &state_buf,
+ suffix_length as u32,
+ self.config.kernel_size as u32,
+ conv_dim as u32,
+ state_stride as u32,
+ conv_dim as u32,
+ inner_dim as u32,
+ proj_dim as u32,
+ &encoder,
)
- .expect("Failed to encode conv scan kernel");
+ }
}
}
@@ -441,6 +447,16 @@ impl MambaMixer {
encoder,
);
}
+
+ fn activation_to_int(activation: &Activation) -> u32 {
+ match activation {
+ Activation::Identity => 0,
+ Activation::SiLU {
+ ..
+ } => 1,
+ Activation::Gelu => 2,
+ }
+ }
}
fn resolve_prefill_mode_from_env() -> SSDPrefillMode {
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 78e0e4d27..edf2b6d24 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -27,9 +27,7 @@ pub use moe::{
MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,
MoeScatterWithMapArguments,
};
-pub use ssm::{
- Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, SSDPrefillArguments, SSDPrefillMode, SSMKernelError,
-};
+pub use ssm::{SSDPrefillArguments, SSDPrefillMode};
impl MatmulKernels for dsl::MetalKernels {
type FullPrecisionMatmulKernel = MatmulKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
new file mode 100644
index 000000000..304da7f7c
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
@@ -0,0 +1,164 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+#include "ssm_common.h"
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(Conv1dPack)(
+ device const T* state_in,
+ device const T* x,
+ device T* padded,
+ constant const uint& state_stride,
+ constant const uint& row_stride,
+ constant const uint& suffix_len,
+ constant const uint& num_channels,
+ const uint channel_idx AXIS(num_channels, 32),
+ const uint row_idx AXIS(state_stride + suffix_len, 1)
+) {
+ const uint padded_index = row_idx * row_stride + channel_idx;
+ if (row_idx < state_stride) {
+ const uint state_index = channel_idx * state_stride + row_idx;
+ padded[padded_index] = state_in[state_index];
+ } else {
+ const uint token = row_idx - state_stride;
+ const uint x_index = token * row_stride + channel_idx;
+ padded[padded_index] = x[x_index];
+ }
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(Conv1dDecode)(
+ device const T* x,
+ device const T* w,
+ device const T* b OPTIONAL(has_bias),
+ device const T* state,
+ device T* x_out,
+ device T* b_out,
+ device T* c_out,
+ device T* next_state,
+ constant const uint& kernel_size,
+ constant const uint& row_stride,
+ constant const uint& state_stride,
+ constant const uint& num_channels,
+ constant const uint& suffix_len,
+ constant const uint& inner_dim,
+ constant const uint& proj_dim,
+ const uint activation_type SPECIALIZE,
+ const bool has_bias SPECIALIZE,
+ const uint token_idx AXIS(suffix_len, 32),
+ const uint channel_idx AXIS(num_channels, 1)
+) {
+ const uint x_idx = token_idx * row_stride + channel_idx;
+ const uint state_offset = channel_idx * state_stride;
+ const device T* w_row = w + channel_idx * kernel_size;
+
+ float acc = 0.0f;
+ if (has_bias) {
+ acc = float(b[channel_idx]);
+ }
+
+ const uint state_taps = max(kernel_size - 1, 0u);
+ for (uint tap = 0; tap < state_taps; ++tap) {
+ const uint state_index = state_offset + tap;
+ const float sample = float(state[state_index]);
+ acc += float(w_row[tap]) * sample;
+ }
+
+ const float current = float(x[x_idx]);
+ const uint current_tap_index = state_taps;
+ if (current_tap_index < kernel_size) {
+ acc += float(w_row[current_tap_index]) * current;
+ }
+
+ const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);
+ if (channel_idx < inner_dim) {
+ const uint dst = token_idx * inner_dim + channel_idx;
+ x_out[dst] = activated;
+ } else if (channel_idx < inner_dim + proj_dim) {
+ const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);
+ b_out[dst] = activated;
+ } else if (channel_idx < inner_dim + 2 * proj_dim) {
+ const uint dst =
+ token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
+ c_out[dst] = activated;
+ }
+
+ if (state_taps == 0) {
+ return;
+ }
+
+ for (uint tap = 0; tap < state_taps - 1; ++tap) {
+ const size_t dst_index = state_offset + size_t(tap);
+ const size_t src_index = state_offset + size_t(tap + 1);
+ next_state[dst_index] = state[src_index];
+ }
+
+ const uint tail_index = state_offset + state_taps - 1;
+ next_state[tail_index] = static_cast<T>(current);
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(Conv1dScan)(
+ device const T* padded, // (prefix+suffix, channels)
+ device const T* w, // (channels, kernel)
+ device const T* b OPTIONAL(has_bias), // optional (channels)
+ device T* x_out, // (suffix, inner_dim)
+ device T* b_out, // (suffix, proj_dim)
+ device T* c_out, // (suffix, proj_dim)
+ device T* state_out, // (channels, kernel-1)
+ constant const uint& suffix_len,
+ constant const uint& kernel_size,
+ constant const uint& row_stride,
+ constant const uint& state_stride,
+ constant const uint& num_channels,
+ constant const uint& inner_dim,
+ constant const uint& proj_dim,
+ const uint activation_type SPECIALIZE,
+ const bool has_bias SPECIALIZE,
+ const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 32),
+ const uint channel_idx AXIS(num_channels, 1)
+) {
+ const uint tap_count = max(kernel_size - 1, 0u);
+ const uint state_offset = channel_idx * state_stride;
+ const uint weight_offset = channel_idx * kernel_size;
+ const device T* w_row = w + weight_offset;
+
+ if (token_idx < suffix_len) {
+ float acc = 0.0f;
+ if (has_bias) {
+ acc = float(b[channel_idx]);
+ }
+ for (uint tap = 0; tap < kernel_size; ++tap) {
+ const uint padded_idx = size_t(token_idx) + size_t(tap);
+ float sample = 0.0f;
+ const uint padded_index = size_t(padded_idx) * row_stride + channel_idx;
+ sample = float(padded[padded_index]);
+ acc += float(w_row[tap]) * sample;
+ }
+
+ const T activated =
+ apply_activation_fn(static_cast<T>(acc), activation_type);
+ if (channel_idx < inner_dim) {
+ const size_t dst = size_t(token_idx) * inner_dim + channel_idx;
+ x_out[dst] = activated;
+ } else if (channel_idx < inner_dim + proj_dim) {
+ const size_t dst =
+ size_t(token_idx) * proj_dim + (channel_idx - inner_dim);
+ b_out[dst] = activated;
+ } else if (channel_idx < inner_dim + 2 * proj_dim) {
+ const size_t dst =
+ size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);
+ c_out[dst] = activated;
+ }
+ } else if (tap_count > 0) {
+ const size_t tap = size_t(token_idx - suffix_len);
+ if (tap >= size_t(tap_count)) {
+ return;
+ }
+ const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;
+ const float sample = float(padded[padded_index]);
+ state_out[state_offset + tap] = static_cast<T>(sample);
+ }
+}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
deleted file mode 100644
index 0b318ed91..000000000
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
+++ /dev/null
@@ -1,289 +0,0 @@
-#include <metal_stdlib>
-#include "../definitions.metal"
-#include "ssm_common.h"
-
-using namespace metal;
-
-constant int activation_type [[function_constant(0)]];
-constant bool has_bias [[function_constant(1)]];
-
-constant int ACTIVATION_IDENTITY = 0;
-constant int ACTIVATION_SILU = 1;
-constant int ACTIVATION_GELU = 2;
-
-constant uint CONV_SCAN_THREADS = 32u;
-
-template <typename T>
-inline T apply_gelu(T x) {
- float xf = float(x);
- return static_cast<T>(
- 0.5f * xf *
- (1.0f + fast::tanh(0.797885f * (xf + 0.044715f * xf * xf * xf)))
- );
-}
-
-template <typename T>
-inline T apply_activation_fn(T x, int activation_type) {
- if (activation_type == ACTIVATION_SILU) {
- return apply_silu(x);
- } else if (activation_type == ACTIVATION_GELU) {
- return apply_gelu(x);
- } else {
- return x; // Identity
- }
-}
-
-template <typename T>
-[[kernel, max_total_threads_per_threadgroup(32)]]
-void conv1d_pack_prefix_kernel(
- device const T* state_in [[buffer(0)]],
- device const T* x [[buffer(1)]],
- device T* padded [[buffer(2)]],
- constant const size_t& state_stride [[buffer(3)]],
- constant const size_t& row_stride [[buffer(4)]],
- constant const size_t& suffix_len [[buffer(5)]],
- constant const uint& num_channels [[buffer(6)]],
- uint2 grid_idx [[thread_position_in_grid]]
-) {
- const uint channel_idx = grid_idx.x;
- const uint row_idx = grid_idx.y;
- const size_t padded_rows = state_stride + suffix_len;
- if (channel_idx >= num_channels || row_idx >= padded_rows) {
- return;
- }
- const size_t padded_index =
- size_t(row_idx) * row_stride + size_t(channel_idx);
- if (row_idx < state_stride) {
- const size_t state_index =
- size_t(channel_idx) * state_stride + size_t(row_idx);
- padded[padded_index] = state_in[state_index];
- } else {
- const size_t token = row_idx - state_stride;
- const size_t x_index = size_t(token) * row_stride + size_t(channel_idx);
- padded[padded_index] = x[x_index];
- }
-}
-
-template <typename T>
-[[kernel, max_total_threads_per_threadgroup(32)]]
-void conv1d_scan_kernel(
- device const T* padded [[buffer(0)]], // (prefix+suffix, channels)
- device const T* w [[buffer(1)]], // (channels, kernel)
- device const T* b
- [[buffer(2), function_constant(has_bias)]], // optional (channels)
- device T* x_out [[buffer(3)]], // (suffix, inner_dim)
- device T* b_out [[buffer(4)]], // (suffix, proj_dim)
- device T* c_out [[buffer(5)]], // (suffix, proj_dim)
- device T* state_out [[buffer(6)]], // (channels, kernel-1)
- constant const size_t& suffix_len [[buffer(7)]],
- constant const int& kernel_size [[buffer(8)]],
- constant const size_t& row_stride [[buffer(9)]],
- constant const size_t& state_stride [[buffer(10)]],
- constant const uint& num_channels [[buffer(11)]],
- constant const uint& inner_dim [[buffer(12)]],
- constant const uint& proj_dim [[buffer(13)]],
- uint3 grid_idx [[thread_position_in_grid]]
-) {
- const int kernel_value = kernel_size;
- if (kernel_value <= 0) {
- return;
- }
-
- const int tap_count = max(kernel_value - 1, 0);
- const size_t work_len = suffix_len + size_t(tap_count);
-
- const uint token_idx = grid_idx.x;
- const uint channel_idx = grid_idx.y;
- if (channel_idx >= num_channels || token_idx >= work_len) {
- return;
- }
-
- const size_t state_offset = size_t(channel_idx) * state_stride;
- const size_t weight_offset = size_t(channel_idx) * size_t(kernel_value);
- const device T* w_row = w + weight_offset;
-
- if (token_idx < suffix_len) {
- float acc = 0.0f;
- if (has_bias) {
- acc = float(b[channel_idx]);
- }
- for (int tap = 0; tap < kernel_value; ++tap) {
- const size_t padded_idx = size_t(token_idx) + size_t(tap);
- float sample = 0.0f;
- const size_t padded_index = size_t(padded_idx) * row_stride + channel_idx;
- sample = float(padded[padded_index]);
- acc += float(w_row[tap]) * sample;
- }
-
- const T activated =
- apply_activation_fn(static_cast<T>(acc), activation_type);
- if (channel_idx < inner_dim) {
- const size_t dst = size_t(token_idx) * inner_dim + channel_idx;
- x_out[dst] = activated;
- } else if (channel_idx < inner_dim + proj_dim) {
- const size_t dst =
- size_t(token_idx) * proj_dim + (channel_idx - inner_dim);
- b_out[dst] = activated;
- } else if (channel_idx < inner_dim + 2 * proj_dim) {
- const size_t dst =
- size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);
- c_out[dst] = activated;
- }
- } else if (tap_count > 0) {
- const size_t tap = size_t(token_idx - suffix_len);
- if (tap >= size_t(tap_count)) {
- return;
- }
- const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;
- const float sample = float(padded[padded_index]);
- state_out[state_offset + tap] = static_cast<T>(sample);
- }
-}
-
-template <typename T>
-[[kernel, max_total_threads_per_threadgroup(32)]]
-void conv1d_decode_kernel(
- device const T* x [[buffer(0)]],
- device const T* w [[buffer(1)]],
- device const T* b [[buffer(2), function_constant(has_bias)]],
- device const T* state [[buffer(3)]],
- device T* x_out [[buffer(4)]],
- device T* b_out [[buffer(5)]],
- device T* c_out [[buffer(6)]],
- device T* next_state [[buffer(7)]],
- constant const int& kernel_size [[buffer(8)]],
- constant const size_t& row_stride [[buffer(9)]],
- constant const size_t& state_stride [[buffer(10)]],
- constant const uint& num_channels [[buffer(11)]],
- constant const size_t& suffix_len [[buffer(12)]],
- constant const uint& inner_dim [[buffer(13)]],
- constant const uint& proj_dim [[buffer(14)]],
- uint2 grid_idx [[thread_position_in_grid]]
-) {
- const int kernel_value = kernel_size;
- if (kernel_value <= 0) {
- return;
- }
-
- const uint token_idx = grid_idx.x;
- const uint channel_idx = grid_idx.y;
- if (channel_idx >= num_channels || token_idx >= suffix_len) {
- return;
- }
-
- const size_t x_idx = size_t(token_idx) * row_stride + size_t(channel_idx);
- const size_t state_offset = size_t(channel_idx) * state_stride;
- const device T* w_row = w + size_t(channel_idx) * size_t(kernel_value);
-
- float acc = 0.0f;
- if (has_bias) {
- acc = float(b[channel_idx]);
- }
- const int state_taps = max(kernel_value - 1, 0);
- for (int tap = 0; tap < state_taps; ++tap) {
- const size_t state_index = state_offset + size_t(tap);
- const float sample = float(state[state_index]);
- acc += float(w_row[tap]) * sample;
- }
-
- const float current = float(x[x_idx]);
- const int current_tap_index = state_taps;
- if (current_tap_index < kernel_value) {
- acc += float(w_row[current_tap_index]) * current;
- }
-
- const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);
- if (channel_idx < inner_dim) {
- const size_t dst = size_t(token_idx) * inner_dim + channel_idx;
- x_out[dst] = activated;
- } else if (channel_idx < inner_dim + proj_dim) {
- const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim);
- b_out[dst] = activated;
- } else if (channel_idx < inner_dim + 2 * proj_dim) {
- const size_t dst =
- size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);
- c_out[dst] = activated;
- }
-
- if (state_taps == 0) {
- return;
- }
-
- for (int tap = 0; tap < state_taps - 1; ++tap) {
- const size_t dst_index = state_offset + size_t(tap);
- const size_t src_index = state_offset + size_t(tap + 1);
- next_state[dst_index] = state[src_index];
- }
-
- const size_t tail_index = state_offset + size_t(state_taps - 1);
- next_state[tail_index] = static_cast<T>(current);
-}
-
-#define instantiate_conv1d_pack_prefix_kernel(type_name, type) \
- template [[host_name("conv1d_pack_prefix_kernel_" #type_name)]] \
- kernel void conv1d_pack_prefix_kernel<type>( \
- device const type* state_in [[buffer(0)]], \
- device const type* x [[buffer(1)]], \
- device type* padded [[buffer(2)]], \
- constant const size_t& state_stride [[buffer(3)]], \
- constant const size_t& row_stride [[buffer(4)]], \
- constant const size_t& suffix_len [[buffer(5)]], \
- constant const uint& num_channels [[buffer(6)]], \
- uint2 grid_idx [[thread_position_in_grid]] \
- );
-
-#define instantiate_conv1d_scan_kernel(type_name, type) \
- template [[host_name("conv1d_scan_kernel_" #type_name)]] \
- kernel void conv1d_scan_kernel<type>( \
- device const type* padded [[buffer(0)]], \
- device const type* w [[buffer(1)]], \
- device const type* b [[buffer(2), function_constant(has_bias)]], \
- device type* x_out [[buffer(3)]], \
- device type* b_out [[buffer(4)]], \
- device type* c_out [[buffer(5)]], \
- device type* state_out [[buffer(6)]], \
- constant const size_t& suffix_len [[buffer(7)]], \
- constant const int& kernel_size [[buffer(8)]], \
- constant const size_t& row_stride [[buffer(9)]], \
- constant const size_t& state_stride [[buffer(10)]], \
- constant const uint& num_channels [[buffer(11)]], \
- constant const uint& inner_dim [[buffer(12)]], \
- constant const uint& proj_dim [[buffer(13)]], \
- uint3 grid_idx [[thread_position_in_grid]] \
- );
-
-instantiate_conv1d_pack_prefix_kernel(float, float);
-instantiate_conv1d_pack_prefix_kernel(bfloat, bfloat);
-instantiate_conv1d_pack_prefix_kernel(half, half);
-
-instantiate_conv1d_scan_kernel(float, float);
-instantiate_conv1d_scan_kernel(bfloat, bfloat);
-instantiate_conv1d_scan_kernel(half, half);
-
-#define instantiate_conv1d_decode_kernel(type_name, type) \
- template [[host_name("conv1d_decode_kernel_" #type_name)]] \
- kernel void conv1d_decode_kernel<type>( \
- device const type* x [[buffer(0)]], \
- device const type* w [[buffer(1)]], \
- device const type* b [[buffer(2), function_constant(has_bias)]], \
- device const type* state [[buffer(3)]], \
- device type* x_out [[buffer(4)]], \
- device type* b_out [[buffer(5)]], \
- device type* c_out [[buffer(6)]], \
- device type* next_state [[buffer(7)]], \
- constant const int& kernel_size [[buffer(8)]], \
- constant const size_t& row_stride [[buffer(9)]], \
- constant const size_t& state_stride [[buffer(10)]], \
- constant const uint& num_channels [[buffer(11)]], \
- constant const size_t& suffix_len [[buffer(12)]], \
- constant const uint& inner_dim [[buffer(13)]], \
- constant const uint& proj_dim [[buffer(14)]], \
- uint2 grid_idx [[thread_position_in_grid]] \
- );
-
-#undef instantiate_conv1d_pack_prefix_kernel
-#undef instantiate_conv1d_scan_kernel
-instantiate_conv1d_decode_kernel(float, float);
-instantiate_conv1d_decode_kernel(bfloat, bfloat);
-instantiate_conv1d_decode_kernel(half, half);
-#undef instantiate_conv1d_decode_kernel
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
deleted file mode 100644
index 6056644ba..000000000
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
+++ /dev/null
@@ -1,282 +0,0 @@
-use super::SSMKernelError;
-use crate::{
- DataType,
- backends::metal::{
- FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
- MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained, data_type::MetalDataTypeExt,
- metal_extensions::ComputeEncoderSetValue,
- },
- config::Activation,
-};
-
-const CONV1D_SCAN_THREADGROUP_SIZE: usize = 32;
-
-const ACTIVATION_IDENTITY: i32 = 0;
-const ACTIVATION_SILU: i32 = 1;
-const ACTIVATION_GELU: i32 = 2;
-
-fn activation_to_int(activation: &Activation) -> i32 {
- match activation {
- Activation::Identity => ACTIVATION_IDENTITY,
- Activation::SiLU {
- ..
- } => ACTIVATION_SILU,
- Activation::Gelu => ACTIVATION_GELU,
- }
-}
-
-fn make_function_constants(
- activation: &Activation,
- has_bias: bool,
-) -> Retained<MTLFunctionConstantValues> {
- let function_constants = MTLFunctionConstantValues::new();
- let activation_type = activation_to_int(activation);
-
- function_constants.set_value(&activation_type, 0);
- function_constants.set_value(&has_bias, 1);
-
- function_constants
-}
-
-pub struct Conv1dScanKernel {
- pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- pack_pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- decode_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- decode_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
-}
-
-pub struct Conv1dScanArguments<'a> {
- pub padded: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0) [prefix+suffix, channels]
- pub w: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1) [channels, kernel]
- pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(2) [channels]
- pub x_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) [suffix, inner_dim]
- pub b_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) [suffix, proj_dim]
- pub c_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) [suffix, proj_dim]
- pub state_out: &'a ProtocolObject<dyn MTLBuffer>, // buffer(6) [channels, kernel-1]
- pub suffix_len: usize,
- pub kernel_size: i32,
- pub row_stride: usize,
- pub state_stride: usize,
- pub channels: usize,
- pub inner_dim: usize,
- pub proj_dim: usize,
-}
-
-pub struct Conv1dPackArguments<'a> {
- pub state_in: &'a ProtocolObject<dyn MTLBuffer>,
- pub x: &'a ProtocolObject<dyn MTLBuffer>,
- pub padded: &'a ProtocolObject<dyn MTLBuffer>,
- pub state_stride: usize,
- pub row_stride: usize,
- pub suffix_len: usize,
- pub channels: usize,
-}
-
-pub struct Conv1dDecodeArguments<'a> {
- pub x: &'a ProtocolObject<dyn MTLBuffer>,
- pub w: &'a ProtocolObject<dyn MTLBuffer>,
- pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>,
- pub state: &'a ProtocolObject<dyn MTLBuffer>,
- pub x_out: &'a ProtocolObject<dyn MTLBuffer>,
- pub b_out: &'a ProtocolObject<dyn MTLBuffer>,
- pub c_out: &'a ProtocolObject<dyn MTLBuffer>,
- pub next_state: &'a ProtocolObject<dyn MTLBuffer>,
- pub suffix_len: usize,
- pub kernel_size: i32,
- pub row_stride: usize,
- pub state_stride: usize,
- pub channels: usize,
- pub inner_dim: usize,
- pub proj_dim: usize,
-}
-
-impl Conv1dScanKernel {
- pub fn new(
- context: &MTLContext,
- data_type: DataType,
- activation: &Activation,
- ) -> Result<Self, SSMKernelError> {
- let fn_name = format!("conv1d_scan_kernel_{}", data_type.metal_type());
- let pack_name = format!("conv1d_pack_prefix_kernel_{}", data_type.metal_type());
- let decode_name = format!("conv1d_decode_kernel_{}", data_type.metal_type());
- let activation_id = activation_to_int(activation);
- let pipeline_no_bias = {
- let function_constants = make_function_constants(activation, false);
- let cache_key = format!("{}_act_{}_bias_0", fn_name, activation_id);
- context
- .compute_pipeline_state_cached(&cache_key, &fn_name, Some(&function_constants))
- .map_err(SSMKernelError::MetalError)?
- };
- let pipeline_with_bias = {
- let function_constants = make_function_constants(activation, true);
- let cache_key = format!("{}_act_{}_bias_1", fn_name, activation_id);
- context
- .compute_pipeline_state_cached(&cache_key, &fn_name, Some(&function_constants))
- .map_err(SSMKernelError::MetalError)?
- };
- let pack_pipeline = context.compute_pipeline_state(&pack_name, None).map_err(SSMKernelError::MetalError)?;
- let decode_pipeline_no_bias = {
- let function_constants = make_function_constants(activation, false);
- let cache_key = format!("{}_act_{}_bias_0", decode_name, activation_id);
- context
- .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))
- .map_err(SSMKernelError::MetalError)?
- };
- let decode_pipeline_with_bias = {
- let function_constants = make_function_constants(activation, true);
- let cache_key = format!("{}_act_{}_bias_1", decode_name, activation_id);
- context
- .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))
- .map_err(SSMKernelError::MetalError)?
- };
- Ok(Self {
- pipeline_no_bias,
- pipeline_with_bias,
- pack_pipeline,
- decode_pipeline_no_bias,
- decode_pipeline_with_bias,
- })
- }
-
- pub fn encode_pack(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: Conv1dPackArguments,
- ) -> Result<(), SSMKernelError> {
- if args.channels == 0 || args.state_stride == 0 {
- return Ok(());
- }
-
- compute_encoder.set_compute_pipeline_state(&self.pack_pipeline);
- compute_encoder.set_buffer(Some(args.state_in), 0, 0);
- compute_encoder.set_buffer(Some(args.x), 0, 1);
- compute_encoder.set_buffer(Some(args.padded), 0, 2);
- compute_encoder.set_value(&args.state_stride, 3);
- compute_encoder.set_value(&args.row_stride, 4);
- compute_encoder.set_value(&args.suffix_len, 5);
- let channels_u32 = args.channels as u32;
- compute_encoder.set_value(&channels_u32, 6);
-
- let threads_per_threadgroup = MTLSize {
- width: CONV1D_SCAN_THREADGROUP_SIZE,
- height: 1,
- depth: 1,
- };
- let padded_rows = args.state_stride + args.suffix_len;
- let total_threads = MTLSize {
- width: args.channels,
- height: padded_rows,
- depth: 1,
- };
- compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);
- Ok(())
- }
-
- pub fn encode_decode(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: Conv1dDecodeArguments,
- ) -> Result<(), SSMKernelError> {
- if args.channels == 0 || args.suffix_len == 0 || args.kernel_size <= 0 {
- return Ok(());
- }
-
- let has_bias = args.b.is_some();
- let pipeline = if has_bias {
- &self.decode_pipeline_with_bias
- } else {
- &self.decode_pipeline_no_bias
- };
- compute_encoder.set_compute_pipeline_state(pipeline);
- compute_encoder.set_buffer(Some(args.x), 0, 0);
- compute_encoder.set_buffer(Some(args.w), 0, 1);
- if has_bias {
- compute_encoder.set_buffer(args.b, 0, 2);
- }
- compute_encoder.set_buffer(Some(args.state), 0, 3);
- compute_encoder.set_buffer(Some(args.x_out), 0, 4);
- compute_encoder.set_buffer(Some(args.b_out), 0, 5);
- compute_encoder.set_buffer(Some(args.c_out), 0, 6);
- compute_encoder.set_buffer(Some(args.next_state), 0, 7);
- compute_encoder.set_value(&args.kernel_size, 8);
- compute_encoder.set_value(&args.row_stride, 9);
- compute_encoder.set_value(&args.state_stride, 10);
- let channels_u32 = args.channels as u32;
- compute_encoder.set_value(&channels_u32, 11);
- compute_encoder.set_value(&args.suffix_len, 12);
- let inner_dim_u32 = args.inner_dim as u32;
- compute_encoder.set_value(&inner_dim_u32, 13);
- let proj_dim_u32 = args.proj_dim as u32;
- compute_encoder.set_value(&proj_dim_u32, 14);
-
- let threads_per_threadgroup = MTLSize {
- width: CONV1D_SCAN_THREADGROUP_SIZE,
- height: 1,
- depth: 1,
- };
- let total_threads = MTLSize {
- width: args.suffix_len,
- height: args.channels,
- depth: 1,
- };
- compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);
- Ok(())
- }
-
- pub fn encode(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: Conv1dScanArguments,
- ) -> Result<(), SSMKernelError> {
- if args.channels == 0 || args.suffix_len == 0 || args.kernel_size <= 0 {
- return Ok(());
- }
-
- let has_bias = args.b.is_some();
- let pipeline = if has_bias {
- &self.pipeline_with_bias
- } else {
- &self.pipeline_no_bias
- };
- compute_encoder.set_compute_pipeline_state(pipeline);
-
- compute_encoder.set_buffer(Some(args.padded), 0, 0);
- compute_encoder.set_buffer(Some(args.w), 0, 1);
- if has_bias {
- compute_encoder.set_buffer(args.b, 0, 2);
- }
- compute_encoder.set_buffer(Some(args.x_out), 0, 3);
- compute_encoder.set_buffer(Some(args.b_out), 0, 4);
- compute_encoder.set_buffer(Some(args.c_out), 0, 5);
- compute_encoder.set_buffer(Some(args.state_out), 0, 6);
-
- compute_encoder.set_value(&args.suffix_len, 7);
- compute_encoder.set_value(&args.kernel_size, 8);
- compute_encoder.set_value(&args.row_stride, 9);
- compute_encoder.set_value(&args.state_stride, 10);
- let channels_u32 = args.channels as u32;
- compute_encoder.set_value(&channels_u32, 11);
- let inner_dim_u32 = args.inner_dim as u32;
- compute_encoder.set_value(&inner_dim_u32, 12);
- let proj_dim_u32 = args.proj_dim as u32;
- compute_encoder.set_value(&proj_dim_u32, 13);
-
- let suffix = args.suffix_len;
- let tap_count = args.kernel_size.saturating_sub(1) as usize;
- let work_len = suffix + tap_count;
- let channels = args.channels;
- let threads_per_threadgroup = MTLSize {
- width: CONV1D_SCAN_THREADGROUP_SIZE,
- height: 1,
- depth: 1,
- };
- let total_threads = MTLSize {
- width: work_len,
- height: channels,
- depth: 1,
- };
- compute_encoder.dispatch_threads(total_threads, threads_per_threadgroup);
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
index c64f8af63..a87e87362 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
@@ -1,15 +1,2 @@
-use crate::backends::metal::MTLError;
-
-#[derive(Debug, thiserror::Error)]
-pub enum SSMKernelError {
- #[error("Metal error: {0}")]
- MetalError(#[from] MTLError),
- #[error("Function not found: {0}")]
- FunctionNotFound(String),
-}
-
-pub mod conv1d_scan;
pub mod ssd_prefill;
-
-pub use conv1d_scan::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel};
pub use ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode};
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
index d96d207a9..1e105a86e 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
@@ -1,3 +1,7 @@
+#define ACTIVATION_IDENTITY 0
+#define ACTIVATION_SILU 1
+#define ACTIVATION_GELU 2
+
template <typename T>
inline T apply_silu(T x) {
float xf = float(x);
@@ -13,4 +17,24 @@ inline T softplus(T x) {
return x;
}
return static_cast<T>(log(1.0f + fast::exp(xf)));
+}
+
+template <typename T>
+inline T apply_gelu(T x) {
+ float xf = float(x);
+ return static_cast<T>(
+ 0.5f * xf *
+ (1.0f + fast::tanh(0.797885f * (xf + 0.044715f * xf * xf * xf)))
+ );
+}
+
+template <typename T>
+inline T apply_activation_fn(T x, int activation_type) {
+ if (activation_type == ACTIVATION_SILU) {
+ return apply_silu(x);
+ } else if (activation_type == ACTIVATION_GELU) {
+ return apply_gelu(x);
+ } else {
+ return x; // Identity
+ }
}
\ No newline at end of file
diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rs
index a4bd00717..aa150d51b 100644
--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs
+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs
@@ -9,16 +9,15 @@ use metal::{
use uzu::{
DataType,
backends::{
- common::Context,
+ common::{Context, kernel::Conv1dScanKernel},
metal::{
MTLBuffer, MTLContext, ProtocolObject,
- kernel::ssm::{
- SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode,
- conv1d_scan::{Conv1dScanArguments, Conv1dScanKernel},
+ kernel::{
+ dsl::Conv1dScanMetalKernel,
+ ssm::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
},
},
},
- config::Activation,
};
const STORAGE_MODE: MTLResourceOptions = MTLResourceOptions::STORAGE_MODE_SHARED;
@@ -261,7 +260,7 @@ fn run_prefill_kernel_mode(
fn run_conv_scan_once(
ctx: &MTLContext,
- kernel: &Conv1dScanKernel,
+ kernel: &Conv1dScanMetalKernel,
suffix_len: usize,
channels: usize,
kernel_size: i32,
@@ -319,26 +318,25 @@ fn run_conv_scan_once(
write_buffer(&padded_buf, &host);
}
- let args = Conv1dScanArguments {
- padded: &padded_buf,
- w: &w_buf,
- b: Some(&b_buf),
- x_out: &y_buf,
- b_out: &y_buf,
- c_out: &y_buf,
- state_out: scratch_buf.as_ref().unwrap_or(&state_buf),
- suffix_len,
- kernel_size,
- row_stride: channels,
- state_stride: tap_count,
- channels,
- inner_dim: channels,
- proj_dim: 0,
- };
-
let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&encoder, args).unwrap();
+ kernel.encode(
+ &padded_buf,
+ &w_buf,
+ Some(&b_buf),
+ &y_buf,
+ &y_buf,
+ &y_buf,
+ scratch_buf.as_ref().unwrap_or(&state_buf),
+ suffix_len as u32,
+ kernel_size as u32,
+ channels as u32,
+ tap_count as u32,
+ channels as u32,
+ channels as u32,
+ 0u32,
+ &encoder,
+ );
encoder.end_encoding();
if let Some(ref scratch) = scratch_buf {
@@ -464,8 +462,7 @@ fn conv1d_scan_is_deterministic() {
eprintln!("Skipping conv1d scan determinism test: no Metal device");
return;
};
- let activation = Activation::Identity;
- let kernel = Conv1dScanKernel::new(&ctx, DataType::F32, &activation).unwrap();
+ let kernel = Conv1dScanMetalKernel::new(&ctx, DataType::F32, 0u32, true).unwrap();
let suffix_len = 192usize;
let channels = 8usize;