Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

short conv update

LuckyIYIa63f7cc4fca8
diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
index b6c0a95c7..65209d626 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
@@ -148,9 +148,8 @@ impl MambaMixer {
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
) {
- let suffix_length = state.aux_buffers_suffix_length();
let active_suffix_length = state.active_suffix_length();
- if suffix_length == 0 || active_suffix_length == 0 {
+ if active_suffix_length == 0 {
return;
}
@@ -158,7 +157,7 @@ impl MambaMixer {
self.run_split_inproj(state, command_buffer, active_suffix_length);
self.run_conv_scan(state, command_buffer, active_suffix_length);
- if suffix_length == 1 {
+ if active_suffix_length == 1 {
self.run_decode_ssm(state, command_buffer, active_suffix_length);
} else {
self.run_prefill_ssm(state, command_buffer, active_suffix_length);
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
index ff919af81..b3eee245d 100644
--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
@@ -10,7 +10,7 @@ use crate::{
compilation_parameters::CompilationConfig,
forward_pass::{ArrayId, ForwardPassState},
kernel::short_conv::{
- ShortConvDecodeArguments, ShortConvKernel,
+ ShortConvDecodeArguments, ShortConvKernel, ShortConvPackArguments,
ShortConvPrefillArguments,
},
},
@@ -121,15 +121,14 @@ impl ShortConvMixer {
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
) {
- let suffix_length = state.aux_buffers_suffix_length();
let active_suffix_length = state.active_suffix_length();
- if suffix_length == 0 || active_suffix_length == 0 {
+ if active_suffix_length == 0 {
return;
}
self.in_projection.encode(state, command_buffer, parameters);
- if suffix_length == 1 {
+ if active_suffix_length == 1 {
self.run_decode_conv(state, command_buffer, active_suffix_length);
} else {
self.run_prefill_conv(state, command_buffer, active_suffix_length);
@@ -151,15 +150,21 @@ impl ShortConvMixer {
command_buffer: &MPSCommandBuffer,
suffix_length: usize,
) {
+ let padded = state
+ .short_conv_padded_buffer()
+ .expect("short_conv_padded scratch buffer not initialized");
+
let arrays = state.arrays(&[
ArrayId::SsmInProj,
ArrayId::ShortConvState(self.layer_index),
ArrayId::AttentionOutput,
]);
+ let mut padded = padded.borrow_mut();
let mut in_proj = arrays[0].borrow_mut();
let mut conv_state = arrays[1].borrow_mut();
let mut out = arrays[2].borrow_mut();
+ let padded_buf = unsafe { padded.mtl_buffer().to_owned() };
let in_proj_buf = unsafe { in_proj.mtl_buffer().to_owned() };
let state_buf = unsafe { conv_state.mtl_buffer().to_owned() };
let out_buf = unsafe { out.mtl_buffer().to_owned() };
@@ -178,14 +183,29 @@ impl ShortConvMixer {
command_buffer.root_command_buffer().to_owned();
let compute = mtl_command_buffer.new_compute_command_encoder();
+ self.short_conv_kernel
+ .encode_pack(
+ &compute,
+ ShortConvPackArguments {
+ state_in: &state_buf,
+ in_proj: &in_proj_buf,
+ padded: &padded_buf,
+ state_stride,
+ suffix_len: suffix_length,
+ in_proj_stride: self.model_dim * 3,
+ model_dim: self.model_dim,
+ },
+ )
+ .expect("Failed to encode short conv pack kernel");
+
self.short_conv_kernel
.encode_prefill(
&compute,
ShortConvPrefillArguments {
+ padded: &padded_buf,
in_proj: &in_proj_buf,
w: &weight_buf,
b: bias_buf.as_ref(),
- state_in: &state_buf,
out: &out_buf,
state_out: &state_buf,
suffix_len: suffix_length,
diff --git a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
index b537b81dd..0c4033239 100644
--- a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
@@ -391,6 +391,12 @@ impl ModelShape {
self.max_mamba_heads > 0
}
+ pub fn has_short_conv_layers(&self) -> bool {
+ self.layer_types.iter().any(|layer_type| {
+ matches!(layer_type, DecoderLayerType::ShortConv { .. })
+ })
+ }
+
pub fn max_mamba_conv_dim(&self) -> Option<usize> {
if self.max_mamba_conv_dim == 0 {
None
@@ -434,6 +440,28 @@ impl ModelShape {
}
}
+ pub fn short_conv_padded_shape(
+ &self,
+ suffix_length: usize,
+ ) -> Option<[usize; 2]> {
+ if !self.has_short_conv_layers() {
+ return None;
+ }
+
+ let max_kernel_size = self
+ .layer_types
+ .iter()
+ .filter_map(|layer_type| match layer_type {
+ DecoderLayerType::ShortConv { kernel_size } => Some(*kernel_size),
+ _ => None,
+ })
+ .max()
+ .unwrap_or(0);
+
+ let max_state_stride = max_kernel_size.saturating_sub(1);
+ Some([suffix_length + max_state_stride, self.model_dim])
+ }
+
pub fn ssm_x_shape(
&self,
suffix_length: usize,
diff --git a/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs
index a74e214cc..27b132a11 100644
--- a/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs
@@ -30,6 +30,7 @@ pub struct ScratchBuffers {
pub ssm_inproj: Option<MTLBuffer>,
pub ssm_packed: Option<MTLBuffer>,
pub ssm_conv_padded: Option<MTLBuffer>,
+ pub short_conv_padded: Option<MTLBuffer>,
pub ssm_x: Option<MTLBuffer>,
pub ssm_b: Option<MTLBuffer>,
pub ssm_c: Option<MTLBuffer>,
@@ -156,6 +157,9 @@ impl ScratchBuffers {
ssm_conv_padded: model_shape
.ssm_conv_padded_shape(max_suffix_len)
.map(|shape| alloc(&shape, act_ty)),
+ short_conv_padded: model_shape
+ .short_conv_padded_shape(max_suffix_len)
+ .map(|shape| alloc(&shape, act_ty)),
ssm_x: model_shape
.ssm_x_shape(max_suffix_len)
.map(|shape| alloc(&shape, act_ty)),
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs
index 4469de591..cce6e48cb 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs
@@ -11,6 +11,7 @@ pub struct LanguageModelGeneratorAuxBuffers {
pub ssm_inproj: Option<ArrayCell>,
pub ssm_packed: Option<ArrayCell>,
pub ssm_conv_padded: Option<ArrayCell>,
+ pub short_conv_padded: Option<ArrayCell>,
pub ssm_x: Option<ArrayCell>,
pub ssm_b: Option<ArrayCell>,
pub ssm_c: Option<ArrayCell>,
@@ -76,6 +77,15 @@ impl LanguageModelGeneratorAuxBuffers {
)),
_ => None,
},
+ short_conv_padded: match (
+ scratch.short_conv_padded.as_ref(),
+ model_shape.short_conv_padded_shape(suffix_length),
+ ) {
+ (Some(buf), Some(shape)) => Some(RefCell::new(
+ MetalArray::new(buf.clone(), &shape, act_dtype),
+ )),
+ _ => None,
+ },
ssm_x: match (
scratch.ssm_x.as_ref(),
model_shape.ssm_x_shape(suffix_length),
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
index 03b98b33a..27fc8e53a 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
@@ -904,6 +904,12 @@ impl ForwardPassState {
self.llm_aux.as_ref().and_then(|aux| aux.ssm_conv_padded.clone())
}
+ pub fn short_conv_padded_buffer(&self) -> Option<ArrayCell> {
+ self.llm_aux
+ .as_ref()
+ .and_then(|aux| aux.short_conv_padded.clone())
+ }
+
// ========================================================================
// Public API Methods (formerly trait methods)
// ========================================================================
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
index 02de18eba..4ea13a2ba 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
@@ -18,15 +18,26 @@ fn fn_suffix(dt: KernelDataType) -> &'static str {
}
pub struct ShortConvKernel {
+ pack_pipeline: MTLComputePipelineState,
prefill_pipeline: MTLComputePipelineState,
decode_pipeline: MTLComputePipelineState,
}
+pub struct ShortConvPackArguments<'a> {
+ pub state_in: &'a MTLBuffer,
+ pub in_proj: &'a MTLBuffer,
+ pub padded: &'a MTLBuffer,
+ pub state_stride: usize,
+ pub suffix_len: usize,
+ pub in_proj_stride: usize,
+ pub model_dim: usize,
+}
+
pub struct ShortConvPrefillArguments<'a> {
+ pub padded: &'a MTLBuffer,
pub in_proj: &'a MTLBuffer,
pub w: &'a MTLBuffer,
pub b: Option<&'a MTLBuffer>,
- pub state_in: &'a MTLBuffer,
pub out: &'a MTLBuffer,
pub state_out: &'a MTLBuffer,
pub suffix_len: usize,
@@ -55,11 +66,16 @@ impl ShortConvKernel {
context: &MTLContext,
data_type: KernelDataType,
) -> Result<Self, ShortConvKernelError> {
+ let pack_name =
+ format!("short_conv_pack_kernel_{}", fn_suffix(data_type));
let prefill_name =
format!("short_conv_prefill_kernel_{}", fn_suffix(data_type));
let decode_name =
format!("short_conv_decode_kernel_{}", fn_suffix(data_type));
+ let pack_pipeline = context
+ .compute_pipeline_state(&pack_name, None)
+ .map_err(ShortConvKernelError::MetalError)?;
let prefill_pipeline = context
.compute_pipeline_state(&prefill_name, None)
.map_err(ShortConvKernelError::MetalError)?;
@@ -68,11 +84,63 @@ impl ShortConvKernel {
.map_err(ShortConvKernelError::MetalError)?;
Ok(Self {
+ pack_pipeline,
prefill_pipeline,
decode_pipeline,
})
}
+ pub fn encode_pack(
+ &self,
+ compute_encoder: &ComputeCommandEncoderRef,
+ args: ShortConvPackArguments,
+ ) -> Result<(), ShortConvKernelError> {
+ if args.model_dim == 0 || args.suffix_len == 0 {
+ return Ok(());
+ }
+
+ compute_encoder.set_compute_pipeline_state(&self.pack_pipeline);
+ compute_encoder.set_buffer(0, Some(args.state_in), 0);
+ compute_encoder.set_buffer(1, Some(args.in_proj), 0);
+ compute_encoder.set_buffer(2, Some(args.padded), 0);
+ compute_encoder.set_bytes(
+ 3,
+ size_of::<usize>() as u64,
+ &args.state_stride as *const usize as *const _,
+ );
+ compute_encoder.set_bytes(
+ 4,
+ size_of::<usize>() as u64,
+ &args.suffix_len as *const usize as *const _,
+ );
+ compute_encoder.set_bytes(
+ 5,
+ size_of::<usize>() as u64,
+ &args.in_proj_stride as *const usize as *const _,
+ );
+ compute_encoder.set_bytes(
+ 6,
+ size_of::<u32>() as u64,
+ &(args.model_dim as u32) as *const u32 as *const _,
+ );
+
+ let threads_per_threadgroup = MTLSize {
+ width: 32,
+ height: 1,
+ depth: 1,
+ };
+ let padded_rows = args.state_stride + args.suffix_len;
+ let threadgroups = MTLSize {
+ width: args.model_dim as u64,
+ height: padded_rows as u64,
+ depth: 1,
+ };
+
+ compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);
+
+ Ok(())
+ }
+
pub fn encode_prefill(
&self,
compute_encoder: &ComputeCommandEncoderRef,
@@ -82,11 +150,14 @@ impl ShortConvKernel {
return Ok(());
}
+ let tap_count = (args.kernel_size - 1).max(0);
+ let work_len = args.suffix_len + tap_count as usize;
+
compute_encoder.set_compute_pipeline_state(&self.prefill_pipeline);
- compute_encoder.set_buffer(0, Some(args.in_proj), 0);
- compute_encoder.set_buffer(1, Some(args.w), 0);
- compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);
- compute_encoder.set_buffer(3, Some(args.state_in), 0);
+ compute_encoder.set_buffer(0, Some(args.padded), 0);
+ compute_encoder.set_buffer(1, Some(args.in_proj), 0);
+ compute_encoder.set_buffer(2, Some(args.w), 0);
+ compute_encoder.set_buffer(3, args.b.map(|v| &**v), 0);
compute_encoder.set_buffer(4, Some(args.out), 0);
compute_encoder.set_buffer(5, Some(args.state_out), 0);
compute_encoder.set_bytes(
@@ -121,7 +192,7 @@ impl ShortConvKernel {
depth: 1,
};
let threadgroups = MTLSize {
- width: args.suffix_len as u64,
+ width: work_len as u64,
height: args.model_dim as u64,
depth: 1,
};
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
index c95655e79..a9bbb2b86 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -3,12 +3,51 @@
using namespace metal;
+template <typename T>
+kernel void short_conv_pack_kernel(
+ device const T* state_in [[buffer(0)]],
+ device const T* in_proj [[buffer(1)]],
+ device T* padded [[buffer(2)]],
+ constant const size_t& state_stride [[buffer(3)]],
+ constant const size_t& suffix_len [[buffer(4)]],
+ constant const size_t& in_proj_stride [[buffer(5)]],
+ constant const uint& model_dim [[buffer(6)]],
+ uint2 grid_idx [[thread_position_in_grid]]
+) {
+ const uint channel_idx = grid_idx.x;
+ const uint row_idx = grid_idx.y;
+ const size_t padded_rows = state_stride + suffix_len;
+
+ if (channel_idx >= model_dim || row_idx >= padded_rows) {
+ return;
+ }
+
+ const size_t padded_offset =
+ size_t(row_idx) * model_dim + size_t(channel_idx);
+
+ if (row_idx < state_stride) {
+ // Copy from state
+ const size_t state_idx = size_t(channel_idx) * state_stride + size_t(row_idx);
+ padded[padded_offset] = state_in[state_idx];
+ } else {
+ // Compute gated input from in_proj
+ const size_t token = row_idx - state_stride;
+ const size_t in_proj_idx = size_t(token) * in_proj_stride + size_t(channel_idx);
+
+ float pre_gate = float(in_proj[in_proj_idx]);
+ float x_in = float(in_proj[in_proj_idx + 2 * model_dim]);
+ float x = x_in * pre_gate;
+
+ padded[padded_offset] = static_cast<T>(x);
+ }
+}
+
template <typename T>
kernel void short_conv_prefill_kernel(
- device const T* in_proj [[buffer(0)]],
- device const T* w [[buffer(1)]],
- device const T* b [[buffer(2)]],
- device const T* state_in [[buffer(3)]],
+ device const T* padded [[buffer(0)]],
+ device const T* in_proj [[buffer(1)]],
+ device const T* w [[buffer(2)]],
+ device const T* b [[buffer(3)]],
device T* out [[buffer(4)]],
device T* state_out [[buffer(5)]],
constant const size_t& suffix_len [[buffer(6)]],
@@ -18,72 +57,53 @@ kernel void short_conv_prefill_kernel(
constant const uint& model_dim [[buffer(10)]],
uint2 grid_idx [[thread_position_in_grid]]
) {
+ const int tap_count = max(kernel_size - 1, 0);
+ const size_t work_len = suffix_len + size_t(tap_count);
+
const uint token_idx = grid_idx.x;
const uint channel_idx = grid_idx.y;
- if (channel_idx >= model_dim || token_idx >= suffix_len) {
+ if (channel_idx >= model_dim || token_idx >= work_len) {
return;
}
- const int tap_count = max(kernel_size - 1, 0);
- const size_t state_offset = size_t(channel_idx) * state_stride;
const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);
const bool has_bias = b != nullptr;
- size_t current_in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);
- float pre_conv_gate = float(in_proj[current_in_proj_idx]);
- float post_conv_gate = float(in_proj[current_in_proj_idx + model_dim]);
- float x_current = float(in_proj[current_in_proj_idx + 2 * model_dim]);
+ // Threads [0..suffix_len-1]: Compute outputs
+ if (token_idx < suffix_len) {
+ float acc = has_bias ? float(b[channel_idx]) : 0.0f;
- float gated_input = x_current * pre_conv_gate;
-
- float acc = has_bias ? float(b[channel_idx]) : 0.0f;
-
- for (int tap = 0; tap < kernel_size; ++tap) {
- int src_token = int(token_idx) - (kernel_size - 1 - tap);
- float sample = 0.0f;
-
- if (src_token < 0) {
- int state_idx = tap_count + src_token;
- if (state_idx >= 0 && state_idx < tap_count) {
- sample = float(state_in[state_offset + size_t(state_idx)]);
- }
- } else if (src_token == int(token_idx)) {
- sample = gated_input;
- } else {
- size_t src_in_proj_idx = size_t(src_token) * in_proj_stride + size_t(channel_idx);
- float src_pre_gate = float(in_proj[src_in_proj_idx]);
- float src_x = float(in_proj[src_in_proj_idx + 2 * model_dim]);
- sample = src_x * src_pre_gate;
+ // Convolve using padded buffer
+ for (int tap = 0; tap < kernel_size; ++tap) {
+ const size_t padded_row = size_t(token_idx + tap);
+ const size_t padded_offset = padded_row * model_dim + channel_idx;
+ float sample = float(padded[padded_offset]);
+ acc += float(w_row[tap]) * sample;
}
- acc += float(w_row[tap]) * sample;
- }
+ // Apply post-gate from in_proj
+ const size_t in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);
+ float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);
+ float gated_output = acc * post_conv_gate;
- float gated_output = acc * post_conv_gate;
+ // Write output
+ const size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);
+ out[out_idx] = static_cast<T>(gated_output);
+ }
+ // Threads [suffix_len..work_len-1]: Write state
+ else if (tap_count > 0) {
+ const size_t tap = size_t(token_idx - suffix_len);
+ if (tap >= size_t(tap_count)) {
+ return;
+ }
- size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);
- out[out_idx] = static_cast<T>(gated_output);
+ // Copy last tap_count values from padded to state_out
+ const size_t padded_row = suffix_len + tap;
+ const size_t padded_offset = padded_row * model_dim + channel_idx;
+ const size_t state_idx = size_t(channel_idx) * state_stride + tap;
- if (tap_count > 0 && token_idx == 0) {
- for (int tap = 0; tap < tap_count; ++tap) {
- int src_token = int(suffix_len) - tap_count + tap;
- float sample = 0.0f;
-
- if (src_token < 0) {
- int old_state_idx = tap_count + src_token;
- if (old_state_idx >= 0 && old_state_idx < tap_count) {
- sample = float(state_in[state_offset + size_t(old_state_idx)]);
- }
- } else {
- size_t src_in_proj_idx = size_t(src_token) * in_proj_stride + size_t(channel_idx);
- float src_pre_gate = float(in_proj[src_in_proj_idx]);
- float src_x = float(in_proj[src_in_proj_idx + 2 * model_dim]);
- sample = src_x * src_pre_gate;
- }
-
- state_out[state_offset + size_t(tap)] = static_cast<T>(sample);
- }
+ state_out[state_idx] = padded[padded_offset];
}
}
@@ -117,9 +137,9 @@ kernel void short_conv_decode_kernel(
size_t in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);
float pre_conv_gate = float(in_proj[in_proj_idx]);
float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);
- float x_val = float(in_proj[in_proj_idx + 2 * model_dim]);
+ float x_in = float(in_proj[in_proj_idx + 2 * model_dim]);
- float gated_input = x_val * pre_conv_gate;
+ float x = x_in * pre_conv_gate;
float acc = has_bias ? float(b[channel_idx]) : 0.0f;
@@ -128,7 +148,7 @@ kernel void short_conv_decode_kernel(
acc += float(w_row[tap]) * sample;
}
- acc += float(w_row[tap_count]) * gated_input;
+ acc += float(w_row[tap_count]) * x;
float gated_output = acc * post_conv_gate;
@@ -139,44 +159,64 @@ kernel void short_conv_decode_kernel(
for (int tap = 0; tap < tap_count - 1; ++tap) {
next_state[state_offset + size_t(tap)] = state[state_offset + size_t(tap + 1)];
}
- next_state[state_offset + size_t(tap_count - 1)] = static_cast<T>(gated_input);
+ next_state[state_offset + size_t(tap_count - 1)] = static_cast<T>(x);
}
}
-#define instantiate_short_conv_prefill_kernel(type_name, type) \
- template [[host_name("short_conv_prefill_kernel_" #type_name)]] \
- kernel void short_conv_prefill_kernel<type>( \
- device const type* x [[buffer(0)]], \
- device const type* w [[buffer(1)]], \
- device const type* b [[buffer(2)]], \
- device const type* state_in [[buffer(3)]], \
- device type* out [[buffer(4)]], \
- device type* state_out [[buffer(5)]], \
- constant const size_t& suffix_len [[buffer(6)]], \
- constant const int& kernel_size [[buffer(7)]], \
- constant const size_t& row_stride [[buffer(8)]], \
- constant const size_t& state_stride [[buffer(9)]], \
- constant const uint& num_channels [[buffer(10)]], \
- uint2 grid_idx [[thread_position_in_grid]] \
+// ============================================================================
+// Template instantiations
+// ============================================================================
+#define instantiate_short_conv_pack_kernel(type_name, type) \
+ template [[host_name("short_conv_pack_kernel_" #type_name)]] \
+ kernel void short_conv_pack_kernel<type>( \
+ device const type* state_in [[buffer(0)]], \
+ device const type* in_proj [[buffer(1)]], \
+ device type* padded [[buffer(2)]], \
+ constant const size_t& state_stride [[buffer(3)]], \
+ constant const size_t& suffix_len [[buffer(4)]], \
+ constant const size_t& in_proj_stride [[buffer(5)]], \
+ constant const uint& model_dim [[buffer(6)]], \
+ uint2 grid_idx [[thread_position_in_grid]] \
);
-#define instantiate_short_conv_decode_kernel(type_name, type) \
- template [[host_name("short_conv_decode_kernel_" #type_name)]] \
- kernel void short_conv_decode_kernel<type>( \
- device const type* x [[buffer(0)]], \
- device const type* w [[buffer(1)]], \
- device const type* b [[buffer(2)]], \
- device const type* state [[buffer(3)]], \
- device type* out [[buffer(4)]], \
- device type* next_state [[buffer(5)]], \
- constant const size_t& suffix_len [[buffer(6)]], \
- constant const int& kernel_size [[buffer(7)]], \
- constant const size_t& row_stride [[buffer(8)]], \
- constant const size_t& state_stride [[buffer(9)]], \
- constant const uint& num_channels [[buffer(10)]], \
- uint2 grid_idx [[thread_position_in_grid]] \
+#define instantiate_short_conv_prefill_kernel(type_name, type) \
+ template [[host_name("short_conv_prefill_kernel_" #type_name)]] \
+ kernel void short_conv_prefill_kernel<type>( \
+ device const type* padded [[buffer(0)]], \
+ device const type* in_proj [[buffer(1)]], \
+ device const type* w [[buffer(2)]], \
+ device const type* b [[buffer(3)]], \
+ device type* out [[buffer(4)]], \
+ device type* state_out [[buffer(5)]], \
+ constant const size_t& suffix_len [[buffer(6)]], \
+ constant const int& kernel_size [[buffer(7)]], \
+ constant const size_t& in_proj_stride [[buffer(8)]], \
+ constant const size_t& state_stride [[buffer(9)]], \
+ constant const uint& model_dim [[buffer(10)]], \
+ uint2 grid_idx [[thread_position_in_grid]] \
);
+#define instantiate_short_conv_decode_kernel(type_name, type) \
+ template [[host_name("short_conv_decode_kernel_" #type_name)]] \
+ kernel void short_conv_decode_kernel<type>( \
+ device const type* x [[buffer(0)]], \
+ device const type* w [[buffer(1)]], \
+ device const type* b [[buffer(2)]], \
+ device const type* state [[buffer(3)]], \
+ device type* out [[buffer(4)]], \
+ device type* next_state [[buffer(5)]], \
+ constant const size_t& suffix_len [[buffer(6)]], \
+ constant const int& kernel_size [[buffer(7)]], \
+ constant const size_t& row_stride [[buffer(8)]], \
+ constant const size_t& state_stride [[buffer(9)]], \
+ constant const uint& num_channels [[buffer(10)]], \
+ uint2 grid_idx [[thread_position_in_grid]] \
+ );
+
+instantiate_short_conv_pack_kernel(float, float);
+instantiate_short_conv_pack_kernel(bfloat, bfloat);
+instantiate_short_conv_pack_kernel(half, half);
+
instantiate_short_conv_prefill_kernel(float, float);
instantiate_short_conv_prefill_kernel(bfloat, bfloat);
instantiate_short_conv_prefill_kernel(half, half);
@@ -185,6 +225,6 @@ instantiate_short_conv_decode_kernel(float, float);
instantiate_short_conv_decode_kernel(bfloat, bfloat);
instantiate_short_conv_decode_kernel(half, half);
+#undef instantiate_short_conv_pack_kernel
#undef instantiate_short_conv_prefill_kernel
#undef instantiate_short_conv_decode_kernel
-