A high-performance inference engine for AI models
short conv update
LuckyIYIa63f7cc4fca8
diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rsindex b6c0a95c7..65209d626 100644--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs@@ -148,9 +148,8 @@ impl MambaMixer {command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,) {- let suffix_length = state.aux_buffers_suffix_length();let active_suffix_length = state.active_suffix_length();- if suffix_length == 0 || active_suffix_length == 0 {+ if active_suffix_length == 0 {return;}@@ -158,7 +157,7 @@ impl MambaMixer {self.run_split_inproj(state, command_buffer, active_suffix_length);self.run_conv_scan(state, command_buffer, active_suffix_length);- if suffix_length == 1 {+ if active_suffix_length == 1 {self.run_decode_ssm(state, command_buffer, active_suffix_length);} else {self.run_prefill_ssm(state, command_buffer, active_suffix_length);diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rsindex ff919af81..b3eee245d 100644--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs@@ -10,7 +10,7 @@ use crate::{compilation_parameters::CompilationConfig,forward_pass::{ArrayId, ForwardPassState},kernel::short_conv::{- ShortConvDecodeArguments, ShortConvKernel,+ ShortConvDecodeArguments, ShortConvKernel, ShortConvPackArguments,ShortConvPrefillArguments,},},@@ -121,15 +121,14 @@ impl ShortConvMixer {command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,) {- let suffix_length = state.aux_buffers_suffix_length();let active_suffix_length = state.active_suffix_length();- if suffix_length == 0 || active_suffix_length == 0 {+ if active_suffix_length == 0 {return;}self.in_projection.encode(state, command_buffer, parameters);- if suffix_length == 1 {+ if active_suffix_length == 1 {self.run_decode_conv(state, command_buffer, active_suffix_length);} else {self.run_prefill_conv(state, command_buffer, active_suffix_length);@@ -151,15 +150,21 @@ impl ShortConvMixer {command_buffer: &MPSCommandBuffer,suffix_length: usize,) {+ let padded = state+ .short_conv_padded_buffer()+ .expect("short_conv_padded scratch buffer not initialized");+let arrays = state.arrays(&[ArrayId::SsmInProj,ArrayId::ShortConvState(self.layer_index),ArrayId::AttentionOutput,]);+ let mut padded = padded.borrow_mut();let mut in_proj = arrays[0].borrow_mut();let mut conv_state = arrays[1].borrow_mut();let mut out = arrays[2].borrow_mut();+ let padded_buf = unsafe { padded.mtl_buffer().to_owned() };let in_proj_buf = unsafe { in_proj.mtl_buffer().to_owned() };let state_buf = unsafe { conv_state.mtl_buffer().to_owned() };let out_buf = unsafe { out.mtl_buffer().to_owned() };@@ -178,14 +183,29 @@ impl ShortConvMixer {command_buffer.root_command_buffer().to_owned();let compute = mtl_command_buffer.new_compute_command_encoder();+ self.short_conv_kernel+ .encode_pack(+ &compute,+ ShortConvPackArguments {+ state_in: &state_buf,+ in_proj: &in_proj_buf,+ padded: &padded_buf,+ state_stride,+ suffix_len: suffix_length,+ in_proj_stride: self.model_dim * 3,+ model_dim: self.model_dim,+ },+ )+ .expect("Failed to encode short conv pack kernel");+self.short_conv_kernel.encode_prefill(&compute,ShortConvPrefillArguments {+ padded: &padded_buf,in_proj: &in_proj_buf,w: &weight_buf,b: bias_buf.as_ref(),- state_in: &state_buf,out: &out_buf,state_out: &state_buf,suffix_len: suffix_length,diff --git a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs b/crates/uzu/src/backends/metal/forward_pass/model_shape.rsindex b537b81dd..0c4033239 100644--- a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs+++ b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs@@ -391,6 +391,12 @@ impl ModelShape {self.max_mamba_heads > 0}+ pub fn has_short_conv_layers(&self) -> bool {+ self.layer_types.iter().any(|layer_type| {+ matches!(layer_type, DecoderLayerType::ShortConv { .. })+ })+ }+pub fn max_mamba_conv_dim(&self) -> Option<usize> {if self.max_mamba_conv_dim == 0 {None@@ -434,6 +440,28 @@ impl ModelShape {}}+ pub fn short_conv_padded_shape(+ &self,+ suffix_length: usize,+ ) -> Option<[usize; 2]> {+ if !self.has_short_conv_layers() {+ return None;+ }++ let max_kernel_size = self+ .layer_types+ .iter()+ .filter_map(|layer_type| match layer_type {+ DecoderLayerType::ShortConv { kernel_size } => Some(*kernel_size),+ _ => None,+ })+ .max()+ .unwrap_or(0);++ let max_state_stride = max_kernel_size.saturating_sub(1);+ Some([suffix_length + max_state_stride, self.model_dim])+ }+pub fn ssm_x_shape(&self,suffix_length: usize,diff --git a/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rsindex a74e214cc..27b132a11 100644--- a/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs+++ b/crates/uzu/src/backends/metal/forward_pass/scratch_buffers.rs@@ -30,6 +30,7 @@ pub struct ScratchBuffers {pub ssm_inproj: Option<MTLBuffer>,pub ssm_packed: Option<MTLBuffer>,pub ssm_conv_padded: Option<MTLBuffer>,+ pub short_conv_padded: Option<MTLBuffer>,pub ssm_x: Option<MTLBuffer>,pub ssm_b: Option<MTLBuffer>,pub ssm_c: Option<MTLBuffer>,@@ -156,6 +157,9 @@ impl ScratchBuffers {ssm_conv_padded: model_shape.ssm_conv_padded_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty)),+ short_conv_padded: model_shape+ .short_conv_padded_shape(max_suffix_len)+ .map(|shape| alloc(&shape, act_ty)),ssm_x: model_shape.ssm_x_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty)),diff --git a/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rsindex 4469de591..cce6e48cb 100644--- a/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state/language_model_generator_aux_buffers.rs@@ -11,6 +11,7 @@ pub struct LanguageModelGeneratorAuxBuffers {pub ssm_inproj: Option<ArrayCell>,pub ssm_packed: Option<ArrayCell>,pub ssm_conv_padded: Option<ArrayCell>,+ pub short_conv_padded: Option<ArrayCell>,pub ssm_x: Option<ArrayCell>,pub ssm_b: Option<ArrayCell>,pub ssm_c: Option<ArrayCell>,@@ -76,6 +77,15 @@ impl LanguageModelGeneratorAuxBuffers {)),_ => None,},+ short_conv_padded: match (+ scratch.short_conv_padded.as_ref(),+ model_shape.short_conv_padded_shape(suffix_length),+ ) {+ (Some(buf), Some(shape)) => Some(RefCell::new(+ MetalArray::new(buf.clone(), &shape, act_dtype),+ )),+ _ => None,+ },ssm_x: match (scratch.ssm_x.as_ref(),model_shape.ssm_x_shape(suffix_length),diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rsindex 03b98b33a..27fc8e53a 100644--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs@@ -904,6 +904,12 @@ impl ForwardPassState {self.llm_aux.as_ref().and_then(|aux| aux.ssm_conv_padded.clone())}+ pub fn short_conv_padded_buffer(&self) -> Option<ArrayCell> {+ self.llm_aux+ .as_ref()+ .and_then(|aux| aux.short_conv_padded.clone())+ }+// ========================================================================// Public API Methods (formerly trait methods)// ========================================================================diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rsindex 02de18eba..4ea13a2ba 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs@@ -18,15 +18,26 @@ fn fn_suffix(dt: KernelDataType) -> &'static str {}pub struct ShortConvKernel {+ pack_pipeline: MTLComputePipelineState,prefill_pipeline: MTLComputePipelineState,decode_pipeline: MTLComputePipelineState,}+pub struct ShortConvPackArguments<'a> {+ pub state_in: &'a MTLBuffer,+ pub in_proj: &'a MTLBuffer,+ pub padded: &'a MTLBuffer,+ pub state_stride: usize,+ pub suffix_len: usize,+ pub in_proj_stride: usize,+ pub model_dim: usize,+}+pub struct ShortConvPrefillArguments<'a> {+ pub padded: &'a MTLBuffer,pub in_proj: &'a MTLBuffer,pub w: &'a MTLBuffer,pub b: Option<&'a MTLBuffer>,- pub state_in: &'a MTLBuffer,pub out: &'a MTLBuffer,pub state_out: &'a MTLBuffer,pub suffix_len: usize,@@ -55,11 +66,16 @@ impl ShortConvKernel {context: &MTLContext,data_type: KernelDataType,) -> Result<Self, ShortConvKernelError> {+ let pack_name =+ format!("short_conv_pack_kernel_{}", fn_suffix(data_type));let prefill_name =format!("short_conv_prefill_kernel_{}", fn_suffix(data_type));let decode_name =format!("short_conv_decode_kernel_{}", fn_suffix(data_type));+ let pack_pipeline = context+ .compute_pipeline_state(&pack_name, None)+ .map_err(ShortConvKernelError::MetalError)?;let prefill_pipeline = context.compute_pipeline_state(&prefill_name, None).map_err(ShortConvKernelError::MetalError)?;@@ -68,11 +84,63 @@ impl ShortConvKernel {.map_err(ShortConvKernelError::MetalError)?;Ok(Self {+ pack_pipeline,prefill_pipeline,decode_pipeline,})}+ pub fn encode_pack(+ &self,+ compute_encoder: &ComputeCommandEncoderRef,+ args: ShortConvPackArguments,+ ) -> Result<(), ShortConvKernelError> {+ if args.model_dim == 0 || args.suffix_len == 0 {+ return Ok(());+ }++ compute_encoder.set_compute_pipeline_state(&self.pack_pipeline);+ compute_encoder.set_buffer(0, Some(args.state_in), 0);+ compute_encoder.set_buffer(1, Some(args.in_proj), 0);+ compute_encoder.set_buffer(2, Some(args.padded), 0);+ compute_encoder.set_bytes(+ 3,+ size_of::<usize>() as u64,+ &args.state_stride as *const usize as *const _,+ );+ compute_encoder.set_bytes(+ 4,+ size_of::<usize>() as u64,+ &args.suffix_len as *const usize as *const _,+ );+ compute_encoder.set_bytes(+ 5,+ size_of::<usize>() as u64,+ &args.in_proj_stride as *const usize as *const _,+ );+ compute_encoder.set_bytes(+ 6,+ size_of::<u32>() as u64,+ &(args.model_dim as u32) as *const u32 as *const _,+ );++ let threads_per_threadgroup = MTLSize {+ width: 32,+ height: 1,+ depth: 1,+ };+ let padded_rows = args.state_stride + args.suffix_len;+ let threadgroups = MTLSize {+ width: args.model_dim as u64,+ height: padded_rows as u64,+ depth: 1,+ };++ compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);++ Ok(())+ }+pub fn encode_prefill(&self,compute_encoder: &ComputeCommandEncoderRef,@@ -82,11 +150,14 @@ impl ShortConvKernel {return Ok(());}+ let tap_count = (args.kernel_size - 1).max(0);+ let work_len = args.suffix_len + tap_count as usize;+compute_encoder.set_compute_pipeline_state(&self.prefill_pipeline);- compute_encoder.set_buffer(0, Some(args.in_proj), 0);- compute_encoder.set_buffer(1, Some(args.w), 0);- compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);- compute_encoder.set_buffer(3, Some(args.state_in), 0);+ compute_encoder.set_buffer(0, Some(args.padded), 0);+ compute_encoder.set_buffer(1, Some(args.in_proj), 0);+ compute_encoder.set_buffer(2, Some(args.w), 0);+ compute_encoder.set_buffer(3, args.b.map(|v| &**v), 0);compute_encoder.set_buffer(4, Some(args.out), 0);compute_encoder.set_buffer(5, Some(args.state_out), 0);compute_encoder.set_bytes(@@ -121,7 +192,7 @@ impl ShortConvKernel {depth: 1,};let threadgroups = MTLSize {- width: args.suffix_len as u64,+ width: work_len as u64,height: args.model_dim as u64,depth: 1,};diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metalindex c95655e79..a9bbb2b86 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -3,12 +3,51 @@using namespace metal;+template <typename T>+kernel void short_conv_pack_kernel(+ device const T* state_in [[buffer(0)]],+ device const T* in_proj [[buffer(1)]],+ device T* padded [[buffer(2)]],+ constant const size_t& state_stride [[buffer(3)]],+ constant const size_t& suffix_len [[buffer(4)]],+ constant const size_t& in_proj_stride [[buffer(5)]],+ constant const uint& model_dim [[buffer(6)]],+ uint2 grid_idx [[thread_position_in_grid]]+) {+ const uint channel_idx = grid_idx.x;+ const uint row_idx = grid_idx.y;+ const size_t padded_rows = state_stride + suffix_len;++ if (channel_idx >= model_dim || row_idx >= padded_rows) {+ return;+ }++ const size_t padded_offset =+ size_t(row_idx) * model_dim + size_t(channel_idx);++ if (row_idx < state_stride) {+ // Copy from state+ const size_t state_idx = size_t(channel_idx) * state_stride + size_t(row_idx);+ padded[padded_offset] = state_in[state_idx];+ } else {+ // Compute gated input from in_proj+ const size_t token = row_idx - state_stride;+ const size_t in_proj_idx = size_t(token) * in_proj_stride + size_t(channel_idx);++ float pre_gate = float(in_proj[in_proj_idx]);+ float x_in = float(in_proj[in_proj_idx + 2 * model_dim]);+ float x = x_in * pre_gate;++ padded[padded_offset] = static_cast<T>(x);+ }+}+template <typename T>kernel void short_conv_prefill_kernel(- device const T* in_proj [[buffer(0)]],- device const T* w [[buffer(1)]],- device const T* b [[buffer(2)]],- device const T* state_in [[buffer(3)]],+ device const T* padded [[buffer(0)]],+ device const T* in_proj [[buffer(1)]],+ device const T* w [[buffer(2)]],+ device const T* b [[buffer(3)]],device T* out [[buffer(4)]],device T* state_out [[buffer(5)]],constant const size_t& suffix_len [[buffer(6)]],@@ -18,72 +57,53 @@ kernel void short_conv_prefill_kernel(constant const uint& model_dim [[buffer(10)]],uint2 grid_idx [[thread_position_in_grid]]) {+ const int tap_count = max(kernel_size - 1, 0);+ const size_t work_len = suffix_len + size_t(tap_count);+const uint token_idx = grid_idx.x;const uint channel_idx = grid_idx.y;- if (channel_idx >= model_dim || token_idx >= suffix_len) {+ if (channel_idx >= model_dim || token_idx >= work_len) {return;}- const int tap_count = max(kernel_size - 1, 0);- const size_t state_offset = size_t(channel_idx) * state_stride;const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);const bool has_bias = b != nullptr;- size_t current_in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);- float pre_conv_gate = float(in_proj[current_in_proj_idx]);- float post_conv_gate = float(in_proj[current_in_proj_idx + model_dim]);- float x_current = float(in_proj[current_in_proj_idx + 2 * model_dim]);+ // Threads [0..suffix_len-1]: Compute outputs+ if (token_idx < suffix_len) {+ float acc = has_bias ? float(b[channel_idx]) : 0.0f;- float gated_input = x_current * pre_conv_gate;-- float acc = has_bias ? float(b[channel_idx]) : 0.0f;-- for (int tap = 0; tap < kernel_size; ++tap) {- int src_token = int(token_idx) - (kernel_size - 1 - tap);- float sample = 0.0f;-- if (src_token < 0) {- int state_idx = tap_count + src_token;- if (state_idx >= 0 && state_idx < tap_count) {- sample = float(state_in[state_offset + size_t(state_idx)]);- }- } else if (src_token == int(token_idx)) {- sample = gated_input;- } else {- size_t src_in_proj_idx = size_t(src_token) * in_proj_stride + size_t(channel_idx);- float src_pre_gate = float(in_proj[src_in_proj_idx]);- float src_x = float(in_proj[src_in_proj_idx + 2 * model_dim]);- sample = src_x * src_pre_gate;+ // Convolve using padded buffer+ for (int tap = 0; tap < kernel_size; ++tap) {+ const size_t padded_row = size_t(token_idx + tap);+ const size_t padded_offset = padded_row * model_dim + channel_idx;+ float sample = float(padded[padded_offset]);+ acc += float(w_row[tap]) * sample;}- acc += float(w_row[tap]) * sample;- }+ // Apply post-gate from in_proj+ const size_t in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);+ float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);+ float gated_output = acc * post_conv_gate;- float gated_output = acc * post_conv_gate;+ // Write output+ const size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);+ out[out_idx] = static_cast<T>(gated_output);+ }+ // Threads [suffix_len..work_len-1]: Write state+ else if (tap_count > 0) {+ const size_t tap = size_t(token_idx - suffix_len);+ if (tap >= size_t(tap_count)) {+ return;+ }- size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);- out[out_idx] = static_cast<T>(gated_output);+ // Copy last tap_count values from padded to state_out+ const size_t padded_row = suffix_len + tap;+ const size_t padded_offset = padded_row * model_dim + channel_idx;+ const size_t state_idx = size_t(channel_idx) * state_stride + tap;- if (tap_count > 0 && token_idx == 0) {- for (int tap = 0; tap < tap_count; ++tap) {- int src_token = int(suffix_len) - tap_count + tap;- float sample = 0.0f;-- if (src_token < 0) {- int old_state_idx = tap_count + src_token;- if (old_state_idx >= 0 && old_state_idx < tap_count) {- sample = float(state_in[state_offset + size_t(old_state_idx)]);- }- } else {- size_t src_in_proj_idx = size_t(src_token) * in_proj_stride + size_t(channel_idx);- float src_pre_gate = float(in_proj[src_in_proj_idx]);- float src_x = float(in_proj[src_in_proj_idx + 2 * model_dim]);- sample = src_x * src_pre_gate;- }-- state_out[state_offset + size_t(tap)] = static_cast<T>(sample);- }+ state_out[state_idx] = padded[padded_offset];}}@@ -117,9 +137,9 @@ kernel void short_conv_decode_kernel(size_t in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);float pre_conv_gate = float(in_proj[in_proj_idx]);float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);- float x_val = float(in_proj[in_proj_idx + 2 * model_dim]);+ float x_in = float(in_proj[in_proj_idx + 2 * model_dim]);- float gated_input = x_val * pre_conv_gate;+ float x = x_in * pre_conv_gate;float acc = has_bias ? float(b[channel_idx]) : 0.0f;@@ -128,7 +148,7 @@ kernel void short_conv_decode_kernel(acc += float(w_row[tap]) * sample;}- acc += float(w_row[tap_count]) * gated_input;+ acc += float(w_row[tap_count]) * x;float gated_output = acc * post_conv_gate;@@ -139,44 +159,64 @@ kernel void short_conv_decode_kernel(for (int tap = 0; tap < tap_count - 1; ++tap) {next_state[state_offset + size_t(tap)] = state[state_offset + size_t(tap + 1)];}- next_state[state_offset + size_t(tap_count - 1)] = static_cast<T>(gated_input);+ next_state[state_offset + size_t(tap_count - 1)] = static_cast<T>(x);}}-#define instantiate_short_conv_prefill_kernel(type_name, type) \- template [[host_name("short_conv_prefill_kernel_" #type_name)]] \- kernel void short_conv_prefill_kernel<type>( \- device const type* x [[buffer(0)]], \- device const type* w [[buffer(1)]], \- device const type* b [[buffer(2)]], \- device const type* state_in [[buffer(3)]], \- device type* out [[buffer(4)]], \- device type* state_out [[buffer(5)]], \- constant const size_t& suffix_len [[buffer(6)]], \- constant const int& kernel_size [[buffer(7)]], \- constant const size_t& row_stride [[buffer(8)]], \- constant const size_t& state_stride [[buffer(9)]], \- constant const uint& num_channels [[buffer(10)]], \- uint2 grid_idx [[thread_position_in_grid]] \+// ============================================================================+// Template instantiations+// ============================================================================+#define instantiate_short_conv_pack_kernel(type_name, type) \+ template [[host_name("short_conv_pack_kernel_" #type_name)]] \+ kernel void short_conv_pack_kernel<type>( \+ device const type* state_in [[buffer(0)]], \+ device const type* in_proj [[buffer(1)]], \+ device type* padded [[buffer(2)]], \+ constant const size_t& state_stride [[buffer(3)]], \+ constant const size_t& suffix_len [[buffer(4)]], \+ constant const size_t& in_proj_stride [[buffer(5)]], \+ constant const uint& model_dim [[buffer(6)]], \+ uint2 grid_idx [[thread_position_in_grid]] \);-#define instantiate_short_conv_decode_kernel(type_name, type) \- template [[host_name("short_conv_decode_kernel_" #type_name)]] \- kernel void short_conv_decode_kernel<type>( \- device const type* x [[buffer(0)]], \- device const type* w [[buffer(1)]], \- device const type* b [[buffer(2)]], \- device const type* state [[buffer(3)]], \- device type* out [[buffer(4)]], \- device type* next_state [[buffer(5)]], \- constant const size_t& suffix_len [[buffer(6)]], \- constant const int& kernel_size [[buffer(7)]], \- constant const size_t& row_stride [[buffer(8)]], \- constant const size_t& state_stride [[buffer(9)]], \- constant const uint& num_channels [[buffer(10)]], \- uint2 grid_idx [[thread_position_in_grid]] \+#define instantiate_short_conv_prefill_kernel(type_name, type) \+ template [[host_name("short_conv_prefill_kernel_" #type_name)]] \+ kernel void short_conv_prefill_kernel<type>( \+ device const type* padded [[buffer(0)]], \+ device const type* in_proj [[buffer(1)]], \+ device const type* w [[buffer(2)]], \+ device const type* b [[buffer(3)]], \+ device type* out [[buffer(4)]], \+ device type* state_out [[buffer(5)]], \+ constant const size_t& suffix_len [[buffer(6)]], \+ constant const int& kernel_size [[buffer(7)]], \+ constant const size_t& in_proj_stride [[buffer(8)]], \+ constant const size_t& state_stride [[buffer(9)]], \+ constant const uint& model_dim [[buffer(10)]], \+ uint2 grid_idx [[thread_position_in_grid]] \);+#define instantiate_short_conv_decode_kernel(type_name, type) \+ template [[host_name("short_conv_decode_kernel_" #type_name)]] \+ kernel void short_conv_decode_kernel<type>( \+ device const type* x [[buffer(0)]], \+ device const type* w [[buffer(1)]], \+ device const type* b [[buffer(2)]], \+ device const type* state [[buffer(3)]], \+ device type* out [[buffer(4)]], \+ device type* next_state [[buffer(5)]], \+ constant const size_t& suffix_len [[buffer(6)]], \+ constant const int& kernel_size [[buffer(7)]], \+ constant const size_t& row_stride [[buffer(8)]], \+ constant const size_t& state_stride [[buffer(9)]], \+ constant const uint& num_channels [[buffer(10)]], \+ uint2 grid_idx [[thread_position_in_grid]] \+ );++instantiate_short_conv_pack_kernel(float, float);+instantiate_short_conv_pack_kernel(bfloat, bfloat);+instantiate_short_conv_pack_kernel(half, half);+instantiate_short_conv_prefill_kernel(float, float);instantiate_short_conv_prefill_kernel(bfloat, bfloat);instantiate_short_conv_prefill_kernel(half, half);@@ -185,6 +225,6 @@ instantiate_short_conv_decode_kernel(float, float);instantiate_short_conv_decode_kernel(bfloat, bfloat);instantiate_short_conv_decode_kernel(half, half);+#undef instantiate_short_conv_pack_kernel#undef instantiate_short_conv_prefill_kernel#undef instantiate_short_conv_decode_kernel-