A high-performance inference engine for AI models
dslify-short-conv (#185)
Aleksandr Golokoz6ee366f0c6cc
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rsindex c3328ecfe..b443231d1 100644--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs@@ -2,13 +2,16 @@ use super::{EncodableBlock, Metal, transformer_layer};use crate::{DataType,backends::{- common::Context,+ common::{+ Context,+ kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},+ },metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,Retained,- kernel::short_conv::{- ShortConvDecodeArguments, ShortConvKernel, ShortConvPackArguments, ShortConvPrefillArguments,- ShortConvTrieArguments,+ kernel::dsl::{+ ShortConvDecodeMetalKernel, ShortConvPackMetalKernel, ShortConvPrefillMetalKernel,+ ShortConvTrieMetalKernel,},},},@@ -24,7 +27,10 @@ pub(crate) struct ShortConvMixer {model_dim: usize,in_projection: Box<dyn EncodableBlock<Metal>>,out_projection: Box<dyn EncodableBlock<Metal>>,- short_conv_kernel: ShortConvKernel,+ short_conv_pack: ShortConvPackMetalKernel,+ short_conv_prefill: ShortConvPrefillMetalKernel,+ short_conv_decode: ShortConvDecodeMetalKernel,+ short_conv_trie: ShortConvTrieMetalKernel,conv_weight: MetalArray,conv_bias: Option<MetalArray>,}@@ -91,8 +97,15 @@ impl ShortConvMixer {None};- let short_conv_kernel =- ShortConvKernel::new(mtl_context, data_type).expect("Failed to create short conv kernel");+ let has_bias = short_conv_config.conv_config.has_biases;+ let short_conv_pack =+ ShortConvPackMetalKernel::new(mtl_context, data_type).expect("Failed to create short conv pack kernel");+ let short_conv_prefill = ShortConvPrefillMetalKernel::new(mtl_context, data_type, has_bias)+ .expect("Failed to create short conv prefill kernel");+ let short_conv_decode = ShortConvDecodeMetalKernel::new(mtl_context, data_type, has_bias)+ .expect("Failed to create short conv decode kernel");+ let short_conv_trie = ShortConvTrieMetalKernel::new(mtl_context, data_type, has_bias)+ .expect("Failed to create short conv trie kernel");Self {layer_index,@@ -100,7 +113,10 @@ impl ShortConvMixer {model_dim,in_projection,out_projection,- short_conv_kernel,+ short_conv_pack,+ short_conv_prefill,+ short_conv_decode,+ short_conv_trie,conv_weight,conv_bias,}@@ -220,6 +236,10 @@ impl ShortConvMixer {compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,suffix_length: usize,) {+ if self.model_dim == 0 || suffix_length == 0 {+ return;+ }+let arrays =state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);let in_proj = arrays[0].borrow_mut();@@ -232,10 +252,7 @@ impl ShortConvMixer {let conv_weight = self.conv_weight.clone();let weight_buf = conv_weight.buffer().clone();- let bias_buf = self.conv_bias.as_ref().map(|b| {- let b = b.clone();- b.buffer().clone()- });+ let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());let kernel_size = self.config.kernel_size;let state_stride = kernel_size.saturating_sub(1);@@ -246,39 +263,31 @@ impl ShortConvMixer {let padded_rows = state_stride + suffix_length;let padded_size = padded_rows * self.model_dim * element_size;let padded_buf = state.mtl_context().create_buffer(padded_size).expect("Failed to create padded buffer");- self.short_conv_kernel- .encode_pack(- compute,- ShortConvPackArguments {- state_in: &state_buf,- in_proj: &in_proj_buf,- padded: &padded_buf,- state_stride,- suffix_len: suffix_length,- in_proj_stride: self.model_dim * 3,- model_dim: self.model_dim,- },- )- .expect("Failed to encode short conv pack kernel");-- self.short_conv_kernel- .encode_prefill(- compute,- ShortConvPrefillArguments {- padded: &padded_buf,- in_proj: &in_proj_buf,- w: &weight_buf,- b: bias_buf.as_deref(),- out: &out_buf,- state_out: &state_buf,- suffix_len: suffix_length,- kernel_size: kernel_size as i32,- in_proj_stride: self.model_dim * 3,- state_stride,- model_dim: self.model_dim,- },- )- .expect("Failed to encode short conv prefill kernel");+ self.short_conv_pack.encode(+ &state_buf,+ &in_proj_buf,+ &padded_buf,+ state_stride as u32,+ suffix_length as u32,+ self.model_dim as u32 * 3,+ self.model_dim as u32,+ compute,+ );++ self.short_conv_prefill.encode(+ &padded_buf,+ &in_proj_buf,+ &weight_buf,+ bias_buf,+ &out_buf,+ &state_buf,+ suffix_length as u32,+ kernel_size as u32,+ self.model_dim as u32 * 3,+ state_stride as u32,+ self.model_dim as u32,+ compute,+ )}fn run_trie_conv(@@ -288,7 +297,7 @@ impl ShortConvMixer {sampling_start: usize,trie_len: usize,) {- if trie_len == 0 {+ if self.model_dim == 0 || trie_len == 0 {return;}@@ -327,35 +336,23 @@ impl ShortConvMixer {let conv_weight = self.conv_weight.clone();let weight_buf = conv_weight.buffer().clone();- let bias_buf = self.conv_bias.as_ref().map(|b| {- let b = b.clone();- b.buffer().clone()- });-- self.short_conv_kernel- .encode_trie(- compute,- ShortConvTrieArguments {- in_proj: &in_proj_buf,- in_proj_offset,- w: &weight_buf,- b: bias_buf.as_deref(),- base_state: &base_state_buf,- base_state_offset,- parents: &parents_buf,- parents_offset,- out: &out_buf,- out_offset,- suffix_state: &suffix_state_buf,- suffix_state_offset,- suffix_len: trie_len,- kernel_size: kernel_size as i32,- in_proj_stride,- state_stride,- model_dim: self.model_dim,- },- )- .expect("Failed to encode short conv trie kernel");+ let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());++ self.short_conv_trie.encode(+ (&in_proj_buf, in_proj_offset),+ &weight_buf,+ bias_buf,+ (&base_state_buf, base_state_offset),+ (&parents_buf, parents_offset),+ (&out_buf, out_offset),+ (&suffix_state_buf, suffix_state_offset),+ trie_len as u32,+ kernel_size as u32,+ in_proj_stride as u32,+ state_stride as u32,+ self.model_dim as u32,+ compute,+ );}fn run_decode_conv(@@ -364,6 +361,10 @@ impl ShortConvMixer {compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,suffix_length: usize,) {+ if self.model_dim == 0 || suffix_length == 0 {+ return;+ }+let arrays =state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);let in_proj = arrays[0].borrow_mut();@@ -376,32 +377,25 @@ impl ShortConvMixer {let conv_weight = self.conv_weight.clone();let weight_buf = conv_weight.buffer().clone();- let bias_buf = self.conv_bias.as_ref().map(|b| {- let b = b.clone();- b.buffer().clone()- });+ let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());let kernel_size = self.config.kernel_size;let state_stride = kernel_size.saturating_sub(1);- self.short_conv_kernel- .encode_decode(- compute,- ShortConvDecodeArguments {- in_proj: &in_proj_buf,- w: &weight_buf,- b: bias_buf.as_deref(),- state: &state_buf,- out: &out_buf,- next_state: &state_buf,- suffix_len: suffix_length,- kernel_size: kernel_size as i32,- in_proj_stride: self.model_dim * 3,- state_stride,- model_dim: self.model_dim,- },- )- .expect("Failed to encode short conv decode kernel");+ self.short_conv_decode.encode(+ &in_proj_buf,+ &weight_buf,+ bias_buf,+ &state_buf,+ &out_buf,+ &state_buf,+ suffix_length as u32,+ kernel_size as u32,+ self.model_dim as u32 * 3,+ state_stride as u32,+ self.model_dim as u32,+ compute,+ )}}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex f46e41e24..78e0e4d27 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -14,7 +14,6 @@ pub use mlp::{pub mod mlp_fused;pub mod moe;pub mod quant_matmul;-pub mod short_conv;pub mod ssm;pub use attention::{@@ -28,7 +27,6 @@ pub use moe::{MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,MoeScatterWithMapArguments,};-pub use short_conv::{ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError, ShortConvPrefillArguments};pub use ssm::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, SSDPrefillArguments, SSDPrefillMode, SSMKernelError,};diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rsdeleted file mode 100644index e0ee7dc4e..000000000--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs+++ /dev/null@@ -1,340 +0,0 @@-use crate::{- DataType,- backends::metal::{- ComputeEncoderSetValue, FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,- data_type::MetalDataTypeExt,- },-};--#[derive(Debug, thiserror::Error)]-pub enum ShortConvKernelError {- #[error("Metal error: {0}")]- MetalError(#[from] MTLError),-}--fn make_function_constants(has_bias: bool) -> Retained<MTLFunctionConstantValues> {- let function_constants = MTLFunctionConstantValues::new();- function_constants.set_value(&has_bias, 0);- function_constants-}--pub struct ShortConvKernel {- pack_pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- prefill_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- prefill_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- decode_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- decode_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- trie_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- trie_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,-}--pub struct ShortConvPackArguments<'a> {- pub state_in: &'a ProtocolObject<dyn MTLBuffer>,- pub in_proj: &'a ProtocolObject<dyn MTLBuffer>,- pub padded: &'a ProtocolObject<dyn MTLBuffer>,- pub state_stride: usize,- pub suffix_len: usize,- pub in_proj_stride: usize,- pub model_dim: usize,-}--pub struct ShortConvPrefillArguments<'a> {- pub padded: &'a ProtocolObject<dyn MTLBuffer>,- pub in_proj: &'a ProtocolObject<dyn MTLBuffer>,- pub w: &'a ProtocolObject<dyn MTLBuffer>,- pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>,- pub out: &'a ProtocolObject<dyn MTLBuffer>,- pub state_out: &'a ProtocolObject<dyn MTLBuffer>,- pub suffix_len: usize,- pub kernel_size: i32,- pub in_proj_stride: usize,- pub state_stride: usize,- pub model_dim: usize,-}--pub struct ShortConvDecodeArguments<'a> {- pub in_proj: &'a ProtocolObject<dyn MTLBuffer>,- pub w: &'a ProtocolObject<dyn MTLBuffer>,- pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>,- pub state: &'a ProtocolObject<dyn MTLBuffer>,- pub out: &'a ProtocolObject<dyn MTLBuffer>,- pub next_state: &'a ProtocolObject<dyn MTLBuffer>,- pub suffix_len: usize,- pub kernel_size: i32,- pub in_proj_stride: usize,- pub state_stride: usize,- pub model_dim: usize,-}--pub struct ShortConvTrieArguments<'a> {- pub in_proj: &'a ProtocolObject<dyn MTLBuffer>,- /// Byte offset into `in_proj`.- pub in_proj_offset: usize,- pub w: &'a ProtocolObject<dyn MTLBuffer>,- pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>,- pub base_state: &'a ProtocolObject<dyn MTLBuffer>,- /// Byte offset into `base_state`.- pub base_state_offset: usize,- pub parents: &'a ProtocolObject<dyn MTLBuffer>,- /// Byte offset into `parents`.- pub parents_offset: usize,- pub out: &'a ProtocolObject<dyn MTLBuffer>,- /// Byte offset into `out`.- pub out_offset: usize,- pub suffix_state: &'a ProtocolObject<dyn MTLBuffer>,- /// Byte offset into `suffix_state`.- pub suffix_state_offset: usize,- pub suffix_len: usize,- pub kernel_size: i32,- pub in_proj_stride: usize,- pub state_stride: usize,- pub model_dim: usize,-}--impl ShortConvKernel {- pub fn new(- context: &MTLContext,- data_type: DataType,- ) -> Result<Self, ShortConvKernelError> {- let pack_name = format!("short_conv_pack_kernel_{}", data_type.metal_type());- let prefill_name = format!("short_conv_prefill_kernel_{}", data_type.metal_type());- let decode_name = format!("short_conv_decode_kernel_{}", data_type.metal_type());- let trie_name = format!("short_conv_trie_kernel_{}", data_type.metal_type());-- let pack_pipeline =- context.compute_pipeline_state(&pack_name, None).map_err(ShortConvKernelError::MetalError)?;-- let prefill_pipeline_no_bias = {- let function_constants = make_function_constants(false);- let cache_key = format!("{}_has_bias_0", prefill_name);- context- .compute_pipeline_state_cached(&cache_key, &prefill_name, Some(&function_constants))- .map_err(ShortConvKernelError::MetalError)?- };- let prefill_pipeline_with_bias = {- let function_constants = make_function_constants(true);- let cache_key = format!("{}_has_bias_1", prefill_name);- context- .compute_pipeline_state_cached(&cache_key, &prefill_name, Some(&function_constants))- .map_err(ShortConvKernelError::MetalError)?- };- let decode_pipeline_no_bias = {- let function_constants = make_function_constants(false);- let cache_key = format!("{}_has_bias_0", decode_name);- context- .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))- .map_err(ShortConvKernelError::MetalError)?- };- let decode_pipeline_with_bias = {- let function_constants = make_function_constants(true);- let cache_key = format!("{}_has_bias_1", decode_name);- context- .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))- .map_err(ShortConvKernelError::MetalError)?- };-- let trie_pipeline_no_bias = {- let function_constants = make_function_constants(false);- let cache_key = format!("{}_has_bias_0", trie_name);- context- .compute_pipeline_state_cached(&cache_key, &trie_name, Some(&function_constants))- .map_err(ShortConvKernelError::MetalError)?- };- let trie_pipeline_with_bias = {- let function_constants = make_function_constants(true);- let cache_key = format!("{}_has_bias_1", trie_name);- context- .compute_pipeline_state_cached(&cache_key, &trie_name, Some(&function_constants))- .map_err(ShortConvKernelError::MetalError)?- };-- Ok(Self {- pack_pipeline,- prefill_pipeline_no_bias,- prefill_pipeline_with_bias,- decode_pipeline_no_bias,- decode_pipeline_with_bias,- trie_pipeline_no_bias,- trie_pipeline_with_bias,- })- }-- pub fn encode_pack(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: ShortConvPackArguments,- ) -> Result<(), ShortConvKernelError> {- if args.model_dim == 0 || args.suffix_len == 0 {- return Ok(());- }-- compute_encoder.set_compute_pipeline_state(&self.pack_pipeline);- compute_encoder.set_buffer(Some(args.state_in), 0, 0);- compute_encoder.set_buffer(Some(args.in_proj), 0, 1);- compute_encoder.set_buffer(Some(args.padded), 0, 2);- compute_encoder.set_value(&args.state_stride, 3);- compute_encoder.set_value(&args.suffix_len, 4);- compute_encoder.set_value(&args.in_proj_stride, 5);- compute_encoder.set_value(&(args.model_dim as u32), 6);-- let threads_per_threadgroup = MTLSize {- width: 32,- height: 1,- depth: 1,- };- let padded_rows = args.state_stride + args.suffix_len;- let threadgroups = MTLSize {- width: args.model_dim,- height: padded_rows,- depth: 1,- };-- compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);-- Ok(())- }-- pub fn encode_prefill(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: ShortConvPrefillArguments,- ) -> Result<(), ShortConvKernelError> {- if args.model_dim == 0 || args.suffix_len == 0 {- return Ok(());- }-- let tap_count = (args.kernel_size - 1).max(0);- let work_len = args.suffix_len + tap_count as usize;-- let has_bias = args.b.is_some();- let pipeline = if has_bias {- &self.prefill_pipeline_with_bias- } else {- &self.prefill_pipeline_no_bias- };- compute_encoder.set_compute_pipeline_state(pipeline);- compute_encoder.set_buffer(Some(args.padded), 0, 0);- compute_encoder.set_buffer(Some(args.in_proj), 0, 1);- compute_encoder.set_buffer(Some(args.w), 0, 2);- if has_bias {- compute_encoder.set_buffer(args.b, 0, 3);- }- compute_encoder.set_buffer(Some(args.out), 0, 4);- compute_encoder.set_buffer(Some(args.state_out), 0, 5);- compute_encoder.set_value(&args.suffix_len, 6);- compute_encoder.set_value(&args.kernel_size, 7);- compute_encoder.set_value(&args.in_proj_stride, 8);- compute_encoder.set_value(&args.state_stride, 9);- compute_encoder.set_value(&(args.model_dim as u32), 10);-- let threads_per_threadgroup = MTLSize {- width: 32,- height: 1,- depth: 1,- };- let threadgroups = MTLSize {- width: work_len,- height: args.model_dim,- depth: 1,- };-- compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);-- Ok(())- }-- pub fn encode_decode(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: ShortConvDecodeArguments,- ) -> Result<(), ShortConvKernelError> {- if args.model_dim == 0 || args.suffix_len == 0 {- return Ok(());- }-- let has_bias = args.b.is_some();- let pipeline = if has_bias {- &self.decode_pipeline_with_bias- } else {- &self.decode_pipeline_no_bias- };- compute_encoder.set_compute_pipeline_state(pipeline);- compute_encoder.set_buffer(Some(args.in_proj), 0, 0);- compute_encoder.set_buffer(Some(args.w), 0, 1);- if has_bias {- compute_encoder.set_buffer(args.b, 0, 2);- }- compute_encoder.set_buffer(Some(args.state), 0, 3);- compute_encoder.set_buffer(Some(args.out), 0, 4);- compute_encoder.set_buffer(Some(args.next_state), 0, 5);- compute_encoder.set_value(&args.suffix_len, 6);- compute_encoder.set_value(&args.kernel_size, 7);- compute_encoder.set_value(&args.in_proj_stride, 8);- compute_encoder.set_value(&args.state_stride, 9);- compute_encoder.set_value(&(args.model_dim as u32), 10);-- let threads_per_threadgroup = MTLSize {- width: 32,- height: 1,- depth: 1,- };- let threadgroups = MTLSize {- width: args.suffix_len,- height: args.model_dim,- depth: 1,- };-- compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);-- Ok(())- }-- pub fn encode_trie(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: ShortConvTrieArguments,- ) -> Result<(), ShortConvKernelError> {- if args.model_dim == 0 || args.suffix_len == 0 {- return Ok(());- }-- let has_bias = args.b.is_some();- let pipeline = if has_bias {- &self.trie_pipeline_with_bias- } else {- &self.trie_pipeline_no_bias- };- compute_encoder.set_compute_pipeline_state(pipeline);-- compute_encoder.set_buffer(Some(args.in_proj), args.in_proj_offset, 0);- compute_encoder.set_buffer(Some(args.w), 0, 1);- if has_bias {- compute_encoder.set_buffer(args.b, 0, 2);- }- compute_encoder.set_buffer(Some(args.base_state), args.base_state_offset, 3);- compute_encoder.set_buffer(Some(args.parents), args.parents_offset, 4);- compute_encoder.set_buffer(Some(args.out), args.out_offset, 5);- compute_encoder.set_buffer(Some(args.suffix_state), args.suffix_state_offset, 6);-- compute_encoder.set_value(&args.suffix_len, 7);- compute_encoder.set_value(&args.kernel_size, 8);- compute_encoder.set_value(&args.in_proj_stride, 9);- compute_encoder.set_value(&args.state_stride, 10);- compute_encoder.set_value(&(args.model_dim as u32), 11);-- let threads_per_threadgroup = MTLSize {- width: 32,- height: 1,- depth: 1,- };- let threadgroups = MTLSize {- width: args.model_dim,- height: 1,- depth: 1,- };-- compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metalindex 13d8b55b8..a5f81cad2 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -1,43 +1,28 @@#include <metal_stdlib>#include "../definitions.metal"-using namespace metal;--constant bool has_bias [[function_constant(0)]];-template <typename T>-[[kernel, max_total_threads_per_threadgroup(32)]]-void short_conv_pack_kernel(- device const T* state_in [[buffer(0)]],- device const T* in_proj [[buffer(1)]],- device T* padded [[buffer(2)]],- constant const size_t& state_stride [[buffer(3)]],- constant const size_t& suffix_len [[buffer(4)]],- constant const size_t& in_proj_stride [[buffer(5)]],- constant const uint& model_dim [[buffer(6)]],- uint2 grid_idx [[thread_position_in_grid]]+VARIANTS(T, float, half, bfloat)+KERNEL(ShortConvPack)(+ device const T* state_in,+ device const T* in_proj,+ device T* padded,+ constant const uint& state_stride,+ constant const uint& suffix_len,+ constant const uint& in_proj_stride,+ constant const uint& model_dim,+ const uint channel_idx AXIS(model_dim, 32),+ const uint row_idx AXIS(state_stride + suffix_len, 1)) {- const uint channel_idx = grid_idx.x;- const uint row_idx = grid_idx.y;- const size_t padded_rows = state_stride + suffix_len;-- if (channel_idx >= model_dim || row_idx >= padded_rows) {- return;- }-- const size_t padded_offset =- size_t(row_idx) * model_dim + size_t(channel_idx);-+ const uint padded_offset = row_idx * model_dim + channel_idx;if (row_idx < state_stride) {// Copy from state- const size_t state_idx =- size_t(channel_idx) * state_stride + size_t(row_idx);+ const uint state_idx = channel_idx * state_stride + row_idx;padded[padded_offset] = state_in[state_idx];} else {// Compute gated input from in_proj- const size_t token = row_idx - state_stride;- const size_t in_proj_idx =- size_t(token) * in_proj_stride + size_t(channel_idx);+ const uint token = row_idx - state_stride;+ const uint in_proj_idx = token * in_proj_stride + channel_idx;float pre_gate = float(in_proj[in_proj_idx]);float x_in = float(in_proj[in_proj_idx + 2 * model_dim]);@@ -48,32 +33,25 @@ void short_conv_pack_kernel(}template <typename T>-[[kernel, max_total_threads_per_threadgroup(32)]]-void short_conv_prefill_kernel(- device const T* padded [[buffer(0)]],- device const T* in_proj [[buffer(1)]],- device const T* w [[buffer(2)]],- device const T* b [[buffer(3), function_constant(has_bias)]],- device T* out [[buffer(4)]],- device T* state_out [[buffer(5)]],- constant const size_t& suffix_len [[buffer(6)]],- constant const int& kernel_size [[buffer(7)]],- constant const size_t& in_proj_stride [[buffer(8)]],- constant const size_t& state_stride [[buffer(9)]],- constant const uint& model_dim [[buffer(10)]],- uint2 grid_idx [[thread_position_in_grid]]+VARIANTS(T, float, half, bfloat)+KERNEL(ShortConvPrefill)(+ device const T* padded,+ device const T* in_proj,+ device const T* w,+ device const T* b OPTIONAL(has_bias),+ device T* out,+ device T* state_out,+ constant const uint& suffix_len,+ constant const uint& kernel_size,+ constant const uint& in_proj_stride,+ constant const uint& state_stride,+ constant const uint& model_dim,+ const bool has_bias SPECIALIZE,+ const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 32),+ const uint channel_idx AXIS(model_dim, 1)) {- const int tap_count = max(kernel_size - 1, 0);- const size_t work_len = suffix_len + size_t(tap_count);-- const uint token_idx = grid_idx.x;- const uint channel_idx = grid_idx.y;-- if (channel_idx >= model_dim || token_idx >= work_len) {- return;- }-- const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);+ const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;+ const device T* w_row = w + channel_idx * kernel_size;// Threads [0..suffix_len-1]: Compute outputsif (token_idx < suffix_len) {@@ -83,67 +61,61 @@ void short_conv_prefill_kernel(}// Convolve using padded buffer- for (int tap = 0; tap < kernel_size; ++tap) {- const size_t padded_row = size_t(token_idx + tap);- const size_t padded_offset = padded_row * model_dim + channel_idx;+ for (uint tap = 0; tap < kernel_size; ++tap) {+ const uint padded_row = token_idx + tap;+ const uint padded_offset = padded_row * model_dim + channel_idx;float sample = float(padded[padded_offset]);acc += float(w_row[tap]) * sample;}// Apply post-gate from in_proj- const size_t in_proj_idx =- size_t(token_idx) * in_proj_stride + size_t(channel_idx);+ const uint in_proj_idx = token_idx * in_proj_stride + channel_idx;float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);float gated_output = acc * post_conv_gate;// Write output- const size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);+ const uint out_idx = token_idx * model_dim + channel_idx;out[out_idx] = static_cast<T>(gated_output);}// Threads [suffix_len..work_len-1]: Write stateelse if (tap_count > 0) {- const size_t tap = size_t(token_idx - suffix_len);- if (tap >= size_t(tap_count)) {+ const uint tap = token_idx - suffix_len;+ if (tap >= tap_count) {return;}// Copy last tap_count values from padded to state_out- const size_t padded_row = suffix_len + tap;- const size_t padded_offset = padded_row * model_dim + channel_idx;- const size_t state_idx = size_t(channel_idx) * state_stride + tap;+ const uint padded_row = suffix_len + tap;+ const uint padded_offset = padded_row * model_dim + channel_idx;+ const uint state_idx = channel_idx * state_stride + tap;state_out[state_idx] = padded[padded_offset];}}template <typename T>-[[kernel, max_total_threads_per_threadgroup(32)]]-void short_conv_decode_kernel(- device const T* in_proj [[buffer(0)]],- device const T* w [[buffer(1)]],- device const T* b [[buffer(2), function_constant(has_bias)]],- device const T* state [[buffer(3)]],- device T* out [[buffer(4)]],- device T* next_state [[buffer(5)]],- constant const size_t& suffix_len [[buffer(6)]],- constant const int& kernel_size [[buffer(7)]],- constant const size_t& in_proj_stride [[buffer(8)]],- constant const size_t& state_stride [[buffer(9)]],- constant const uint& model_dim [[buffer(10)]],- uint2 grid_idx [[thread_position_in_grid]]+VARIANTS(T, float, half, bfloat)+KERNEL(ShortConvDecode)(+ device const T* in_proj,+ device const T* w,+ device const T* b OPTIONAL(has_bias),+ device const T* state,+ device T* out,+ device T* next_state,+ constant const uint& suffix_len,+ constant const uint& kernel_size,+ constant const uint& in_proj_stride,+ constant const uint& state_stride,+ constant const uint& model_dim,+ const bool has_bias SPECIALIZE,+ const uint token_idx AXIS(suffix_len, 32),+ const uint channel_idx AXIS(model_dim, 1)) {- const uint token_idx = grid_idx.x;- const uint channel_idx = grid_idx.y;-- if (channel_idx >= model_dim || token_idx >= suffix_len) {- return;- }-- const int tap_count = max(kernel_size - 1, 0);- const size_t state_offset = size_t(channel_idx) * state_stride;- const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);+ const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;+ const uint state_offset = channel_idx * state_stride;+ const device T* w_row = w + channel_idx * kernel_size;- size_t in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);+ uint in_proj_idx = token_idx * in_proj_stride + channel_idx;float pre_conv_gate = float(in_proj[in_proj_idx]);float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);float x_in = float(in_proj[in_proj_idx + 2 * model_dim]);@@ -155,8 +127,8 @@ void short_conv_decode_kernel(acc = float(b[channel_idx]);}- for (int tap = 0; tap < tap_count; ++tap) {- float sample = float(state[state_offset + size_t(tap)]);+ for (uint tap = 0; tap < tap_count; ++tap) {+ float sample = float(state[state_offset + tap]);acc += float(w_row[tap]) * sample;}@@ -164,46 +136,42 @@ void short_conv_decode_kernel(float gated_output = acc * post_conv_gate;- size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);+ uint out_idx = token_idx * model_dim + channel_idx;out[out_idx] = static_cast<T>(gated_output);if (tap_count > 0) {- for (int tap = 0; tap < tap_count - 1; ++tap) {- next_state[state_offset + size_t(tap)] =- state[state_offset + size_t(tap + 1)];+ for (uint tap = 0; tap < tap_count - 1; ++tap) {+ next_state[state_offset + tap] = state[state_offset + tap + 1];}- next_state[state_offset + size_t(tap_count - 1)] = static_cast<T>(x);+ next_state[state_offset + tap_count - 1] = static_cast<T>(x);}}template <typename T>-[[kernel, max_total_threads_per_threadgroup(32)]]-void short_conv_trie_kernel(- device const T* in_proj [[buffer(0)]],- device const T* w [[buffer(1)]],- device const T* b [[buffer(2), function_constant(has_bias)]],- device const T* base_state [[buffer(3)]],- device const int* parents [[buffer(4)]],- device T* out [[buffer(5)]],- device T* suffix_state [[buffer(6)]],- constant const size_t& suffix_len [[buffer(7)]],- constant const int& kernel_size [[buffer(8)]],- constant const size_t& in_proj_stride [[buffer(9)]],- constant const size_t& state_stride [[buffer(10)]],- constant const uint& model_dim [[buffer(11)]],- uint channel_idx [[thread_position_in_grid]]+VARIANTS(T, float, half, bfloat)+KERNEL(ShortConvTrie)(+ device const T* in_proj,+ device const T* w,+ device const T* b OPTIONAL(has_bias),+ device const T* base_state,+ device const int* parents,+ device T* out,+ device T* suffix_state,+ constant const uint& suffix_len,+ constant const uint& kernel_size,+ constant const uint& in_proj_stride,+ constant const uint& state_stride,+ constant const uint& model_dim,+ const bool has_bias SPECIALIZE,+ const uint channel_idx AXIS(model_dim, 32)) {- if (channel_idx >= model_dim) {- return;- }-- const int tap_count = max(kernel_size - 1, 0);- const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);- const size_t base_state_offset = size_t(channel_idx) * state_stride;+ const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;+ const device T* w_row = w + channel_idx * kernel_size;+ const uint base_state_offset = channel_idx * state_stride;- for (size_t node = 0; node < suffix_len; ++node) {+ for (uint node = 0; node < suffix_len; ++node) {// Read gated input from in_proj- const size_t in_proj_idx = node * in_proj_stride + size_t(channel_idx);+ const uint in_proj_idx = node * in_proj_stride + channel_idx;float pre_conv_gate = float(in_proj[in_proj_idx]);float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);float x_in = float(in_proj[in_proj_idx + 2 * model_dim]);@@ -214,15 +182,14 @@ void short_conv_trie_kernel(const device T* parent_state =(parent < 0) ? (base_state + base_state_offset): (suffix_state +- (size_t(parent) * model_dim + size_t(channel_idx)) *- state_stride);+ (parent * model_dim + channel_idx) * state_stride);float acc = 0.0f;if (has_bias) {acc = float(b[channel_idx]);}- for (int tap = 0; tap < tap_count; ++tap) {+ for (uint tap = 0; tap < tap_count; ++tap) {float sample = float(parent_state[size_t(tap)]);acc += float(w_row[tap]) * sample;}@@ -233,106 +200,16 @@ void short_conv_trie_kernel(float gated_output = acc * post_conv_gate;// Write output- out[node * model_dim + size_t(channel_idx)] = static_cast<T>(gated_output);+ out[node * model_dim + channel_idx] = static_cast<T>(gated_output);// Write post-state for this nodeif (tap_count > 0) {device T* dst_state =- suffix_state +- (node * model_dim + size_t(channel_idx)) * state_stride;- for (int tap = 0; tap < tap_count - 1; ++tap) {- dst_state[size_t(tap)] = parent_state[size_t(tap + 1)];+ suffix_state + (node * model_dim + channel_idx) * state_stride;+ for (uint tap = 0; tap < tap_count - 1; ++tap) {+ dst_state[tap] = parent_state[tap + 1];}- dst_state[size_t(tap_count - 1)] = static_cast<T>(x);+ dst_state[tap_count - 1] = static_cast<T>(x);}}-}--// ============================================================================-// Template instantiations-// ============================================================================-#define instantiate_short_conv_pack_kernel(type_name, type) \- template [[host_name("short_conv_pack_kernel_" #type_name)]] \- kernel void short_conv_pack_kernel<type>( \- device const type* state_in [[buffer(0)]], \- device const type* in_proj [[buffer(1)]], \- device type* padded [[buffer(2)]], \- constant const size_t& state_stride [[buffer(3)]], \- constant const size_t& suffix_len [[buffer(4)]], \- constant const size_t& in_proj_stride [[buffer(5)]], \- constant const uint& model_dim [[buffer(6)]], \- uint2 grid_idx [[thread_position_in_grid]] \- );--#define instantiate_short_conv_prefill_kernel(type_name, type) \- template [[host_name("short_conv_prefill_kernel_" #type_name)]] \- kernel void short_conv_prefill_kernel<type>( \- device const type* padded [[buffer(0)]], \- device const type* in_proj [[buffer(1)]], \- device const type* w [[buffer(2)]], \- device const type* b [[buffer(3), function_constant(has_bias)]], \- device type* out [[buffer(4)]], \- device type* state_out [[buffer(5)]], \- constant const size_t& suffix_len [[buffer(6)]], \- constant const int& kernel_size [[buffer(7)]], \- constant const size_t& in_proj_stride [[buffer(8)]], \- constant const size_t& state_stride [[buffer(9)]], \- constant const uint& model_dim [[buffer(10)]], \- uint2 grid_idx [[thread_position_in_grid]] \- );--#define instantiate_short_conv_decode_kernel(type_name, type) \- template [[host_name("short_conv_decode_kernel_" #type_name)]] \- kernel void short_conv_decode_kernel<type>( \- device const type* x [[buffer(0)]], \- device const type* w [[buffer(1)]], \- device const type* b [[buffer(2), function_constant(has_bias)]], \- device const type* state [[buffer(3)]], \- device type* out [[buffer(4)]], \- device type* next_state [[buffer(5)]], \- constant const size_t& suffix_len [[buffer(6)]], \- constant const int& kernel_size [[buffer(7)]], \- constant const size_t& row_stride [[buffer(8)]], \- constant const size_t& state_stride [[buffer(9)]], \- constant const uint& num_channels [[buffer(10)]], \- uint2 grid_idx [[thread_position_in_grid]] \- );--#define instantiate_short_conv_trie_kernel(type_name, type) \- template [[host_name("short_conv_trie_kernel_" #type_name)]] \- kernel void short_conv_trie_kernel<type>( \- device const type* in_proj [[buffer(0)]], \- device const type* w [[buffer(1)]], \- device const type* b [[buffer(2), function_constant(has_bias)]], \- device const type* base_state [[buffer(3)]], \- device const int* parents [[buffer(4)]], \- device type* out [[buffer(5)]], \- device type* suffix_state [[buffer(6)]], \- constant const size_t& suffix_len [[buffer(7)]], \- constant const int& kernel_size [[buffer(8)]], \- constant const size_t& in_proj_stride [[buffer(9)]], \- constant const size_t& state_stride [[buffer(10)]], \- constant const uint& model_dim [[buffer(11)]], \- uint channel_idx [[thread_position_in_grid]] \- );--instantiate_short_conv_pack_kernel(float, float);-instantiate_short_conv_pack_kernel(bfloat, bfloat);-instantiate_short_conv_pack_kernel(half, half);--instantiate_short_conv_prefill_kernel(float, float);-instantiate_short_conv_prefill_kernel(bfloat, bfloat);-instantiate_short_conv_prefill_kernel(half, half);--instantiate_short_conv_decode_kernel(float, float);-instantiate_short_conv_decode_kernel(bfloat, bfloat);-instantiate_short_conv_decode_kernel(half, half);--instantiate_short_conv_trie_kernel(float, float);-instantiate_short_conv_trie_kernel(bfloat, bfloat);-instantiate_short_conv_trie_kernel(half, half);--#undef instantiate_short_conv_pack_kernel-#undef instantiate_short_conv_prefill_kernel-#undef instantiate_short_conv_decode_kernel-#undef instantiate_short_conv_trie_kernel+}\ No newline at end of file