Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dslify-short-conv (#185)

Aleksandr Golokoz6ee366f0c6cc
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
index c3328ecfe..b443231d1 100644
--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
@@ -2,13 +2,16 @@ use super::{EncodableBlock, Metal, transformer_layer};
use crate::{
DataType,
backends::{
- common::Context,
+ common::{
+ Context,
+ kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},
+ },
metal::{
MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,
Retained,
- kernel::short_conv::{
- ShortConvDecodeArguments, ShortConvKernel, ShortConvPackArguments, ShortConvPrefillArguments,
- ShortConvTrieArguments,
+ kernel::dsl::{
+ ShortConvDecodeMetalKernel, ShortConvPackMetalKernel, ShortConvPrefillMetalKernel,
+ ShortConvTrieMetalKernel,
},
},
},
@@ -24,7 +27,10 @@ pub(crate) struct ShortConvMixer {
model_dim: usize,
in_projection: Box<dyn EncodableBlock<Metal>>,
out_projection: Box<dyn EncodableBlock<Metal>>,
- short_conv_kernel: ShortConvKernel,
+ short_conv_pack: ShortConvPackMetalKernel,
+ short_conv_prefill: ShortConvPrefillMetalKernel,
+ short_conv_decode: ShortConvDecodeMetalKernel,
+ short_conv_trie: ShortConvTrieMetalKernel,
conv_weight: MetalArray,
conv_bias: Option<MetalArray>,
}
@@ -91,8 +97,15 @@ impl ShortConvMixer {
None
};
- let short_conv_kernel =
- ShortConvKernel::new(mtl_context, data_type).expect("Failed to create short conv kernel");
+ let has_bias = short_conv_config.conv_config.has_biases;
+ let short_conv_pack =
+ ShortConvPackMetalKernel::new(mtl_context, data_type).expect("Failed to create short conv pack kernel");
+ let short_conv_prefill = ShortConvPrefillMetalKernel::new(mtl_context, data_type, has_bias)
+ .expect("Failed to create short conv prefill kernel");
+ let short_conv_decode = ShortConvDecodeMetalKernel::new(mtl_context, data_type, has_bias)
+ .expect("Failed to create short conv decode kernel");
+ let short_conv_trie = ShortConvTrieMetalKernel::new(mtl_context, data_type, has_bias)
+ .expect("Failed to create short conv trie kernel");
Self {
layer_index,
@@ -100,7 +113,10 @@ impl ShortConvMixer {
model_dim,
in_projection,
out_projection,
- short_conv_kernel,
+ short_conv_pack,
+ short_conv_prefill,
+ short_conv_decode,
+ short_conv_trie,
conv_weight,
conv_bias,
}
@@ -220,6 +236,10 @@ impl ShortConvMixer {
compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,
suffix_length: usize,
) {
+ if self.model_dim == 0 || suffix_length == 0 {
+ return;
+ }
+
let arrays =
state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);
let in_proj = arrays[0].borrow_mut();
@@ -232,10 +252,7 @@ impl ShortConvMixer {
let conv_weight = self.conv_weight.clone();
let weight_buf = conv_weight.buffer().clone();
- let bias_buf = self.conv_bias.as_ref().map(|b| {
- let b = b.clone();
- b.buffer().clone()
- });
+ let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());
let kernel_size = self.config.kernel_size;
let state_stride = kernel_size.saturating_sub(1);
@@ -246,39 +263,31 @@ impl ShortConvMixer {
let padded_rows = state_stride + suffix_length;
let padded_size = padded_rows * self.model_dim * element_size;
let padded_buf = state.mtl_context().create_buffer(padded_size).expect("Failed to create padded buffer");
- self.short_conv_kernel
- .encode_pack(
- compute,
- ShortConvPackArguments {
- state_in: &state_buf,
- in_proj: &in_proj_buf,
- padded: &padded_buf,
- state_stride,
- suffix_len: suffix_length,
- in_proj_stride: self.model_dim * 3,
- model_dim: self.model_dim,
- },
- )
- .expect("Failed to encode short conv pack kernel");
-
- self.short_conv_kernel
- .encode_prefill(
- compute,
- ShortConvPrefillArguments {
- padded: &padded_buf,
- in_proj: &in_proj_buf,
- w: &weight_buf,
- b: bias_buf.as_deref(),
- out: &out_buf,
- state_out: &state_buf,
- suffix_len: suffix_length,
- kernel_size: kernel_size as i32,
- in_proj_stride: self.model_dim * 3,
- state_stride,
- model_dim: self.model_dim,
- },
- )
- .expect("Failed to encode short conv prefill kernel");
+ self.short_conv_pack.encode(
+ &state_buf,
+ &in_proj_buf,
+ &padded_buf,
+ state_stride as u32,
+ suffix_length as u32,
+ self.model_dim as u32 * 3,
+ self.model_dim as u32,
+ compute,
+ );
+
+ self.short_conv_prefill.encode(
+ &padded_buf,
+ &in_proj_buf,
+ &weight_buf,
+ bias_buf,
+ &out_buf,
+ &state_buf,
+ suffix_length as u32,
+ kernel_size as u32,
+ self.model_dim as u32 * 3,
+ state_stride as u32,
+ self.model_dim as u32,
+ compute,
+ )
}
fn run_trie_conv(
@@ -288,7 +297,7 @@ impl ShortConvMixer {
sampling_start: usize,
trie_len: usize,
) {
- if trie_len == 0 {
+ if self.model_dim == 0 || trie_len == 0 {
return;
}
@@ -327,35 +336,23 @@ impl ShortConvMixer {
let conv_weight = self.conv_weight.clone();
let weight_buf = conv_weight.buffer().clone();
- let bias_buf = self.conv_bias.as_ref().map(|b| {
- let b = b.clone();
- b.buffer().clone()
- });
-
- self.short_conv_kernel
- .encode_trie(
- compute,
- ShortConvTrieArguments {
- in_proj: &in_proj_buf,
- in_proj_offset,
- w: &weight_buf,
- b: bias_buf.as_deref(),
- base_state: &base_state_buf,
- base_state_offset,
- parents: &parents_buf,
- parents_offset,
- out: &out_buf,
- out_offset,
- suffix_state: &suffix_state_buf,
- suffix_state_offset,
- suffix_len: trie_len,
- kernel_size: kernel_size as i32,
- in_proj_stride,
- state_stride,
- model_dim: self.model_dim,
- },
- )
- .expect("Failed to encode short conv trie kernel");
+ let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());
+
+ self.short_conv_trie.encode(
+ (&in_proj_buf, in_proj_offset),
+ &weight_buf,
+ bias_buf,
+ (&base_state_buf, base_state_offset),
+ (&parents_buf, parents_offset),
+ (&out_buf, out_offset),
+ (&suffix_state_buf, suffix_state_offset),
+ trie_len as u32,
+ kernel_size as u32,
+ in_proj_stride as u32,
+ state_stride as u32,
+ self.model_dim as u32,
+ compute,
+ );
}
fn run_decode_conv(
@@ -364,6 +361,10 @@ impl ShortConvMixer {
compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,
suffix_length: usize,
) {
+ if self.model_dim == 0 || suffix_length == 0 {
+ return;
+ }
+
let arrays =
state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);
let in_proj = arrays[0].borrow_mut();
@@ -376,32 +377,25 @@ impl ShortConvMixer {
let conv_weight = self.conv_weight.clone();
let weight_buf = conv_weight.buffer().clone();
- let bias_buf = self.conv_bias.as_ref().map(|b| {
- let b = b.clone();
- b.buffer().clone()
- });
+ let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());
let kernel_size = self.config.kernel_size;
let state_stride = kernel_size.saturating_sub(1);
- self.short_conv_kernel
- .encode_decode(
- compute,
- ShortConvDecodeArguments {
- in_proj: &in_proj_buf,
- w: &weight_buf,
- b: bias_buf.as_deref(),
- state: &state_buf,
- out: &out_buf,
- next_state: &state_buf,
- suffix_len: suffix_length,
- kernel_size: kernel_size as i32,
- in_proj_stride: self.model_dim * 3,
- state_stride,
- model_dim: self.model_dim,
- },
- )
- .expect("Failed to encode short conv decode kernel");
+ self.short_conv_decode.encode(
+ &in_proj_buf,
+ &weight_buf,
+ bias_buf,
+ &state_buf,
+ &out_buf,
+ &state_buf,
+ suffix_length as u32,
+ kernel_size as u32,
+ self.model_dim as u32 * 3,
+ state_stride as u32,
+ self.model_dim as u32,
+ compute,
+ )
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index f46e41e24..78e0e4d27 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -14,7 +14,6 @@ pub use mlp::{
pub mod mlp_fused;
pub mod moe;
pub mod quant_matmul;
-pub mod short_conv;
pub mod ssm;
pub use attention::{
@@ -28,7 +27,6 @@ pub use moe::{
MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,
MoeScatterWithMapArguments,
};
-pub use short_conv::{ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError, ShortConvPrefillArguments};
pub use ssm::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, SSDPrefillArguments, SSDPrefillMode, SSMKernelError,
};
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
deleted file mode 100644
index e0ee7dc4e..000000000
--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
+++ /dev/null
@@ -1,340 +0,0 @@
-use crate::{
- DataType,
- backends::metal::{
- ComputeEncoderSetValue, FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,
- data_type::MetalDataTypeExt,
- },
-};
-
-#[derive(Debug, thiserror::Error)]
-pub enum ShortConvKernelError {
- #[error("Metal error: {0}")]
- MetalError(#[from] MTLError),
-}
-
-fn make_function_constants(has_bias: bool) -> Retained<MTLFunctionConstantValues> {
- let function_constants = MTLFunctionConstantValues::new();
- function_constants.set_value(&has_bias, 0);
- function_constants
-}
-
-pub struct ShortConvKernel {
- pack_pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- prefill_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- prefill_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- decode_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- decode_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- trie_pipeline_no_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- trie_pipeline_with_bias: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
-}
-
-pub struct ShortConvPackArguments<'a> {
- pub state_in: &'a ProtocolObject<dyn MTLBuffer>,
- pub in_proj: &'a ProtocolObject<dyn MTLBuffer>,
- pub padded: &'a ProtocolObject<dyn MTLBuffer>,
- pub state_stride: usize,
- pub suffix_len: usize,
- pub in_proj_stride: usize,
- pub model_dim: usize,
-}
-
-pub struct ShortConvPrefillArguments<'a> {
- pub padded: &'a ProtocolObject<dyn MTLBuffer>,
- pub in_proj: &'a ProtocolObject<dyn MTLBuffer>,
- pub w: &'a ProtocolObject<dyn MTLBuffer>,
- pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>,
- pub out: &'a ProtocolObject<dyn MTLBuffer>,
- pub state_out: &'a ProtocolObject<dyn MTLBuffer>,
- pub suffix_len: usize,
- pub kernel_size: i32,
- pub in_proj_stride: usize,
- pub state_stride: usize,
- pub model_dim: usize,
-}
-
-pub struct ShortConvDecodeArguments<'a> {
- pub in_proj: &'a ProtocolObject<dyn MTLBuffer>,
- pub w: &'a ProtocolObject<dyn MTLBuffer>,
- pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>,
- pub state: &'a ProtocolObject<dyn MTLBuffer>,
- pub out: &'a ProtocolObject<dyn MTLBuffer>,
- pub next_state: &'a ProtocolObject<dyn MTLBuffer>,
- pub suffix_len: usize,
- pub kernel_size: i32,
- pub in_proj_stride: usize,
- pub state_stride: usize,
- pub model_dim: usize,
-}
-
-pub struct ShortConvTrieArguments<'a> {
- pub in_proj: &'a ProtocolObject<dyn MTLBuffer>,
- /// Byte offset into `in_proj`.
- pub in_proj_offset: usize,
- pub w: &'a ProtocolObject<dyn MTLBuffer>,
- pub b: Option<&'a ProtocolObject<dyn MTLBuffer>>,
- pub base_state: &'a ProtocolObject<dyn MTLBuffer>,
- /// Byte offset into `base_state`.
- pub base_state_offset: usize,
- pub parents: &'a ProtocolObject<dyn MTLBuffer>,
- /// Byte offset into `parents`.
- pub parents_offset: usize,
- pub out: &'a ProtocolObject<dyn MTLBuffer>,
- /// Byte offset into `out`.
- pub out_offset: usize,
- pub suffix_state: &'a ProtocolObject<dyn MTLBuffer>,
- /// Byte offset into `suffix_state`.
- pub suffix_state_offset: usize,
- pub suffix_len: usize,
- pub kernel_size: i32,
- pub in_proj_stride: usize,
- pub state_stride: usize,
- pub model_dim: usize,
-}
-
-impl ShortConvKernel {
- pub fn new(
- context: &MTLContext,
- data_type: DataType,
- ) -> Result<Self, ShortConvKernelError> {
- let pack_name = format!("short_conv_pack_kernel_{}", data_type.metal_type());
- let prefill_name = format!("short_conv_prefill_kernel_{}", data_type.metal_type());
- let decode_name = format!("short_conv_decode_kernel_{}", data_type.metal_type());
- let trie_name = format!("short_conv_trie_kernel_{}", data_type.metal_type());
-
- let pack_pipeline =
- context.compute_pipeline_state(&pack_name, None).map_err(ShortConvKernelError::MetalError)?;
-
- let prefill_pipeline_no_bias = {
- let function_constants = make_function_constants(false);
- let cache_key = format!("{}_has_bias_0", prefill_name);
- context
- .compute_pipeline_state_cached(&cache_key, &prefill_name, Some(&function_constants))
- .map_err(ShortConvKernelError::MetalError)?
- };
- let prefill_pipeline_with_bias = {
- let function_constants = make_function_constants(true);
- let cache_key = format!("{}_has_bias_1", prefill_name);
- context
- .compute_pipeline_state_cached(&cache_key, &prefill_name, Some(&function_constants))
- .map_err(ShortConvKernelError::MetalError)?
- };
- let decode_pipeline_no_bias = {
- let function_constants = make_function_constants(false);
- let cache_key = format!("{}_has_bias_0", decode_name);
- context
- .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))
- .map_err(ShortConvKernelError::MetalError)?
- };
- let decode_pipeline_with_bias = {
- let function_constants = make_function_constants(true);
- let cache_key = format!("{}_has_bias_1", decode_name);
- context
- .compute_pipeline_state_cached(&cache_key, &decode_name, Some(&function_constants))
- .map_err(ShortConvKernelError::MetalError)?
- };
-
- let trie_pipeline_no_bias = {
- let function_constants = make_function_constants(false);
- let cache_key = format!("{}_has_bias_0", trie_name);
- context
- .compute_pipeline_state_cached(&cache_key, &trie_name, Some(&function_constants))
- .map_err(ShortConvKernelError::MetalError)?
- };
- let trie_pipeline_with_bias = {
- let function_constants = make_function_constants(true);
- let cache_key = format!("{}_has_bias_1", trie_name);
- context
- .compute_pipeline_state_cached(&cache_key, &trie_name, Some(&function_constants))
- .map_err(ShortConvKernelError::MetalError)?
- };
-
- Ok(Self {
- pack_pipeline,
- prefill_pipeline_no_bias,
- prefill_pipeline_with_bias,
- decode_pipeline_no_bias,
- decode_pipeline_with_bias,
- trie_pipeline_no_bias,
- trie_pipeline_with_bias,
- })
- }
-
- pub fn encode_pack(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: ShortConvPackArguments,
- ) -> Result<(), ShortConvKernelError> {
- if args.model_dim == 0 || args.suffix_len == 0 {
- return Ok(());
- }
-
- compute_encoder.set_compute_pipeline_state(&self.pack_pipeline);
- compute_encoder.set_buffer(Some(args.state_in), 0, 0);
- compute_encoder.set_buffer(Some(args.in_proj), 0, 1);
- compute_encoder.set_buffer(Some(args.padded), 0, 2);
- compute_encoder.set_value(&args.state_stride, 3);
- compute_encoder.set_value(&args.suffix_len, 4);
- compute_encoder.set_value(&args.in_proj_stride, 5);
- compute_encoder.set_value(&(args.model_dim as u32), 6);
-
- let threads_per_threadgroup = MTLSize {
- width: 32,
- height: 1,
- depth: 1,
- };
- let padded_rows = args.state_stride + args.suffix_len;
- let threadgroups = MTLSize {
- width: args.model_dim,
- height: padded_rows,
- depth: 1,
- };
-
- compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);
-
- Ok(())
- }
-
- pub fn encode_prefill(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: ShortConvPrefillArguments,
- ) -> Result<(), ShortConvKernelError> {
- if args.model_dim == 0 || args.suffix_len == 0 {
- return Ok(());
- }
-
- let tap_count = (args.kernel_size - 1).max(0);
- let work_len = args.suffix_len + tap_count as usize;
-
- let has_bias = args.b.is_some();
- let pipeline = if has_bias {
- &self.prefill_pipeline_with_bias
- } else {
- &self.prefill_pipeline_no_bias
- };
- compute_encoder.set_compute_pipeline_state(pipeline);
- compute_encoder.set_buffer(Some(args.padded), 0, 0);
- compute_encoder.set_buffer(Some(args.in_proj), 0, 1);
- compute_encoder.set_buffer(Some(args.w), 0, 2);
- if has_bias {
- compute_encoder.set_buffer(args.b, 0, 3);
- }
- compute_encoder.set_buffer(Some(args.out), 0, 4);
- compute_encoder.set_buffer(Some(args.state_out), 0, 5);
- compute_encoder.set_value(&args.suffix_len, 6);
- compute_encoder.set_value(&args.kernel_size, 7);
- compute_encoder.set_value(&args.in_proj_stride, 8);
- compute_encoder.set_value(&args.state_stride, 9);
- compute_encoder.set_value(&(args.model_dim as u32), 10);
-
- let threads_per_threadgroup = MTLSize {
- width: 32,
- height: 1,
- depth: 1,
- };
- let threadgroups = MTLSize {
- width: work_len,
- height: args.model_dim,
- depth: 1,
- };
-
- compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);
-
- Ok(())
- }
-
- pub fn encode_decode(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: ShortConvDecodeArguments,
- ) -> Result<(), ShortConvKernelError> {
- if args.model_dim == 0 || args.suffix_len == 0 {
- return Ok(());
- }
-
- let has_bias = args.b.is_some();
- let pipeline = if has_bias {
- &self.decode_pipeline_with_bias
- } else {
- &self.decode_pipeline_no_bias
- };
- compute_encoder.set_compute_pipeline_state(pipeline);
- compute_encoder.set_buffer(Some(args.in_proj), 0, 0);
- compute_encoder.set_buffer(Some(args.w), 0, 1);
- if has_bias {
- compute_encoder.set_buffer(args.b, 0, 2);
- }
- compute_encoder.set_buffer(Some(args.state), 0, 3);
- compute_encoder.set_buffer(Some(args.out), 0, 4);
- compute_encoder.set_buffer(Some(args.next_state), 0, 5);
- compute_encoder.set_value(&args.suffix_len, 6);
- compute_encoder.set_value(&args.kernel_size, 7);
- compute_encoder.set_value(&args.in_proj_stride, 8);
- compute_encoder.set_value(&args.state_stride, 9);
- compute_encoder.set_value(&(args.model_dim as u32), 10);
-
- let threads_per_threadgroup = MTLSize {
- width: 32,
- height: 1,
- depth: 1,
- };
- let threadgroups = MTLSize {
- width: args.suffix_len,
- height: args.model_dim,
- depth: 1,
- };
-
- compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);
-
- Ok(())
- }
-
- pub fn encode_trie(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: ShortConvTrieArguments,
- ) -> Result<(), ShortConvKernelError> {
- if args.model_dim == 0 || args.suffix_len == 0 {
- return Ok(());
- }
-
- let has_bias = args.b.is_some();
- let pipeline = if has_bias {
- &self.trie_pipeline_with_bias
- } else {
- &self.trie_pipeline_no_bias
- };
- compute_encoder.set_compute_pipeline_state(pipeline);
-
- compute_encoder.set_buffer(Some(args.in_proj), args.in_proj_offset, 0);
- compute_encoder.set_buffer(Some(args.w), 0, 1);
- if has_bias {
- compute_encoder.set_buffer(args.b, 0, 2);
- }
- compute_encoder.set_buffer(Some(args.base_state), args.base_state_offset, 3);
- compute_encoder.set_buffer(Some(args.parents), args.parents_offset, 4);
- compute_encoder.set_buffer(Some(args.out), args.out_offset, 5);
- compute_encoder.set_buffer(Some(args.suffix_state), args.suffix_state_offset, 6);
-
- compute_encoder.set_value(&args.suffix_len, 7);
- compute_encoder.set_value(&args.kernel_size, 8);
- compute_encoder.set_value(&args.in_proj_stride, 9);
- compute_encoder.set_value(&args.state_stride, 10);
- compute_encoder.set_value(&(args.model_dim as u32), 11);
-
- let threads_per_threadgroup = MTLSize {
- width: 32,
- height: 1,
- depth: 1,
- };
- let threadgroups = MTLSize {
- width: args.model_dim,
- height: 1,
- depth: 1,
- };
-
- compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
index 13d8b55b8..a5f81cad2 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -1,43 +1,28 @@
#include <metal_stdlib>
#include "../definitions.metal"
-using namespace metal;
-
-constant bool has_bias [[function_constant(0)]];
-
template <typename T>
-[[kernel, max_total_threads_per_threadgroup(32)]]
-void short_conv_pack_kernel(
- device const T* state_in [[buffer(0)]],
- device const T* in_proj [[buffer(1)]],
- device T* padded [[buffer(2)]],
- constant const size_t& state_stride [[buffer(3)]],
- constant const size_t& suffix_len [[buffer(4)]],
- constant const size_t& in_proj_stride [[buffer(5)]],
- constant const uint& model_dim [[buffer(6)]],
- uint2 grid_idx [[thread_position_in_grid]]
+VARIANTS(T, float, half, bfloat)
+KERNEL(ShortConvPack)(
+ device const T* state_in,
+ device const T* in_proj,
+ device T* padded,
+ constant const uint& state_stride,
+ constant const uint& suffix_len,
+ constant const uint& in_proj_stride,
+ constant const uint& model_dim,
+ const uint channel_idx AXIS(model_dim, 32),
+ const uint row_idx AXIS(state_stride + suffix_len, 1)
) {
- const uint channel_idx = grid_idx.x;
- const uint row_idx = grid_idx.y;
- const size_t padded_rows = state_stride + suffix_len;
-
- if (channel_idx >= model_dim || row_idx >= padded_rows) {
- return;
- }
-
- const size_t padded_offset =
- size_t(row_idx) * model_dim + size_t(channel_idx);
-
+ const uint padded_offset = row_idx * model_dim + channel_idx;
if (row_idx < state_stride) {
// Copy from state
- const size_t state_idx =
- size_t(channel_idx) * state_stride + size_t(row_idx);
+ const uint state_idx = channel_idx * state_stride + row_idx;
padded[padded_offset] = state_in[state_idx];
} else {
// Compute gated input from in_proj
- const size_t token = row_idx - state_stride;
- const size_t in_proj_idx =
- size_t(token) * in_proj_stride + size_t(channel_idx);
+ const uint token = row_idx - state_stride;
+ const uint in_proj_idx = token * in_proj_stride + channel_idx;
float pre_gate = float(in_proj[in_proj_idx]);
float x_in = float(in_proj[in_proj_idx + 2 * model_dim]);
@@ -48,32 +33,25 @@ void short_conv_pack_kernel(
}
template <typename T>
-[[kernel, max_total_threads_per_threadgroup(32)]]
-void short_conv_prefill_kernel(
- device const T* padded [[buffer(0)]],
- device const T* in_proj [[buffer(1)]],
- device const T* w [[buffer(2)]],
- device const T* b [[buffer(3), function_constant(has_bias)]],
- device T* out [[buffer(4)]],
- device T* state_out [[buffer(5)]],
- constant const size_t& suffix_len [[buffer(6)]],
- constant const int& kernel_size [[buffer(7)]],
- constant const size_t& in_proj_stride [[buffer(8)]],
- constant const size_t& state_stride [[buffer(9)]],
- constant const uint& model_dim [[buffer(10)]],
- uint2 grid_idx [[thread_position_in_grid]]
+VARIANTS(T, float, half, bfloat)
+KERNEL(ShortConvPrefill)(
+ device const T* padded,
+ device const T* in_proj,
+ device const T* w,
+ device const T* b OPTIONAL(has_bias),
+ device T* out,
+ device T* state_out,
+ constant const uint& suffix_len,
+ constant const uint& kernel_size,
+ constant const uint& in_proj_stride,
+ constant const uint& state_stride,
+ constant const uint& model_dim,
+ const bool has_bias SPECIALIZE,
+ const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 32),
+ const uint channel_idx AXIS(model_dim, 1)
) {
- const int tap_count = max(kernel_size - 1, 0);
- const size_t work_len = suffix_len + size_t(tap_count);
-
- const uint token_idx = grid_idx.x;
- const uint channel_idx = grid_idx.y;
-
- if (channel_idx >= model_dim || token_idx >= work_len) {
- return;
- }
-
- const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);
+ const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;
+ const device T* w_row = w + channel_idx * kernel_size;
// Threads [0..suffix_len-1]: Compute outputs
if (token_idx < suffix_len) {
@@ -83,67 +61,61 @@ void short_conv_prefill_kernel(
}
// Convolve using padded buffer
- for (int tap = 0; tap < kernel_size; ++tap) {
- const size_t padded_row = size_t(token_idx + tap);
- const size_t padded_offset = padded_row * model_dim + channel_idx;
+ for (uint tap = 0; tap < kernel_size; ++tap) {
+ const uint padded_row = token_idx + tap;
+ const uint padded_offset = padded_row * model_dim + channel_idx;
float sample = float(padded[padded_offset]);
acc += float(w_row[tap]) * sample;
}
// Apply post-gate from in_proj
- const size_t in_proj_idx =
- size_t(token_idx) * in_proj_stride + size_t(channel_idx);
+ const uint in_proj_idx = token_idx * in_proj_stride + channel_idx;
float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);
float gated_output = acc * post_conv_gate;
// Write output
- const size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);
+ const uint out_idx = token_idx * model_dim + channel_idx;
out[out_idx] = static_cast<T>(gated_output);
}
// Threads [suffix_len..work_len-1]: Write state
else if (tap_count > 0) {
- const size_t tap = size_t(token_idx - suffix_len);
- if (tap >= size_t(tap_count)) {
+ const uint tap = token_idx - suffix_len;
+ if (tap >= tap_count) {
return;
}
// Copy last tap_count values from padded to state_out
- const size_t padded_row = suffix_len + tap;
- const size_t padded_offset = padded_row * model_dim + channel_idx;
- const size_t state_idx = size_t(channel_idx) * state_stride + tap;
+ const uint padded_row = suffix_len + tap;
+ const uint padded_offset = padded_row * model_dim + channel_idx;
+ const uint state_idx = channel_idx * state_stride + tap;
state_out[state_idx] = padded[padded_offset];
}
}
template <typename T>
-[[kernel, max_total_threads_per_threadgroup(32)]]
-void short_conv_decode_kernel(
- device const T* in_proj [[buffer(0)]],
- device const T* w [[buffer(1)]],
- device const T* b [[buffer(2), function_constant(has_bias)]],
- device const T* state [[buffer(3)]],
- device T* out [[buffer(4)]],
- device T* next_state [[buffer(5)]],
- constant const size_t& suffix_len [[buffer(6)]],
- constant const int& kernel_size [[buffer(7)]],
- constant const size_t& in_proj_stride [[buffer(8)]],
- constant const size_t& state_stride [[buffer(9)]],
- constant const uint& model_dim [[buffer(10)]],
- uint2 grid_idx [[thread_position_in_grid]]
+VARIANTS(T, float, half, bfloat)
+KERNEL(ShortConvDecode)(
+ device const T* in_proj,
+ device const T* w,
+ device const T* b OPTIONAL(has_bias),
+ device const T* state,
+ device T* out,
+ device T* next_state,
+ constant const uint& suffix_len,
+ constant const uint& kernel_size,
+ constant const uint& in_proj_stride,
+ constant const uint& state_stride,
+ constant const uint& model_dim,
+ const bool has_bias SPECIALIZE,
+ const uint token_idx AXIS(suffix_len, 32),
+ const uint channel_idx AXIS(model_dim, 1)
) {
- const uint token_idx = grid_idx.x;
- const uint channel_idx = grid_idx.y;
-
- if (channel_idx >= model_dim || token_idx >= suffix_len) {
- return;
- }
-
- const int tap_count = max(kernel_size - 1, 0);
- const size_t state_offset = size_t(channel_idx) * state_stride;
- const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);
+ const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;
+ const uint state_offset = channel_idx * state_stride;
+ const device T* w_row = w + channel_idx * kernel_size;
- size_t in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);
+ uint in_proj_idx = token_idx * in_proj_stride + channel_idx;
float pre_conv_gate = float(in_proj[in_proj_idx]);
float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);
float x_in = float(in_proj[in_proj_idx + 2 * model_dim]);
@@ -155,8 +127,8 @@ void short_conv_decode_kernel(
acc = float(b[channel_idx]);
}
- for (int tap = 0; tap < tap_count; ++tap) {
- float sample = float(state[state_offset + size_t(tap)]);
+ for (uint tap = 0; tap < tap_count; ++tap) {
+ float sample = float(state[state_offset + tap]);
acc += float(w_row[tap]) * sample;
}
@@ -164,46 +136,42 @@ void short_conv_decode_kernel(
float gated_output = acc * post_conv_gate;
- size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);
+ uint out_idx = token_idx * model_dim + channel_idx;
out[out_idx] = static_cast<T>(gated_output);
if (tap_count > 0) {
- for (int tap = 0; tap < tap_count - 1; ++tap) {
- next_state[state_offset + size_t(tap)] =
- state[state_offset + size_t(tap + 1)];
+ for (uint tap = 0; tap < tap_count - 1; ++tap) {
+ next_state[state_offset + tap] = state[state_offset + tap + 1];
}
- next_state[state_offset + size_t(tap_count - 1)] = static_cast<T>(x);
+ next_state[state_offset + tap_count - 1] = static_cast<T>(x);
}
}
template <typename T>
-[[kernel, max_total_threads_per_threadgroup(32)]]
-void short_conv_trie_kernel(
- device const T* in_proj [[buffer(0)]],
- device const T* w [[buffer(1)]],
- device const T* b [[buffer(2), function_constant(has_bias)]],
- device const T* base_state [[buffer(3)]],
- device const int* parents [[buffer(4)]],
- device T* out [[buffer(5)]],
- device T* suffix_state [[buffer(6)]],
- constant const size_t& suffix_len [[buffer(7)]],
- constant const int& kernel_size [[buffer(8)]],
- constant const size_t& in_proj_stride [[buffer(9)]],
- constant const size_t& state_stride [[buffer(10)]],
- constant const uint& model_dim [[buffer(11)]],
- uint channel_idx [[thread_position_in_grid]]
+VARIANTS(T, float, half, bfloat)
+KERNEL(ShortConvTrie)(
+ device const T* in_proj,
+ device const T* w,
+ device const T* b OPTIONAL(has_bias),
+ device const T* base_state,
+ device const int* parents,
+ device T* out,
+ device T* suffix_state,
+ constant const uint& suffix_len,
+ constant const uint& kernel_size,
+ constant const uint& in_proj_stride,
+ constant const uint& state_stride,
+ constant const uint& model_dim,
+ const bool has_bias SPECIALIZE,
+ const uint channel_idx AXIS(model_dim, 32)
) {
- if (channel_idx >= model_dim) {
- return;
- }
-
- const int tap_count = max(kernel_size - 1, 0);
- const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);
- const size_t base_state_offset = size_t(channel_idx) * state_stride;
+ const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;
+ const device T* w_row = w + channel_idx * kernel_size;
+ const uint base_state_offset = channel_idx * state_stride;
- for (size_t node = 0; node < suffix_len; ++node) {
+ for (uint node = 0; node < suffix_len; ++node) {
// Read gated input from in_proj
- const size_t in_proj_idx = node * in_proj_stride + size_t(channel_idx);
+ const uint in_proj_idx = node * in_proj_stride + channel_idx;
float pre_conv_gate = float(in_proj[in_proj_idx]);
float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);
float x_in = float(in_proj[in_proj_idx + 2 * model_dim]);
@@ -214,15 +182,14 @@ void short_conv_trie_kernel(
const device T* parent_state =
(parent < 0) ? (base_state + base_state_offset)
: (suffix_state +
- (size_t(parent) * model_dim + size_t(channel_idx)) *
- state_stride);
+ (parent * model_dim + channel_idx) * state_stride);
float acc = 0.0f;
if (has_bias) {
acc = float(b[channel_idx]);
}
- for (int tap = 0; tap < tap_count; ++tap) {
+ for (uint tap = 0; tap < tap_count; ++tap) {
float sample = float(parent_state[size_t(tap)]);
acc += float(w_row[tap]) * sample;
}
@@ -233,106 +200,16 @@ void short_conv_trie_kernel(
float gated_output = acc * post_conv_gate;
// Write output
- out[node * model_dim + size_t(channel_idx)] = static_cast<T>(gated_output);
+ out[node * model_dim + channel_idx] = static_cast<T>(gated_output);
// Write post-state for this node
if (tap_count > 0) {
device T* dst_state =
- suffix_state +
- (node * model_dim + size_t(channel_idx)) * state_stride;
- for (int tap = 0; tap < tap_count - 1; ++tap) {
- dst_state[size_t(tap)] = parent_state[size_t(tap + 1)];
+ suffix_state + (node * model_dim + channel_idx) * state_stride;
+ for (uint tap = 0; tap < tap_count - 1; ++tap) {
+ dst_state[tap] = parent_state[tap + 1];
}
- dst_state[size_t(tap_count - 1)] = static_cast<T>(x);
+ dst_state[tap_count - 1] = static_cast<T>(x);
}
}
-}
-
-// ============================================================================
-// Template instantiations
-// ============================================================================
-#define instantiate_short_conv_pack_kernel(type_name, type) \
- template [[host_name("short_conv_pack_kernel_" #type_name)]] \
- kernel void short_conv_pack_kernel<type>( \
- device const type* state_in [[buffer(0)]], \
- device const type* in_proj [[buffer(1)]], \
- device type* padded [[buffer(2)]], \
- constant const size_t& state_stride [[buffer(3)]], \
- constant const size_t& suffix_len [[buffer(4)]], \
- constant const size_t& in_proj_stride [[buffer(5)]], \
- constant const uint& model_dim [[buffer(6)]], \
- uint2 grid_idx [[thread_position_in_grid]] \
- );
-
-#define instantiate_short_conv_prefill_kernel(type_name, type) \
- template [[host_name("short_conv_prefill_kernel_" #type_name)]] \
- kernel void short_conv_prefill_kernel<type>( \
- device const type* padded [[buffer(0)]], \
- device const type* in_proj [[buffer(1)]], \
- device const type* w [[buffer(2)]], \
- device const type* b [[buffer(3), function_constant(has_bias)]], \
- device type* out [[buffer(4)]], \
- device type* state_out [[buffer(5)]], \
- constant const size_t& suffix_len [[buffer(6)]], \
- constant const int& kernel_size [[buffer(7)]], \
- constant const size_t& in_proj_stride [[buffer(8)]], \
- constant const size_t& state_stride [[buffer(9)]], \
- constant const uint& model_dim [[buffer(10)]], \
- uint2 grid_idx [[thread_position_in_grid]] \
- );
-
-#define instantiate_short_conv_decode_kernel(type_name, type) \
- template [[host_name("short_conv_decode_kernel_" #type_name)]] \
- kernel void short_conv_decode_kernel<type>( \
- device const type* x [[buffer(0)]], \
- device const type* w [[buffer(1)]], \
- device const type* b [[buffer(2), function_constant(has_bias)]], \
- device const type* state [[buffer(3)]], \
- device type* out [[buffer(4)]], \
- device type* next_state [[buffer(5)]], \
- constant const size_t& suffix_len [[buffer(6)]], \
- constant const int& kernel_size [[buffer(7)]], \
- constant const size_t& row_stride [[buffer(8)]], \
- constant const size_t& state_stride [[buffer(9)]], \
- constant const uint& num_channels [[buffer(10)]], \
- uint2 grid_idx [[thread_position_in_grid]] \
- );
-
-#define instantiate_short_conv_trie_kernel(type_name, type) \
- template [[host_name("short_conv_trie_kernel_" #type_name)]] \
- kernel void short_conv_trie_kernel<type>( \
- device const type* in_proj [[buffer(0)]], \
- device const type* w [[buffer(1)]], \
- device const type* b [[buffer(2), function_constant(has_bias)]], \
- device const type* base_state [[buffer(3)]], \
- device const int* parents [[buffer(4)]], \
- device type* out [[buffer(5)]], \
- device type* suffix_state [[buffer(6)]], \
- constant const size_t& suffix_len [[buffer(7)]], \
- constant const int& kernel_size [[buffer(8)]], \
- constant const size_t& in_proj_stride [[buffer(9)]], \
- constant const size_t& state_stride [[buffer(10)]], \
- constant const uint& model_dim [[buffer(11)]], \
- uint channel_idx [[thread_position_in_grid]] \
- );
-
-instantiate_short_conv_pack_kernel(float, float);
-instantiate_short_conv_pack_kernel(bfloat, bfloat);
-instantiate_short_conv_pack_kernel(half, half);
-
-instantiate_short_conv_prefill_kernel(float, float);
-instantiate_short_conv_prefill_kernel(bfloat, bfloat);
-instantiate_short_conv_prefill_kernel(half, half);
-
-instantiate_short_conv_decode_kernel(float, float);
-instantiate_short_conv_decode_kernel(bfloat, bfloat);
-instantiate_short_conv_decode_kernel(half, half);
-
-instantiate_short_conv_trie_kernel(float, float);
-instantiate_short_conv_trie_kernel(bfloat, bfloat);
-instantiate_short_conv_trie_kernel(half, half);
-
-#undef instantiate_short_conv_pack_kernel
-#undef instantiate_short_conv_prefill_kernel
-#undef instantiate_short_conv_decode_kernel
-#undef instantiate_short_conv_trie_kernel
+}
\ No newline at end of file