A high-performance inference engine for AI models
tune kernel dispatch threadgroup sizes (#304)
uuuvn86b635cfe8f0
diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metalindex dca7edd39..0bee9eb34 100644--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal@@ -36,7 +36,7 @@ PUBLIC KERNEL(Rope)(constant uint& max_sequence_length,const uint head_index AXIS(num_heads, 1),const uint token_index AXIS(suffix_length, 1),- const uint dimension_index AXIS(head_dim, 32)+ const uint dimension_index AXIS(head_dim, 128)) {if (head_index >= num_heads || token_index >= suffix_length ||dimension_index >= head_dim)diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metalindex 7b5d757a6..5f375544b 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -11,7 +11,7 @@ PUBLIC KERNEL(ShortConvPack)(constant const uint& suffix_len,constant const uint& in_proj_stride,constant const uint& model_dim,- const uint channel_idx AXIS(model_dim, 32),+ const uint channel_idx AXIS(model_dim, 256),const uint row_idx AXIS(state_stride + suffix_len, 1)) {const uint padded_offset = row_idx * model_dim + channel_idx;@@ -47,8 +47,8 @@ PUBLIC KERNEL(ShortConvPrefill)(constant const uint& state_stride,constant const uint& model_dim,const bool has_bias SPECIALIZE,- const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 32),- const uint channel_idx AXIS(model_dim, 1)+ const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 1),+ const uint channel_idx AXIS(model_dim, 256)) {const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;const device T* w_row = w + channel_idx * kernel_size;@@ -109,8 +109,8 @@ PUBLIC KERNEL(ShortConvDecode)(constant const uint& model_dim,const bool has_bias SPECIALIZE,const bool state_in_place SPECIALIZE,- const uint token_idx AXIS(suffix_len, 32),- const uint channel_idx AXIS(model_dim, 1)+ const uint token_idx AXIS(suffix_len, 1),+ const uint channel_idx AXIS(model_dim, 256)) {if (state_in_place) {state = next_state;@@ -168,7 +168,7 @@ PUBLIC KERNEL(ShortConvTrie)(constant const uint& state_stride,constant const uint& model_dim,const bool has_bias SPECIALIZE,- const uint channel_idx AXIS(model_dim, 32)+ const uint channel_idx AXIS(model_dim, 256)) {const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;const device T* w_row = w + channel_idx * kernel_size;