Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

tune kernel dispatch threadgroup sizes (#304)

uuuvn86b635cfe8f0
diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
index dca7edd39..0bee9eb34 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal
+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
@@ -36,7 +36,7 @@ PUBLIC KERNEL(Rope)(
constant uint& max_sequence_length,
const uint head_index AXIS(num_heads, 1),
const uint token_index AXIS(suffix_length, 1),
- const uint dimension_index AXIS(head_dim, 32)
+ const uint dimension_index AXIS(head_dim, 128)
) {
if (head_index >= num_heads || token_index >= suffix_length ||
dimension_index >= head_dim)
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
index 7b5d757a6..5f375544b 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -11,7 +11,7 @@ PUBLIC KERNEL(ShortConvPack)(
constant const uint& suffix_len,
constant const uint& in_proj_stride,
constant const uint& model_dim,
- const uint channel_idx AXIS(model_dim, 32),
+ const uint channel_idx AXIS(model_dim, 256),
const uint row_idx AXIS(state_stride + suffix_len, 1)
) {
const uint padded_offset = row_idx * model_dim + channel_idx;
@@ -47,8 +47,8 @@ PUBLIC KERNEL(ShortConvPrefill)(
constant const uint& state_stride,
constant const uint& model_dim,
const bool has_bias SPECIALIZE,
- const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 32),
- const uint channel_idx AXIS(model_dim, 1)
+ const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 1),
+ const uint channel_idx AXIS(model_dim, 256)
) {
const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;
const device T* w_row = w + channel_idx * kernel_size;
@@ -109,8 +109,8 @@ PUBLIC KERNEL(ShortConvDecode)(
constant const uint& model_dim,
const bool has_bias SPECIALIZE,
const bool state_in_place SPECIALIZE,
- const uint token_idx AXIS(suffix_len, 32),
- const uint channel_idx AXIS(model_dim, 1)
+ const uint token_idx AXIS(suffix_len, 1),
+ const uint channel_idx AXIS(model_dim, 256)
) {
if (state_in_place) {
state = next_state;
@@ -168,7 +168,7 @@ PUBLIC KERNEL(ShortConvTrie)(
constant const uint& state_stride,
constant const uint& model_dim,
const bool has_bias SPECIALIZE,
- const uint channel_idx AXIS(model_dim, 32)
+ const uint channel_idx AXIS(model_dim, 256)
) {
const uint tap_count = kernel_size > 0 ? kernel_size - 1 : 0u;
const device T* w_row = w + channel_idx * kernel_size;