Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

fix topK for m3+ machines

LuckyIYI7cadd610b725
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index 51d7ef616..bda8c1e8d 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -2,13 +2,14 @@
#include "../definitions.metal"
#define BLOCK_SIZE 1024
+#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)
#define MAX_ITERS 16
template <typename T>
void batched_topk(
device const T* logits_data,
device T* processed_logits,
- threadgroup float shared_reduce_buffer[BLOCK_SIZE],
+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],
constant uint& vocab_size,
constant uint& top_k,
uint batch_idx,
@@ -29,13 +30,13 @@ void batched_topk(
logit_value > -INFINITY
);
}
- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
- local_max,
+ float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
+ local_min,
shared_reduce_buffer,
thread_idx
);
- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
- local_min,
+ float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
+ local_max,
shared_reduce_buffer,
thread_idx
);
@@ -102,7 +103,7 @@ void batched_topk(
[[max_total_threads_per_threadgroup( \
1024 \
)]] kernel void functionName##_##scalarType outerArgs { \
- threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \
+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \
functionName innerArgs; \
}