Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Small perf improvements to sampling kernels (#46)

uuuvnb4752bf6c17a
diff --git a/.gitignore b/.gitignore
index bd22a0711..1052a077a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,5 +21,8 @@ target/
#.idea/
*.DS_Store
+# Metal traces
+*.gputrace
+
# Models
-models/
\ No newline at end of file
+models/
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
index 6cb34bacd..cad698734 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
@@ -29,10 +29,11 @@ void batched_gumbel(
PhiloxState rng;
philox_init(&rng, rng_seed, rng_offset);
+ #pragma unroll(4)
for (uint i = 0; i < GRAIN_SIZE; i++) {
uint global_idx = grain_offset + i * BLOCK_SIZE;
if (global_idx < batch_end) {
- processed_logits[global_idx] = logits_data[global_idx] + T(-log(-log(uniform_float(&rng))));
+ processed_logits[global_idx] = logits_data[global_idx] + T(-fast::log(-fast::log(uniform_float(&rng))));
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
index e451397dd..51fb94f9e 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
@@ -17,6 +17,7 @@ void batched_temperature(
uint base_idx = batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
uint batch_end = batch_idx * vocab_size + vocab_size;
+ #pragma unroll(4)
for (uint i = 0; i < GRAIN_SIZE; i++) {
uint global_idx = base_idx + i * BLOCK_SIZE;
if (global_idx < batch_end) {
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index a88d9af88..0e1a4c55c 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -2,7 +2,7 @@
#include "../definitions.metal"
#define BLOCK_SIZE 1024
-#define MAX_ITERS 20
+#define MAX_ITERS 16
template <typename T>
void batched_topk(
@@ -19,6 +19,7 @@ void batched_topk(
// Find min and max logit for binary search
float local_max = -INFINITY;
float local_min = INFINITY;
+ #pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
float logit_value = float(logits_data[batch_start + i]);
local_max = fmax(local_max, logit_value);
@@ -36,11 +37,10 @@ void batched_topk(
for (uint iter = 0; iter < MAX_ITERS; iter++) {
// Find the mass above threshold
uint local_num_above_threshold = 0;
+ #pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
float logit_value = float(logits_data[batch_start + i]);
- if (logit_value >= threshold) {
- local_num_above_threshold += 1;
- }
+ local_num_above_threshold += select(0, 1, logit_value >= threshold);
}
uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_num_above_threshold, (threadgroup uint*)shared_reduce_buffer, thread_idx);
@@ -58,9 +58,10 @@ void batched_topk(
T t_threshold = T(threshold);
// We know the threshold, just mask everything below it
+ #pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
T logit_value = logits_data[batch_start + i];
- processed_logits[batch_start + i] = (logit_value >= t_threshold) ? logit_value : T(-INFINITY);
+ processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
index e255ee79c..a188782c4 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
@@ -2,14 +2,15 @@
#include "../definitions.metal"
#define BLOCK_SIZE 1024
-#define MAX_ITERS 20
-#define ATOL 1e-6
+#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)
+
+#define MAX_ITERS 16
template <typename T>
void batched_topp(
device const T* logits_data,
device T* processed_logits,
- threadgroup float shared_reduce_buffer[BLOCK_SIZE],
+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],
constant uint& vocab_size,
constant float& top_p,
uint batch_idx,
@@ -20,6 +21,7 @@ void batched_topp(
// Find min (for binary search) and max (for binary search and softmax)
float local_max = -INFINITY;
float local_min = INFINITY;
+ #pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
float logit_value = float(logits_data[batch_start + i]);
local_max = fmax(local_max, logit_value);
@@ -30,9 +32,10 @@ void batched_topp(
// Find denominator for softmax
float local_sum = 0.0f;
+ #pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
float logit_value = float(logits_data[batch_start + i]);
- local_sum += exp(logit_value - max_logit);
+ local_sum += fast::exp(logit_value - max_logit);
}
float total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_sum, shared_reduce_buffer, thread_idx);
@@ -47,14 +50,21 @@ void batched_topp(
for (uint iter = 0; iter < MAX_ITERS; iter++) {
// Find the mass above threshold
float local_sum_above_threshold = 0.0f;
+ float local_min_above_threshold = INFINITY;
+ #pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
float logit_value = float(logits_data[batch_start + i]);
- if (logit_value >= threshold) {
- local_sum_above_threshold += exp(logit_value - max_logit);
- }
+ float logit_mass = fast::exp(logit_value - max_logit);
+ local_sum_above_threshold += select(0.0, logit_mass, logit_value >= threshold);
+ local_min_above_threshold = fmin(local_min_above_threshold, select(INFINITY, logit_mass, logit_value >= threshold));
}
float sum_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_sum_above_threshold, shared_reduce_buffer, thread_idx);
+ float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min_above_threshold, shared_reduce_buffer, thread_idx);
+ // Early exit
+ if (sum_above_threshold >= target_mass && sum_above_threshold - min_above_threshold < target_mass) {
+ break;
+ }
// Update binary search
if (sum_above_threshold >= target_mass) {
low = threshold;
@@ -62,17 +72,15 @@ void batched_topp(
high = threshold;
}
threshold = (low + high) / 2.0;
-
- // Exit early if we found it
- if (high - low < ATOL) break;
}
- T t_threshold = T(low);
+ T t_threshold = T(threshold);
// We know the threshold, just mask everything below it
+ #pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
T logit_value = logits_data[batch_start + i];
- processed_logits[batch_start + i] = (logit_value >= t_threshold) ? logit_value : T(-INFINITY);
+ processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);
}
}
@@ -88,7 +96,7 @@ uint3 thread_idx [[ thread_position_in_threadgroup ]])
#define generateToppKernel(functionName, scalarType, outerArgs, innerArgs) \
kernel void functionName##_##scalarType outerArgs { \
- threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \
+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \
functionName innerArgs; \
}