A high-performance inference engine for AI models
Small perf improvements to sampling kernels (#46)
uuuvnb4752bf6c17a
diff --git a/.gitignore b/.gitignoreindex bd22a0711..1052a077a 100644--- a/.gitignore+++ b/.gitignore@@ -21,5 +21,8 @@ target/#.idea/*.DS_Store+# Metal traces+*.gputrace+# Models-models/\ No newline at end of file+models/diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metalindex 6cb34bacd..cad698734 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal@@ -29,10 +29,11 @@ void batched_gumbel(PhiloxState rng;philox_init(&rng, rng_seed, rng_offset);+ #pragma unroll(4)for (uint i = 0; i < GRAIN_SIZE; i++) {uint global_idx = grain_offset + i * BLOCK_SIZE;if (global_idx < batch_end) {- processed_logits[global_idx] = logits_data[global_idx] + T(-log(-log(uniform_float(&rng))));+ processed_logits[global_idx] = logits_data[global_idx] + T(-fast::log(-fast::log(uniform_float(&rng))));}}}diff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metalindex e451397dd..51fb94f9e 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal@@ -17,6 +17,7 @@ void batched_temperature(uint base_idx = batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;uint batch_end = batch_idx * vocab_size + vocab_size;+ #pragma unroll(4)for (uint i = 0; i < GRAIN_SIZE; i++) {uint global_idx = base_idx + i * BLOCK_SIZE;if (global_idx < batch_end) {diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metalindex a88d9af88..0e1a4c55c 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal@@ -2,7 +2,7 @@#include "../definitions.metal"#define BLOCK_SIZE 1024-#define MAX_ITERS 20+#define MAX_ITERS 16template <typename T>void batched_topk(@@ -19,6 +19,7 @@ void batched_topk(// Find min and max logit for binary searchfloat local_max = -INFINITY;float local_min = INFINITY;+ #pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {float logit_value = float(logits_data[batch_start + i]);local_max = fmax(local_max, logit_value);@@ -36,11 +37,10 @@ void batched_topk(for (uint iter = 0; iter < MAX_ITERS; iter++) {// Find the mass above thresholduint local_num_above_threshold = 0;+ #pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {float logit_value = float(logits_data[batch_start + i]);- if (logit_value >= threshold) {- local_num_above_threshold += 1;- }+ local_num_above_threshold += select(0, 1, logit_value >= threshold);}uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_num_above_threshold, (threadgroup uint*)shared_reduce_buffer, thread_idx);@@ -58,9 +58,10 @@ void batched_topk(T t_threshold = T(threshold);// We know the threshold, just mask everything below it+ #pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {T logit_value = logits_data[batch_start + i];- processed_logits[batch_start + i] = (logit_value >= t_threshold) ? logit_value : T(-INFINITY);+ processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);}}diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metalindex e255ee79c..a188782c4 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal@@ -2,14 +2,15 @@#include "../definitions.metal"#define BLOCK_SIZE 1024-#define MAX_ITERS 20-#define ATOL 1e-6+#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)++#define MAX_ITERS 16template <typename T>void batched_topp(device const T* logits_data,device T* processed_logits,- threadgroup float shared_reduce_buffer[BLOCK_SIZE],+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],constant uint& vocab_size,constant float& top_p,uint batch_idx,@@ -20,6 +21,7 @@ void batched_topp(// Find min (for binary search) and max (for binary search and softmax)float local_max = -INFINITY;float local_min = INFINITY;+ #pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {float logit_value = float(logits_data[batch_start + i]);local_max = fmax(local_max, logit_value);@@ -30,9 +32,10 @@ void batched_topp(// Find denominator for softmaxfloat local_sum = 0.0f;+ #pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {float logit_value = float(logits_data[batch_start + i]);- local_sum += exp(logit_value - max_logit);+ local_sum += fast::exp(logit_value - max_logit);}float total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_sum, shared_reduce_buffer, thread_idx);@@ -47,14 +50,21 @@ void batched_topp(for (uint iter = 0; iter < MAX_ITERS; iter++) {// Find the mass above thresholdfloat local_sum_above_threshold = 0.0f;+ float local_min_above_threshold = INFINITY;+ #pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {float logit_value = float(logits_data[batch_start + i]);- if (logit_value >= threshold) {- local_sum_above_threshold += exp(logit_value - max_logit);- }+ float logit_mass = fast::exp(logit_value - max_logit);+ local_sum_above_threshold += select(0.0, logit_mass, logit_value >= threshold);+ local_min_above_threshold = fmin(local_min_above_threshold, select(INFINITY, logit_mass, logit_value >= threshold));}float sum_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_sum_above_threshold, shared_reduce_buffer, thread_idx);+ float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min_above_threshold, shared_reduce_buffer, thread_idx);+ // Early exit+ if (sum_above_threshold >= target_mass && sum_above_threshold - min_above_threshold < target_mass) {+ break;+ }// Update binary searchif (sum_above_threshold >= target_mass) {low = threshold;@@ -62,17 +72,15 @@ void batched_topp(high = threshold;}threshold = (low + high) / 2.0;-- // Exit early if we found it- if (high - low < ATOL) break;}- T t_threshold = T(low);+ T t_threshold = T(threshold);// We know the threshold, just mask everything below it+ #pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {T logit_value = logits_data[batch_start + i];- processed_logits[batch_start + i] = (logit_value >= t_threshold) ? logit_value : T(-INFINITY);+ processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);}}@@ -88,7 +96,7 @@ uint3 thread_idx [[ thread_position_in_threadgroup ]])#define generateToppKernel(functionName, scalarType, outerArgs, innerArgs) \kernel void functionName##_##scalarType outerArgs { \- threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \functionName innerArgs; \}