A high-performance inference engine for AI models
Radix top-k: aligned loads for unaligned rows (#1106)
CCYeh0f56efe20b5e
diff --git a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metalindex c6826adb0..a8c346791 100644--- a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal@@ -24,15 +24,15 @@ constant uint HALF_MASK = (1 << HALF_BITS) - 1;METAL_FUNC void load_row_keys(const device ushort* row_scores,- uint columns,+ uint leading_pad,+ uint padded_columns,uint first_column,thread uint (&keys)[SCORES_PER_THREAD]) {- const bool vector_aligned = (reinterpret_cast<ulong>(row_scores) & 15) == 0;METAL_PRAGMA_UNROLLfor (uint vector = 0; vector < VECTORS_PER_THREAD; ++vector) {const uint vector_column = first_column + vector * VECTOR_WIDTH;- if (vector_aligned && vector_column + VECTOR_WIDTH <= columns) {+ if ((vector_column > 0 || leading_pad == 0) && vector_column + VECTOR_WIDTH <= padded_columns) {const uint4 packed_scores = *reinterpret_cast<const device uint4*>(row_scores + vector_column);METAL_PRAGMA_UNROLLfor (uint lane = 0; lane < VECTOR_WIDTH; ++lane) {@@ -40,8 +40,9 @@ METAL_FUNC void load_row_keys(}} else {for (uint lane = 0; lane < VECTOR_WIDTH; ++lane) {- const uint column = vector_column + lane;- keys[vector * VECTOR_WIDTH + lane] = column < columns ? top_k_bf16_key(row_scores[column]) : 0;+ const uint position = vector_column + lane;+ keys[vector * VECTOR_WIDTH + lane] =+ position - leading_pad < padded_columns - leading_pad ? top_k_bf16_key(row_scores[position]) : 0;}}}@@ -83,7 +84,9 @@ KERNEL(RadixTopKSmall)(const uint lid THREADS(THREADS_PER_TG)) {const device bfloat* row_input = input + ulong(row) * columns;- const device ushort* row_scores = reinterpret_cast<const device ushort*>(row_input);+ const uint leading_pad = (reinterpret_cast<ulong>(row_input) / sizeof(bfloat)) % VECTOR_WIDTH;+ const device ushort* row_scores = reinterpret_cast<const device ushort*>(row_input) - leading_pad;+ const uint padded_columns = leading_pad + columns;device uint* row_ids = output_ids + row * k;device float* row_output_scores = output_scores + row * k;@@ -98,9 +101,9 @@ KERNEL(RadixTopKSmall)(for (uint pass = 0; pass < 2; ++pass) {const uint shift = RADIX_BITS * (1 - pass);threadgroup atomic_uint* histogram = histograms + pass * HISTOGRAM_SIZE;- for (uint first_column = lid * SCORES_PER_THREAD; first_column < columns; first_column += SCORES_PER_BLOCK) {+ for (uint first_column = lid * SCORES_PER_THREAD; first_column < padded_columns; first_column += SCORES_PER_BLOCK) {uint keys[SCORES_PER_THREAD];- load_row_keys(row_scores, columns, first_column, keys);+ load_row_keys(row_scores, leading_pad, padded_columns, first_column, keys);METAL_PRAGMA_UNROLLfor (uint index = 0; index < SCORES_PER_THREAD; ++index) {if ((keys[index] >> (shift + RADIX_BITS)) == threshold_key) {@@ -133,16 +136,16 @@ KERNEL(RadixTopKSmall)(const uint above_threshold_count = k - tie_winner_count;uint above_threshold_before = 0;uint ties_before = 0;- for (uint block = 0; block * SCORES_PER_BLOCK < columns; ++block) {+ for (uint block = 0; block * SCORES_PER_BLOCK < padded_columns; ++block) {const uint first_column = block * SCORES_PER_BLOCK + lid * SCORES_PER_THREAD;uint keys[SCORES_PER_THREAD];- load_row_keys(row_scores, columns, first_column, keys);+ load_row_keys(row_scores, leading_pad, padded_columns, first_column, keys);uint thread_above_threshold = 0;uint thread_ties = 0;METAL_PRAGMA_UNROLLfor (uint index = 0; index < SCORES_PER_THREAD; ++index) {thread_above_threshold += keys[index] > threshold_key;- thread_ties += keys[index] == threshold_key && first_column + index < columns;+ thread_ties += keys[index] == threshold_key && first_column + index - leading_pad < columns;}const uint2 prefix_and_total = threadgroup_prefix_sum(thread_above_threshold | thread_ties << HALF_BITS,@@ -153,7 +156,7 @@ KERNEL(RadixTopKSmall)(uint tie_rank = ties_before + (prefix_and_total.x >> HALF_BITS);METAL_PRAGMA_UNROLLfor (uint index = 0; index < SCORES_PER_THREAD; ++index) {- const uint column = first_column + index;+ const uint column = first_column + index - leading_pad;const bool above_threshold = keys[index] > threshold_key;const bool tie = keys[index] == threshold_key && column < columns;const uint slot = above_threshold_slot + min(tie_rank, tie_winner_count);