A high-performance inference engine for AI models
metal validation fixes (#48)
uuuvn4e15264d7440
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metalindex 47fffab20..1b2aac913 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention.metal@@ -510,7 +510,7 @@ void attention_2pass_2(// Generate single-pass kernels for different head dimensions#define GENERATE_SINGLE_PASS_KERNELS(head_dim_value) \-kernel void attention_single_pass_half_##head_dim_value outerArguments(half) { \+[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_half_##head_dim_value outerArguments(half) { \constexpr int sequence_block_size = 32; \constexpr int head_block_size = 32; \threadgroup float shared_max_scores[sequence_block_size]; \@@ -518,7 +518,7 @@ kernel void attention_single_pass_half_##head_dim_value outerArguments(half) { \threadgroup float shared_outputs[sequence_block_size * head_block_size]; \attention_single_pass<half, head_dim_value> innerArguments; \} \-kernel void attention_single_pass_bfloat_##head_dim_value outerArguments(bfloat) { \+[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_bfloat_##head_dim_value outerArguments(bfloat) { \constexpr int sequence_block_size = 32; \constexpr int head_block_size = 32; \threadgroup float shared_max_scores[sequence_block_size]; \@@ -526,7 +526,7 @@ kernel void attention_single_pass_bfloat_##head_dim_value outerArguments(bfloat)threadgroup float shared_outputs[sequence_block_size * head_block_size]; \attention_single_pass<bfloat, head_dim_value> innerArguments; \} \-kernel void attention_single_pass_float_##head_dim_value outerArguments(float) { \+[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_float_##head_dim_value outerArguments(float) { \constexpr int sequence_block_size = 32; \constexpr int head_block_size = 32; \threadgroup float shared_max_scores[sequence_block_size]; \@@ -575,7 +575,7 @@ GENERATE_SINGLE_PASS_KERNELS(256)// Generate 2-pass Pass 1 kernels for different head dimensions#define GENERATE_2PASS_1_KERNELS(head_dim_value) \-kernel void attention_2pass_1_half_##head_dim_value outerArguments(half) { \+[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_half_##head_dim_value outerArguments(half) { \constexpr int sequence_block_size = 8; \constexpr int head_block_size = 32; \threadgroup float shared_max_scores[sequence_block_size]; \@@ -583,7 +583,7 @@ kernel void attention_2pass_1_half_##head_dim_value outerArguments(half) { \threadgroup float shared_outputs[sequence_block_size * head_block_size]; \attention_2pass_1<half, head_dim_value> innerArguments; \} \-kernel void attention_2pass_1_bfloat_##head_dim_value outerArguments(bfloat) { \+[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_bfloat_##head_dim_value outerArguments(bfloat) { \constexpr int sequence_block_size = 8; \constexpr int head_block_size = 32; \threadgroup float shared_max_scores[sequence_block_size]; \@@ -591,7 +591,7 @@ kernel void attention_2pass_1_bfloat_##head_dim_value outerArguments(bfloat) { \threadgroup float shared_outputs[sequence_block_size * head_block_size]; \attention_2pass_1<bfloat, head_dim_value> innerArguments; \} \-kernel void attention_2pass_1_float_##head_dim_value outerArguments(float) { \+[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_float_##head_dim_value outerArguments(float) { \constexpr int sequence_block_size = 8; \constexpr int head_block_size = 32; \threadgroup float shared_max_scores[sequence_block_size]; \@@ -622,13 +622,13 @@ GENERATE_2PASS_1_KERNELS(256)// Generate 2-pass Pass 2 kernels for different head dimensions#define GENERATE_2PASS_2_KERNELS(head_dim_value) \-kernel void attention_2pass_2_half_##head_dim_value outerArguments(half) { \+[[max_total_threads_per_threadgroup(1024)]] kernel void attention_2pass_2_half_##head_dim_value outerArguments(half) { \constexpr int sequence_block_size = 32; \constexpr int head_block_size = 32; \threadgroup float shared_outputs[sequence_block_size * head_block_size]; \attention_2pass_2<half, head_dim_value> innerArguments; \} \-kernel void attention_2pass_2_float_##head_dim_value outerArguments(float) { \+[[max_total_threads_per_threadgroup(1024)]] kernel void attention_2pass_2_float_##head_dim_value outerArguments(float) { \constexpr int sequence_block_size = 32; \constexpr int head_block_size = 32; \threadgroup float shared_outputs[sequence_block_size * head_block_size]; \diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 84dcb9de7..30e0b7573 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -207,7 +207,7 @@ void rms_norm_core(// Generate RMS norm kernels#define DEFINE_RMS_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \-kernel void rms_norm_##SUF( \+[[max_total_threads_per_threadgroup(1024)]] kernel void rms_norm_##SUF( \const device IN* input [[buffer(0)]], \const device SC* scales [[buffer(1)]], \device OUT* output [[buffer(2)]], \@@ -241,7 +241,7 @@ FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)// Generate QK norm kernels#define DEFINE_QK_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \-kernel void qk_norm_##SUF( \+[[max_total_threads_per_threadgroup(1024)]] kernel void qk_norm_##SUF( \const device IN* qkv_input [[buffer(0)]], \const device SC* scales [[buffer(1)]], \device OUT* qkv_output [[buffer(2)]], \@@ -295,4 +295,4 @@ kernel void qk_norm_##SUF( \}FOREACH_RMS_COMBO(DEFINE_QK_KERNEL)-#undef DEFINE_QK_KERNEL\ No newline at end of file+#undef DEFINE_QK_KERNELdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metalindex 8a8eb09ae..c3966c0c3 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal@@ -205,7 +205,7 @@ uint2 local_position [[ thread_position_in_threadgroup ]])(logits_data, partial_results, batch_size, vocab_size, shared, group_position.x, group_position.y, (ushort)local_position.x)#define generateArgmaxMainKernel(functionName, T, outerArgs, innerArgs) \-kernel void functionName##_##T outerArgs { \+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \threadgroup ArgmaxPair shared[BLOCK_SIZE]; \batched_argmax_main_impl<T> innerArgs; \}@@ -231,7 +231,7 @@ uint local_id [[ thread_position_in_threadgroup ]])(partial_results, final_tokens, batch_size, vocab_size, shared, batch_idx, (ushort)local_id)#define generateArgmaxFinalKernel(functionName, T, outerArgs, innerArgs) \-kernel void functionName##_##T outerArgs { \+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \threadgroup ArgmaxPair shared[BLOCK_SIZE]; \batched_argmax_final_impl innerArgs; \}@@ -290,7 +290,7 @@ uint local_id [[ thread_position_in_threadgroup ]])(logits_data, final_tokens, batch_size, vocab_size, shared, batch_idx, (ushort)local_id)#define generateArgmaxSingleKernel(functionName, T, outerArgs, innerArgs) \-kernel void functionName##_##T outerArgs { \+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \threadgroup ArgmaxPair shared[BLOCK_SIZE]; \batched_argmax_single_impl<T> innerArgs; \}diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metalindex cad698734..47b804e33 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal@@ -48,7 +48,19 @@ uint3 thread_idx [[ thread_position_in_threadgroup ]])#define innerArguments (logits_data, batch_seeds, processed_logits, vocab_size, threadgroup_idx.x, threadgroup_idx.y, thread_idx.x)-generateKernels(batched_gumbel)+#define generateGumbelKernel(functionName, scalarType, outerArgs, innerArgs) \+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \+ functionName innerArgs; \+}++#define generateGumbelKernels(functionName) \+generateGumbelKernel(functionName, float, outerArguments(float), innerArguments); \+generateGumbelKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \+generateGumbelKernel(functionName, half, outerArguments(half), innerArguments);++generateGumbelKernels(batched_gumbel)#undef outerArguments#undef innerArguments+#undef generateGumbelKernel+#undef generateGumbelKernelsdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metalindex 2a989e6fe..50b2e83e7 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal@@ -36,7 +36,19 @@ uint3 thread_idx [[ thread_position_in_threadgroup ]])#define innerArguments (logits_data, processed_logits, vocab_size, temperature, threadgroup_idx.x, threadgroup_idx.y, thread_idx.x)-generateKernels(batched_temperature)+#define generateTemperatureKernel(functionName, scalarType, outerArgs, innerArgs) \+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \+ functionName innerArgs; \+}++#define generateTemperatureKernels(functionName) \+generateTemperatureKernel(functionName, float, outerArguments(float), innerArguments); \+generateTemperatureKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \+generateTemperatureKernel(functionName, half, outerArguments(half), innerArguments);++generateTemperatureKernels(batched_temperature)#undef outerArguments#undef innerArguments+#undef generateTemperatureKernel+#undef generateTemperatureKernelsdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metalindex 0e1a4c55c..06dd82d52 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal@@ -76,13 +76,13 @@ uint3 thread_idx [[ thread_position_in_threadgroup ]])#define innerArguments (logits_data, processed_logits, shared_reduce_buffer, vocab_size, top_k, threadgroup_idx.x, thread_idx.x)#define generateTopkKernel(functionName, scalarType, outerArgs, innerArgs) \-kernel void functionName##_##scalarType outerArgs { \- threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \- functionName innerArgs; \+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \+ threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \+ functionName innerArgs; \}-#define generateTopkKernels(functionName) \-generateTopkKernel(functionName, float, outerArguments(float), innerArguments); \+#define generateTopkKernels(functionName) \+generateTopkKernel(functionName, float, outerArguments(float), innerArguments); \generateTopkKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \generateTopkKernel(functionName, half, outerArguments(half), innerArguments);diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metalindex a188782c4..10b00eb2c 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal@@ -95,13 +95,13 @@ uint3 thread_idx [[ thread_position_in_threadgroup ]])#define innerArguments (logits_data, processed_logits, shared_reduce_buffer, vocab_size, top_p, threadgroup_idx.x, thread_idx.x)#define generateToppKernel(functionName, scalarType, outerArgs, innerArgs) \-kernel void functionName##_##scalarType outerArgs { \- threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \- functionName innerArgs; \+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \+ functionName innerArgs; \}-#define generateToppKernels(functionName) \-generateToppKernel(functionName, float, outerArguments(float), innerArguments); \+#define generateToppKernels(functionName) \+generateToppKernel(functionName, float, outerArguments(float), innerArguments); \generateToppKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \generateToppKernel(functionName, half, outerArguments(half), innerArguments);