Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

metal validation fixes (#48)

uuuvn4e15264d7440
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
index 47fffab20..1b2aac913 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
@@ -510,7 +510,7 @@ void attention_2pass_2(
// Generate single-pass kernels for different head dimensions
#define GENERATE_SINGLE_PASS_KERNELS(head_dim_value) \
-kernel void attention_single_pass_half_##head_dim_value outerArguments(half) { \
+[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_half_##head_dim_value outerArguments(half) { \
constexpr int sequence_block_size = 32; \
constexpr int head_block_size = 32; \
threadgroup float shared_max_scores[sequence_block_size]; \
@@ -518,7 +518,7 @@ kernel void attention_single_pass_half_##head_dim_value outerArguments(half) { \
threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
attention_single_pass<half, head_dim_value> innerArguments; \
} \
-kernel void attention_single_pass_bfloat_##head_dim_value outerArguments(bfloat) { \
+[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_bfloat_##head_dim_value outerArguments(bfloat) { \
constexpr int sequence_block_size = 32; \
constexpr int head_block_size = 32; \
threadgroup float shared_max_scores[sequence_block_size]; \
@@ -526,7 +526,7 @@ kernel void attention_single_pass_bfloat_##head_dim_value outerArguments(bfloat)
threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
attention_single_pass<bfloat, head_dim_value> innerArguments; \
} \
-kernel void attention_single_pass_float_##head_dim_value outerArguments(float) { \
+[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_float_##head_dim_value outerArguments(float) { \
constexpr int sequence_block_size = 32; \
constexpr int head_block_size = 32; \
threadgroup float shared_max_scores[sequence_block_size]; \
@@ -575,7 +575,7 @@ GENERATE_SINGLE_PASS_KERNELS(256)
// Generate 2-pass Pass 1 kernels for different head dimensions
#define GENERATE_2PASS_1_KERNELS(head_dim_value) \
-kernel void attention_2pass_1_half_##head_dim_value outerArguments(half) { \
+[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_half_##head_dim_value outerArguments(half) { \
constexpr int sequence_block_size = 8; \
constexpr int head_block_size = 32; \
threadgroup float shared_max_scores[sequence_block_size]; \
@@ -583,7 +583,7 @@ kernel void attention_2pass_1_half_##head_dim_value outerArguments(half) { \
threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
attention_2pass_1<half, head_dim_value> innerArguments; \
} \
-kernel void attention_2pass_1_bfloat_##head_dim_value outerArguments(bfloat) { \
+[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_bfloat_##head_dim_value outerArguments(bfloat) { \
constexpr int sequence_block_size = 8; \
constexpr int head_block_size = 32; \
threadgroup float shared_max_scores[sequence_block_size]; \
@@ -591,7 +591,7 @@ kernel void attention_2pass_1_bfloat_##head_dim_value outerArguments(bfloat) { \
threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
attention_2pass_1<bfloat, head_dim_value> innerArguments; \
} \
-kernel void attention_2pass_1_float_##head_dim_value outerArguments(float) { \
+[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_float_##head_dim_value outerArguments(float) { \
constexpr int sequence_block_size = 8; \
constexpr int head_block_size = 32; \
threadgroup float shared_max_scores[sequence_block_size]; \
@@ -622,13 +622,13 @@ GENERATE_2PASS_1_KERNELS(256)
// Generate 2-pass Pass 2 kernels for different head dimensions
#define GENERATE_2PASS_2_KERNELS(head_dim_value) \
-kernel void attention_2pass_2_half_##head_dim_value outerArguments(half) { \
+[[max_total_threads_per_threadgroup(1024)]] kernel void attention_2pass_2_half_##head_dim_value outerArguments(half) { \
constexpr int sequence_block_size = 32; \
constexpr int head_block_size = 32; \
threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
attention_2pass_2<half, head_dim_value> innerArguments; \
} \
-kernel void attention_2pass_2_float_##head_dim_value outerArguments(float) { \
+[[max_total_threads_per_threadgroup(1024)]] kernel void attention_2pass_2_float_##head_dim_value outerArguments(float) { \
constexpr int sequence_block_size = 32; \
constexpr int head_block_size = 32; \
threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 84dcb9de7..30e0b7573 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -207,7 +207,7 @@ void rms_norm_core(
// Generate RMS norm kernels
#define DEFINE_RMS_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
-kernel void rms_norm_##SUF( \
+[[max_total_threads_per_threadgroup(1024)]] kernel void rms_norm_##SUF( \
const device IN* input [[buffer(0)]], \
const device SC* scales [[buffer(1)]], \
device OUT* output [[buffer(2)]], \
@@ -241,7 +241,7 @@ FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)
// Generate QK norm kernels
#define DEFINE_QK_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
-kernel void qk_norm_##SUF( \
+[[max_total_threads_per_threadgroup(1024)]] kernel void qk_norm_##SUF( \
const device IN* qkv_input [[buffer(0)]], \
const device SC* scales [[buffer(1)]], \
device OUT* qkv_output [[buffer(2)]], \
@@ -295,4 +295,4 @@ kernel void qk_norm_##SUF( \
}
FOREACH_RMS_COMBO(DEFINE_QK_KERNEL)
-#undef DEFINE_QK_KERNEL
\ No newline at end of file
+#undef DEFINE_QK_KERNEL
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
index 8a8eb09ae..c3966c0c3 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
@@ -205,7 +205,7 @@ uint2 local_position [[ thread_position_in_threadgroup ]])
(logits_data, partial_results, batch_size, vocab_size, shared, group_position.x, group_position.y, (ushort)local_position.x)
#define generateArgmaxMainKernel(functionName, T, outerArgs, innerArgs) \
-kernel void functionName##_##T outerArgs { \
+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \
threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
batched_argmax_main_impl<T> innerArgs; \
}
@@ -231,7 +231,7 @@ uint local_id [[ thread_position_in_threadgroup ]])
(partial_results, final_tokens, batch_size, vocab_size, shared, batch_idx, (ushort)local_id)
#define generateArgmaxFinalKernel(functionName, T, outerArgs, innerArgs) \
-kernel void functionName##_##T outerArgs { \
+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \
threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
batched_argmax_final_impl innerArgs; \
}
@@ -290,7 +290,7 @@ uint local_id [[ thread_position_in_threadgroup ]])
(logits_data, final_tokens, batch_size, vocab_size, shared, batch_idx, (ushort)local_id)
#define generateArgmaxSingleKernel(functionName, T, outerArgs, innerArgs) \
-kernel void functionName##_##T outerArgs { \
+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \
threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
batched_argmax_single_impl<T> innerArgs; \
}
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
index cad698734..47b804e33 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
@@ -48,7 +48,19 @@ uint3 thread_idx [[ thread_position_in_threadgroup ]])
#define innerArguments (logits_data, batch_seeds, processed_logits, vocab_size, threadgroup_idx.x, threadgroup_idx.y, thread_idx.x)
-generateKernels(batched_gumbel)
+#define generateGumbelKernel(functionName, scalarType, outerArgs, innerArgs) \
+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \
+ functionName innerArgs; \
+}
+
+#define generateGumbelKernels(functionName) \
+generateGumbelKernel(functionName, float, outerArguments(float), innerArguments); \
+generateGumbelKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \
+generateGumbelKernel(functionName, half, outerArguments(half), innerArguments);
+
+generateGumbelKernels(batched_gumbel)
#undef outerArguments
#undef innerArguments
+#undef generateGumbelKernel
+#undef generateGumbelKernels
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
index 2a989e6fe..50b2e83e7 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
@@ -36,7 +36,19 @@ uint3 thread_idx [[ thread_position_in_threadgroup ]])
#define innerArguments (logits_data, processed_logits, vocab_size, temperature, threadgroup_idx.x, threadgroup_idx.y, thread_idx.x)
-generateKernels(batched_temperature)
+#define generateTemperatureKernel(functionName, scalarType, outerArgs, innerArgs) \
+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \
+ functionName innerArgs; \
+}
+
+#define generateTemperatureKernels(functionName) \
+generateTemperatureKernel(functionName, float, outerArguments(float), innerArguments); \
+generateTemperatureKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \
+generateTemperatureKernel(functionName, half, outerArguments(half), innerArguments);
+
+generateTemperatureKernels(batched_temperature)
#undef outerArguments
#undef innerArguments
+#undef generateTemperatureKernel
+#undef generateTemperatureKernels
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index 0e1a4c55c..06dd82d52 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -76,13 +76,13 @@ uint3 thread_idx [[ thread_position_in_threadgroup ]])
#define innerArguments (logits_data, processed_logits, shared_reduce_buffer, vocab_size, top_k, threadgroup_idx.x, thread_idx.x)
#define generateTopkKernel(functionName, scalarType, outerArgs, innerArgs) \
-kernel void functionName##_##scalarType outerArgs { \
- threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \
- functionName innerArgs; \
+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \
+ threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \
+ functionName innerArgs; \
}
-#define generateTopkKernels(functionName) \
-generateTopkKernel(functionName, float, outerArguments(float), innerArguments); \
+#define generateTopkKernels(functionName) \
+generateTopkKernel(functionName, float, outerArguments(float), innerArguments); \
generateTopkKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \
generateTopkKernel(functionName, half, outerArguments(half), innerArguments);
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
index a188782c4..10b00eb2c 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
@@ -95,13 +95,13 @@ uint3 thread_idx [[ thread_position_in_threadgroup ]])
#define innerArguments (logits_data, processed_logits, shared_reduce_buffer, vocab_size, top_p, threadgroup_idx.x, thread_idx.x)
#define generateToppKernel(functionName, scalarType, outerArgs, innerArgs) \
-kernel void functionName##_##scalarType outerArgs { \
- threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \
- functionName innerArgs; \
+[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \
+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \
+ functionName innerArgs; \
}
-#define generateToppKernels(functionName) \
-generateToppKernel(functionName, float, outerArguments(float), innerArguments); \
+#define generateToppKernels(functionName) \
+generateToppKernel(functionName, float, outerArguments(float), innerArguments); \
generateToppKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \
generateToppKernel(functionName, half, outerArguments(half), innerArguments);