A high-performance inference engine for AI models
actual fix of race condition for reduction ops
LuckyIYI1fa975ddf4e4
diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metalindex 972ccd1c7..78a880369 100644--- a/crates/uzu/src/backends/metal/kernel/definitions.metal+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal@@ -224,7 +224,9 @@ static T threadgroup_raking_prefix_exclusive_sum(}threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[lid];+ const T result = shared[lid];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;}//------------------------------------------------------------------------------------------------//@@ -261,7 +263,9 @@ static T threadgroup_raking_reduce_sum(threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[0];+ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;}//------------------------------------------------------------------------------------------------//@@ -298,7 +302,9 @@ static T threadgroup_raking_reduce_max(threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[0];+ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;}//------------------------------------------------------------------------------------------------//@@ -334,7 +340,9 @@ static T threadgroup_raking_reduce_min(}threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[0];+ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;}//------------------------------------------------------------------------------------------------//@@ -373,7 +381,9 @@ static T threadgroup_cooperative_reduce_sum(}threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[0];+ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;}//------------------------------------------------------------------------------------------------//@@ -408,7 +418,9 @@ static T threadgroup_cooperative_reduce_max(}threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[0];+ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;}//------------------------------------------------------------------------------------------------//@@ -443,7 +455,9 @@ static T threadgroup_cooperative_reduce_min(}threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[0];+ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;}// MARK: - Generate Template Kernelsdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metalindex 7cb44b155..0d2e127e4 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal@@ -71,7 +71,9 @@ static ArgmaxPair threadgroup_cooperative_argmax(}threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[0];+ const ArgmaxPair result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;}template <ushort GRAIN_SIZE_PARAM>@@ -146,7 +148,9 @@ static ArgmaxPair threadgroup_raking_argmax(}threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[lid];+ const ArgmaxPair result = shared[lid];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;}template <typename T>diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metalindex bda8c1e8d..cdee11a50 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal@@ -30,17 +30,16 @@ void batched_topk(logit_value > -INFINITY);}- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(- local_min,- shared_reduce_buffer,- thread_idx- );float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(local_max,shared_reduce_buffer,thread_idx);-+ float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(+ local_min,+ shared_reduce_buffer,+ thread_idx+ );// Do the binary search on the thresholdfloat low = min_logit;float high = max_logit;