Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

actual fix of race condition for reduction ops

LuckyIYI1fa975ddf4e4
diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metal
index 972ccd1c7..78a880369 100644
--- a/crates/uzu/src/backends/metal/kernel/definitions.metal
+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal
@@ -224,7 +224,9 @@ static T threadgroup_raking_prefix_exclusive_sum(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[lid];
+ const T result = shared[lid];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
}
//------------------------------------------------------------------------------------------------//
@@ -261,7 +263,9 @@ static T threadgroup_raking_reduce_sum(
threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[0];
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
}
//------------------------------------------------------------------------------------------------//
@@ -298,7 +302,9 @@ static T threadgroup_raking_reduce_max(
threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[0];
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
}
//------------------------------------------------------------------------------------------------//
@@ -334,7 +340,9 @@ static T threadgroup_raking_reduce_min(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[0];
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
}
//------------------------------------------------------------------------------------------------//
@@ -373,7 +381,9 @@ static T threadgroup_cooperative_reduce_sum(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[0];
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
}
//------------------------------------------------------------------------------------------------//
@@ -408,7 +418,9 @@ static T threadgroup_cooperative_reduce_max(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[0];
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
}
//------------------------------------------------------------------------------------------------//
@@ -443,7 +455,9 @@ static T threadgroup_cooperative_reduce_min(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[0];
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
}
// MARK: - Generate Template Kernels
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
index 7cb44b155..0d2e127e4 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
@@ -71,7 +71,9 @@ static ArgmaxPair threadgroup_cooperative_argmax(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[0];
+ const ArgmaxPair result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
}
template <ushort GRAIN_SIZE_PARAM>
@@ -146,7 +148,9 @@ static ArgmaxPair threadgroup_raking_argmax(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[lid];
+ const ArgmaxPair result = shared[lid];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
}
template <typename T>
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index bda8c1e8d..cdee11a50 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -30,17 +30,16 @@ void batched_topk(
logit_value > -INFINITY
);
}
- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
- local_min,
- shared_reduce_buffer,
- thread_idx
- );
float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
local_max,
shared_reduce_buffer,
thread_idx
);
-
+ float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
+ local_min,
+ shared_reduce_buffer,
+ thread_idx
+ );
// Do the binary search on the threshold
float low = min_logit;
float high = max_logit;