Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Match rustfmt line length in clang-format (#513)

uuuvnf2fe36d77725
diff --git a/.clang-format b/.clang-format
index ad70f0798..e3ab56814 100644
--- a/.clang-format
+++ b/.clang-format
@@ -1,7 +1,7 @@
BasedOnStyle: LLVM
IndentWidth: 2
ContinuationIndentWidth: 4
-ColumnLimit: 80
+ColumnLimit: 120
# Keep return type with function name
AlwaysBreakAfterReturnType: None
diff --git a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h
index ede7a07cd..a369e4bd6 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h
@@ -23,8 +23,7 @@ inline T activate_gelu(T x) {
constexpr float k0 = 0.044715f;
constexpr float k1 = 0.7978845608f; // sqrt(2/pi)
float xf = float(x);
- float yf =
- 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));
+ float yf = 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));
return T(yf);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal
index cccd17c89..14e2e12b2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal
@@ -36,11 +36,8 @@ PUBLIC KERNEL(AttentionFallbackScatterScores)(
const uint head_index = group_index * gqa_factor + head_in_group;
const uint prefix_length = sequence_length - suffix_length;
- const uint suffix_position =
- is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
- const uint query_position = is_trie
- ? suffix_position + trie[query_index].height
- : suffix_position + query_index;
+ const uint suffix_position = is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
+ const uint query_position = is_trie ? suffix_position + trie[query_index].height : suffix_position + query_index;
const bool use_key = should_use_key(
ring_params,
@@ -57,10 +54,8 @@ PUBLIC KERNEL(AttentionFallbackScatterScores)(
is_sliding_window
);
- scores
- [(head_index * suffix_length + query_index) * sequence_length +
- sequence_index] =
- use_key ? group_scores[group_score_index] : T(-INFINITY);
+ scores[(head_index * suffix_length + query_index) * sequence_length + sequence_index] =
+ use_key ? group_scores[group_score_index] : T(-INFINITY);
}
template <typename T>
@@ -81,6 +76,5 @@ PUBLIC KERNEL(AttentionFallbackScatterValues)(
const uint query_index = row_index % suffix_length;
const uint head_in_group = row_index / suffix_length;
const uint head_index = group_index * gqa_factor + head_in_group;
- out[(query_index * num_heads + head_index) * head_dim + dim_index] =
- group_output[group_output_index];
+ out[(query_index * num_heads + head_index) * head_dim + dim_index] = group_output[group_output_index];
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index 33d1ed248..252d45687 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -82,10 +82,8 @@ PUBLIC KERNEL(AttentionGemm)(
q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.q_strides[2];
const int kv_head_idx = int(head_idx) / params.gqa_factor;
- k += batch_idx * params.k_strides[0] +
- int64_t(kv_head_idx) * params.k_strides[1];
- v += batch_idx * params.v_strides[0] +
- int64_t(kv_head_idx) * params.v_strides[1];
+ k += batch_idx * params.k_strides[0] + int64_t(kv_head_idx) * params.k_strides[1];
+ v += batch_idx * params.v_strides[0] + int64_t(kv_head_idx) * params.v_strides[1];
o += batch_idx * params.o_strides[0] + head_idx * params.o_strides[1] +
q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.o_strides[2];
@@ -119,31 +117,19 @@ PUBLIC KERNEL(AttentionGemm)(
1,
SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
- using KeyLoader = ThreadgroupLoader<
- T,
- BK,
- BD,
- key_leading_dimension,
- 0,
- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
+ using KeyLoader =
+ ThreadgroupLoader<T, BK, BD, key_leading_dimension, 0, SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
- using ValueLoader = ThreadgroupLoader<
- T,
- BK,
- BD,
- value_leading_dimension,
- 0,
- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
+ using ValueLoader =
+ ThreadgroupLoader<T, BK, BD, value_leading_dimension, 0, SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
const int query_source_stride = int(params.q_strides[2]);
const int key_source_stride = int(params.k_strides[2]);
const int value_source_stride = int(params.v_strides[2]);
- thread QueryLoader
- query_loader(q, query_source_stride, query_shared, thread_context);
+ thread QueryLoader query_loader(q, query_source_stride, query_shared, thread_context);
thread KeyLoader key_loader(k, key_source_stride, key_shared, thread_context);
- thread ValueLoader
- value_loader(v, value_source_stride, value_shared, thread_context);
+ thread ValueLoader value_loader(v, value_source_stride, value_shared, thread_context);
TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));
@@ -151,80 +137,47 @@ PUBLIC KERNEL(AttentionGemm)(
// MMA tiles
constexpr short SIMDGROUP_BLOCK_SIZE = 8;
using AccumType = float;
- using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<
- AccumType,
- SIMDGROUP_BLOCK_SIZE,
- SIMDGROUP_BLOCK_SIZE>;
+ using SimdgroupMultiplyAccumulateType =
+ SimdgroupMultiplyAccumulate<AccumType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;
- constexpr int SIMDGROUPS_PER_THREADGROUP =
- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN;
+ constexpr int SIMDGROUPS_PER_THREADGROUP = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN;
static_assert(
BLOCK_QUERY_ROWS >= (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) &&
- BLOCK_QUERY_ROWS %
- (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) ==
- 0,
+ BLOCK_QUERY_ROWS % (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) == 0,
"Each simdgroup must host at least 1 simdgroup matrix along Q sequence."
);
// Q sequence multiply-accumulate blocks per simdgroup (QUERY_GRID_ROWS == 1
// for the 32-row block layout)
- constexpr int QUERY_GRID_ROWS =
- BLOCK_QUERY_ROWS / (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE);
+ constexpr int QUERY_GRID_ROWS = BLOCK_QUERY_ROWS / (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE);
constexpr int KEY_GRID_COLS = BK / SIMDGROUP_BLOCK_SIZE;
constexpr int HEAD_DIM_GRID_COLS = BD / SIMDGROUP_BLOCK_SIZE;
static_assert(QUERY_GRID_ROWS == 1, "Expected QUERY_GRID_ROWS == 1");
- SimdgroupFragment<
- AccumType,
- QUERY_GRID_ROWS,
- 1,
- SimdgroupMultiplyAccumulateType>
- query_fragment;
- SimdgroupFragment<
- AccumType,
- 1,
- KEY_GRID_COLS,
- SimdgroupMultiplyAccumulateType>
- key_fragment;
- SimdgroupFragment<
- AccumType,
- QUERY_GRID_ROWS,
- KEY_GRID_COLS,
- SimdgroupMultiplyAccumulateType>
- score_fragment;
- SimdgroupFragment<AccumType, 1, 1, SimdgroupMultiplyAccumulateType>
- value_fragment;
- SimdgroupFragment<
- AccumType,
- QUERY_GRID_ROWS,
- HEAD_DIM_GRID_COLS,
- SimdgroupMultiplyAccumulateType>
- output_fragment;
+ SimdgroupFragment<AccumType, QUERY_GRID_ROWS, 1, SimdgroupMultiplyAccumulateType> query_fragment;
+ SimdgroupFragment<AccumType, 1, KEY_GRID_COLS, SimdgroupMultiplyAccumulateType> key_fragment;
+ SimdgroupFragment<AccumType, QUERY_GRID_ROWS, KEY_GRID_COLS, SimdgroupMultiplyAccumulateType> score_fragment;
+ SimdgroupFragment<AccumType, 1, 1, SimdgroupMultiplyAccumulateType> value_fragment;
+ SimdgroupFragment<AccumType, QUERY_GRID_ROWS, HEAD_DIM_GRID_COLS, SimdgroupMultiplyAccumulateType> output_fragment;
output_fragment.clear();
// -------------------------------------------------------------------------
// Lane coordinates and pointer offsets
- const short2 lane_coordinates =
- SimdgroupMultiplyAccumulateType::get_lane_coordinates(
- thread_context.simd_lane_id
- );
+ const short2 lane_coordinates = SimdgroupMultiplyAccumulateType::get_lane_coordinates(thread_context.simd_lane_id);
const short lane_row = lane_coordinates.y;
const short lane_col = lane_coordinates.x;
- const short simdgroup_row_base = SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS *
- short(thread_context.simdgroup_index);
+ const short simdgroup_row_base = SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS * short(thread_context.simdgroup_index);
- const short query_shared_offset =
- (simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;
+ const short query_shared_offset = (simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;
constexpr short query_tile_stride = SIMDGROUP_BLOCK_SIZE;
const short key_shared_offset = lane_col * key_leading_dimension + lane_row;
constexpr short key_tile_stride = SIMDGROUP_BLOCK_SIZE;
- const short value_shared_offset =
- lane_row * value_leading_dimension + lane_col;
+ const short value_shared_offset = lane_row * value_leading_dimension + lane_col;
// -------------------------------------------------------------------------
// Load Q block once (and apply scaling)
@@ -256,17 +209,14 @@ PUBLIC KERNEL(AttentionGemm)(
kb_lim = min(params.nk, kb_lim);
}
- const int q_rel = int(q_tile_idx) * BLOCK_QUERY_ROWS +
- int(simdgroup_row_base) + int(lane_row); // [0, q_len)
+ const int q_rel = int(q_tile_idx) * BLOCK_QUERY_ROWS + int(simdgroup_row_base) + int(lane_row); // [0, q_len)
const int prefix_length = params.q_off;
- const int suffix_position =
- is_kv_cache_ring ? int(ring_params.ring_length) : prefix_length;
+ const int suffix_position = is_kv_cache_ring ? int(ring_params.ring_length) : prefix_length;
- const int query_position = (is_trie && q_rel < params.q_len)
- ? suffix_position + int(trie[q_rel].height)
- : suffix_position + q_rel;
+ const int query_position =
+ (is_trie && q_rel < params.q_len) ? suffix_position + int(trie[q_rel].height) : suffix_position + q_rel;
// Loop over KV blocks
for (int kb = 0; kb < kb_lim; kb++) {
@@ -295,12 +245,7 @@ PUBLIC KERNEL(AttentionGemm)(
);
simdgroup_barrier(mem_flags::mem_none);
- tile_multiply_accumulate(
- score_fragment,
- query_fragment,
- key_fragment,
- score_fragment
- );
+ tile_multiply_accumulate(score_fragment, query_fragment, key_fragment, score_fragment);
}
// Mask out tail keys for the last (unaligned) K block
@@ -324,8 +269,7 @@ PUBLIC KERNEL(AttentionGemm)(
METAL_PRAGMA_UNROLL
for (short j = 0; j < KEY_GRID_COLS; j++) {
thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
- const int col_base =
- kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;
+ const int col_base = kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;
const int k0 = col_base;
const int k1 = col_base + 1;
@@ -426,8 +370,7 @@ PUBLIC KERNEL(AttentionGemm)(
const short dd = id * SIMDGROUP_BLOCK_SIZE;
value_fragment.template load<T, 1, 1, value_leading_dimension, 1>(
- &value_shared
- [value_shared_offset + kk * value_leading_dimension + dd]
+ &value_shared[value_shared_offset + kk * value_leading_dimension + dd]
);
if constexpr (BD == 128) {
@@ -461,22 +404,16 @@ PUBLIC KERNEL(AttentionGemm)(
threadgroup_barrier(mem_flags::mem_none);
// Store results (O is row-major with row-stride params.o_strides[2])
- o += int64_t(simdgroup_row_base + lane_row) * params.o_strides[2] +
- int64_t(lane_col);
+ o += int64_t(simdgroup_row_base + lane_row) * params.o_strides[2] + int64_t(lane_col);
if (!align_q && int(q_tile_idx) == params.nq_aligned) {
- const short2 dst_tile_dims =
- short2(BD - lane_col, params.q_rem - (simdgroup_row_base + lane_row));
+ const short2 dst_tile_dims = short2(BD - lane_col, params.q_rem - (simdgroup_row_base + lane_row));
if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {
return;
}
- output_fragment.template store_safe<T, 1, 1>(
- o,
- int(params.o_strides[2]),
- dst_tile_dims
- );
+ output_fragment.template store_safe<T, 1, 1>(o, int(params.o_strides[2]), dst_tile_dims);
} else {
output_fragment.template store<T, 1, 1>(o, int(params.o_strides[2]));
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index 221d1ed3d..4fcf79338 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -62,28 +62,21 @@ PUBLIC KERNEL(AttentionSinglePass)(
const uint kv_head_idx = head_idx / gqa_factor;
const uint o_offset = q_seq_idx * num_heads + head_idx;
- const uint q_offset = query_transposed ? num_heads * q_seq_idx + head_idx
- : head_idx * suffix_length + q_seq_idx;
+ const uint q_offset = query_transposed ? num_heads * q_seq_idx + head_idx : head_idx * suffix_length + q_seq_idx;
const uint prefix_length = sequence_length - suffix_length;
- const uint suffix_position =
- is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
+ const uint suffix_position = is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
- const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height
- : suffix_position + q_seq_idx;
+ const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height : suffix_position + q_seq_idx;
- queries += q_offset * HEAD_DIM +
- thread_context.simd_lane_id * qk_elements_per_thread;
- keys += kv_head_idx * k_head_stride +
- thread_context.simdgroup_index * k_seq_stride +
+ queries += q_offset * HEAD_DIM + thread_context.simd_lane_id * qk_elements_per_thread;
+ keys += kv_head_idx * k_head_stride + thread_context.simdgroup_index * k_seq_stride +
thread_context.simd_lane_id * qk_elements_per_thread;
- values += kv_head_idx * v_head_stride +
- thread_context.simdgroup_index * v_seq_stride +
+ values += kv_head_idx * v_head_stride + thread_context.simdgroup_index * v_seq_stride +
thread_context.simd_lane_id * value_elements_per_thread;
- out += o_offset * value_dim +
- thread_context.simdgroup_index * value_elements_per_thread;
+ out += o_offset * value_dim + thread_context.simdgroup_index * value_elements_per_thread;
// Read the query and 0 the output accumulator
for (int i = 0; i < qk_elements_per_thread; i++) {
@@ -103,8 +96,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
}
// For each key
- for (uint i = thread_context.simdgroup_index; i < sequence_length;
- i += SEQUENCE_BLOCK_SIZE) {
+ for (uint i = thread_context.simdgroup_index; i < sequence_length; i += SEQUENCE_BLOCK_SIZE) {
if (should_use_key(
ring_params,
trie,
@@ -159,20 +151,14 @@ PUBLIC KERNEL(AttentionSinglePass)(
max_score = shared_max_scores[thread_context.simd_lane_id];
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
- sum_exp_score =
- simd_sum(shared_sum_exp_scores[thread_context.simd_lane_id] * factor);
+ sum_exp_score = simd_sum(shared_sum_exp_scores[thread_context.simd_lane_id] * factor);
// Now we need to aggregate all the outputs
for (int i = 0; i < value_elements_per_thread; i++) {
- shared_outputs
- [thread_context.simd_lane_id * HEAD_BLOCK_SIZE +
- thread_context.simdgroup_index] = o[i];
+ shared_outputs[thread_context.simd_lane_id * HEAD_BLOCK_SIZE + thread_context.simdgroup_index] = o[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
o[i] = simd_sum(
- shared_outputs
- [thread_context.simdgroup_index * HEAD_BLOCK_SIZE +
- thread_context.simd_lane_id] *
- factor
+ shared_outputs[thread_context.simdgroup_index * HEAD_BLOCK_SIZE + thread_context.simd_lane_id] * factor
) /
sum_exp_score;
threadgroup_barrier(mem_flags::mem_threadgroup);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index 5f4b95300..a88f9433e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -56,8 +56,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
typedef float U;
// Flat query head mapping — fixed hardware shape, dynamic logical mapping
- const uint head_idx = head_group_idx * SIMDGROUPS_PER_THREADGROUP +
- thread_context.simdgroup_index;
+ const uint head_idx = head_group_idx * SIMDGROUPS_PER_THREADGROUP + thread_context.simdgroup_index;
if (head_idx >= num_heads)
return;
@@ -66,20 +65,15 @@ PUBLIC KERNEL(AttentionTwoPass1)(
const uint q_offset = head_idx * suffix_length + q_seq_idx;
const uint prefix_length = sequence_length - suffix_length;
- const uint suffix_position =
- is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
- const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height
- : suffix_position + q_seq_idx;
+ const uint suffix_position = is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
+ const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height : suffix_position + q_seq_idx;
thread U q[elements_per_thread];
thread U o[elements_per_thread] = {0};
- queries +=
- q_offset * HEAD_DIM + thread_context.simd_lane_id * elements_per_thread;
- keys += kv_head_idx * k_head_stride + block_idx * k_seq_stride +
- thread_context.simd_lane_id * elements_per_thread;
- values += kv_head_idx * v_head_stride + block_idx * v_seq_stride +
- thread_context.simd_lane_id * elements_per_thread;
+ queries += q_offset * HEAD_DIM + thread_context.simd_lane_id * elements_per_thread;
+ keys += kv_head_idx * k_head_stride + block_idx * k_seq_stride + thread_context.simd_lane_id * elements_per_thread;
+ values += kv_head_idx * v_head_stride + block_idx * v_seq_stride + thread_context.simd_lane_id * elements_per_thread;
out += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM +
thread_context.simd_lane_id * elements_per_thread;
sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
@@ -170,16 +164,14 @@ PUBLIC KERNEL(AttentionTwoPass2)(
// Find global max across all blocks (each lane handles strided blocks)
U max_score = -1e9;
- for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT;
- b += METAL_SIMD_SIZE) {
+ for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT; b += METAL_SIMD_SIZE) {
max_score = max(max_score, maxs[b]);
}
U new_max = simd_max(max_score);
// Find global sum across all blocks
U sum_exp = 0;
- for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT;
- b += METAL_SIMD_SIZE) {
+ for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT; b += METAL_SIMD_SIZE) {
sum_exp += sums[b] * fast::exp(maxs[b] - new_max);
}
U sum_exp_score = simd_sum(sum_exp);
@@ -190,14 +182,10 @@ PUBLIC KERNEL(AttentionTwoPass2)(
for (uint chunk = 0; chunk < TOTAL_BLOCKS_COUNT; chunk += METAL_SIMD_SIZE) {
uint block_idx = chunk + thread_context.simd_lane_id;
- U factor = (block_idx < TOTAL_BLOCKS_COUNT)
- ? fast::exp(maxs[block_idx] - new_max)
- : 0;
+ U factor = (block_idx < TOTAL_BLOCKS_COUNT) ? fast::exp(maxs[block_idx] - new_max) : 0;
- const device float* block_partials =
- partials + o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +
- block_idx * HEAD_DIM +
- thread_context.simdgroup_index * elements_per_thread;
+ const device float* block_partials = partials + o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM +
+ thread_context.simdgroup_index * elements_per_thread;
for (uint i = 0; i < elements_per_thread; i++) {
U val = (block_idx < TOTAL_BLOCKS_COUNT) ? block_partials[i] * factor : 0;
@@ -207,8 +195,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(
// Write the output
if (thread_context.simd_lane_id == 0) {
- out += o_offset * HEAD_DIM +
- thread_context.simdgroup_index * elements_per_thread;
+ out += o_offset * HEAD_DIM + thread_context.simdgroup_index * elements_per_thread;
for (uint i = 0; i < elements_per_thread; i++) {
out[i] = static_cast<T>(o[i] / sum_exp_score);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
index f38009c50..f8dcd5365 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
@@ -28,22 +28,17 @@ PUBLIC KERNEL(AttentionUpdateKVCache)(
// keys_in_place=true: destination shares rotated_keys' group-major layout.
// Otherwise, KV cache is token-major:
// [max_sequence_length, num_groups, head_dim].
- const uint cacheOffset =
- keys_in_place
- ? ((group_index * max_sequence_length + cacheTokenIndex) * head_dim +
- dim_index)
- : ((cacheTokenIndex * num_groups + group_index) * head_dim +
- dim_index);
+ const uint cacheOffset = keys_in_place
+ ? ((group_index * max_sequence_length + cacheTokenIndex) * head_dim + dim_index)
+ : ((cacheTokenIndex * num_groups + group_index) * head_dim + dim_index);
- const uint rotatedKeyOffset =
- (group_index * suffix_length + token_index) * head_dim + dim_index;
+ const uint rotatedKeyOffset = (group_index * suffix_length + token_index) * head_dim + dim_index;
key_cache[cacheOffset] = rotated_keys[rotatedKeyOffset];
// qkv layout: [suffix_length, (num_heads + 2*num_groups) * head_dim]
// Values start at offset: (num_heads * head_dim) + (num_groups * head_dim)
const uint qkvStride = (num_heads + 2 * num_groups) * head_dim;
const uint valueBaseOffset = (num_heads + num_groups) * head_dim;
- const uint valueOffset = token_index * qkvStride + valueBaseOffset +
- group_index * head_dim + dim_index;
+ const uint valueOffset = token_index * qkvStride + valueBaseOffset + group_index * head_dim + dim_index;
value_cache[cacheOffset] = qkv[valueOffset];
}
\ No newline at end of file
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/mask.h b/crates/backend-uzu/src/backends/metal/kernel/attention/mask.h
index e553eabc0..8b6f4cb5e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/mask.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/mask.h
@@ -35,8 +35,7 @@ static inline bool should_use_key(
key_position = suffix_position + key_trie.height;
if (is_causal) {
- use_key &=
- q_seq_idx >= key_trie.trie_start && q_seq_idx <= key_trie.trie_end;
+ use_key &= q_seq_idx >= key_trie.trie_start && q_seq_idx <= key_trie.trie_end;
}
} else {
// Non-trie, position equals index in the suffix
@@ -51,8 +50,7 @@ static inline bool should_use_key(
// We're in the prefix
if (is_kv_cache_ring) {
// Ring, calculate position in the ring
- key_position =
- (prefix_length + i - ring_params.ring_offset) % prefix_length;
+ key_position = (prefix_length + i - ring_params.ring_offset) % prefix_length;
// Ring also skips entries that aren't filled yet
use_key &= key_position < ring_params.ring_length;
} else {
@@ -63,8 +61,7 @@ static inline bool should_use_key(
if (is_sliding_window) {
if (is_causal) {
- use_key &= key_position <= query_position &&
- (query_position - key_position) < sliding_window_size;
+ use_key &= key_position <= query_position && (query_position - key_position) < sliding_window_size;
} else {
if (key_position <= query_position) {
use_key &= (query_position - key_position) <= (sliding_window_size / 2);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metal
index 23dab48e5..b2b19ff5e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metal
@@ -20,14 +20,11 @@ PUBLIC KERNEL(QkUnpack)(
const uint total_heads = num_heads + 2 * num_groups;
const uint first_head_in_group = group_index * (num_heads / num_groups);
- TensorView3D<const T> qkv_tensor_view =
- TensorView3D<const T>(qkv).shaped(suffix_length, total_heads, head_dim);
+ TensorView3D<const T> qkv_tensor_view = TensorView3D<const T>(qkv).shaped(suffix_length, total_heads, head_dim);
TensorView3D<T> unpacked_queries_tensor_view =
- TensorView3D<T>(unpacked_queries)
- .shaped(num_heads, suffix_length, head_dim);
+ TensorView3D<T>(unpacked_queries).shaped(num_heads, suffix_length, head_dim);
TensorView3D<T> unpacked_keys_tensor_view =
- TensorView3D<T>(unpacked_keys)
- .shaped(num_groups, suffix_length, head_dim);
+ TensorView3D<T>(unpacked_keys).shaped(num_groups, suffix_length, head_dim);
unpacked_queries_tensor_view(head_index, token_index, dimension_index) =
qkv_tensor_view(token_index, head_index, dimension_index);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/add.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/add.metal
index 545920cd5..a2cb4ffe5 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/add.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/add.metal
@@ -4,13 +4,7 @@
using namespace metal;
template <typename T>
-void add_tensors(
- device const T* a,
- device const T* b,
- device T* out,
- const constant int& n,
- const uint tid
-) {
+void add_tensors(device const T* a, device const T* b, device T* out, const constant int& n, const uint tid) {
if ((int)tid >= n) {
return;
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d.metal
index cfcd6e82f..2f29db10e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d.metal
@@ -74,8 +74,7 @@ void causal_conv1d(
acc2 += w * float(input[x_idx + 2]);
acc3 += w * float(input[x_idx + 3]);
} else {
- const uint x_idx0 =
- (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;
+ const uint x_idx0 = (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;
acc0 += w * float(input[x_idx0]);
acc1 += w * float(input[x_idx0 + (uint)cin]);
acc2 += w * float(input[x_idx0 + (uint)(2 * cin)]);
@@ -84,34 +83,27 @@ void causal_conv1d(
continue;
}
if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {
- const uint x_idx =
- (input_layout == AUDIO_LAYOUT_NCS)
- ? (x_base_ncs + (uint)x_t)
- : ((b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic);
+ const uint x_idx = (input_layout == AUDIO_LAYOUT_NCS)
+ ? (x_base_ncs + (uint)x_t)
+ : ((b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic);
acc0 += w * float(input[x_idx]);
}
if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {
- const uint x_idx =
- (input_layout == AUDIO_LAYOUT_NCS)
- ? (x_base_ncs + (uint)(x_t + 1))
- : ((b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin +
- (uint)ic);
+ const uint x_idx = (input_layout == AUDIO_LAYOUT_NCS)
+ ? (x_base_ncs + (uint)(x_t + 1))
+ : ((b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin + (uint)ic);
acc1 += w * float(input[x_idx]);
}
if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {
- const uint x_idx =
- (input_layout == AUDIO_LAYOUT_NCS)
- ? (x_base_ncs + (uint)(x_t + 2))
- : ((b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin +
- (uint)ic);
+ const uint x_idx = (input_layout == AUDIO_LAYOUT_NCS)
+ ? (x_base_ncs + (uint)(x_t + 2))
+ : ((b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin + (uint)ic);
acc2 += w * float(input[x_idx]);
}
if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {
- const uint x_idx =
- (input_layout == AUDIO_LAYOUT_NCS)
- ? (x_base_ncs + (uint)(x_t + 3))
- : ((b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin +
- (uint)ic);
+ const uint x_idx = (input_layout == AUDIO_LAYOUT_NCS)
+ ? (x_base_ncs + (uint)(x_t + 3))
+ : ((b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin + (uint)ic);
acc3 += w * float(input[x_idx]);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal
index 5e1a5ef4e..35ed3ea05 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal
@@ -71,9 +71,7 @@ void causal_conv1d_grouped_ncs(
const bool full_tile = valid_count == (int)AUDIO_TIME_TILE;
for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {
const uint w_base =
- ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group +
- (uint)ic_local) *
- (uint)kernel_size;
+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group + (uint)ic_local) * (uint)kernel_size;
const int ic = in_begin + ic_local;
const uint x_base_ncs = (b * (uint)cin + (uint)ic) * (uint)seq_len;
@@ -216,16 +214,13 @@ void causal_conv1d_grouped_nsc(
for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {
const int ic = in_begin + ic_local;
const uint w_base =
- ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group +
- (uint)ic_local) *
- (uint)kernel_size;
+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group + (uint)ic_local) * (uint)kernel_size;
for (int k = 0; k < kernel_size; ++k) {
const float w = float(weight[w_base + (uint)k]);
const int x_t = (int)t + k * dilation - pad;
if (full_tile && x_t >= 0 && x_t + 7 < seq_len) {
- const uint x_idx0 =
- (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;
+ const uint x_idx0 = (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;
acc0 += w * float(input[x_idx0]);
acc1 += w * float(input[x_idx0 + (uint)cin]);
acc2 += w * float(input[x_idx0 + (uint)(2 * cin)]);
@@ -237,43 +232,35 @@ void causal_conv1d_grouped_nsc(
continue;
}
if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {
- const uint x_idx =
- (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;
+ const uint x_idx = (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;
acc0 += w * float(input[x_idx]);
}
if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {
- const uint x_idx =
- (b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin + (uint)ic;
+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin + (uint)ic;
acc1 += w * float(input[x_idx]);
}
if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {
- const uint x_idx =
- (b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin + (uint)ic;
+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin + (uint)ic;
acc2 += w * float(input[x_idx]);
}
if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {
- const uint x_idx =
- (b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin + (uint)ic;
+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin + (uint)ic;
acc3 += w * float(input[x_idx]);
}
if (valid_count > 4 && x_t + 4 >= 0 && x_t + 4 < seq_len) {
- const uint x_idx =
- (b * (uint)seq_len + (uint)(x_t + 4)) * (uint)cin + (uint)ic;
+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 4)) * (uint)cin + (uint)ic;
acc4 += w * float(input[x_idx]);
}
if (valid_count > 5 && x_t + 5 >= 0 && x_t + 5 < seq_len) {
- const uint x_idx =
- (b * (uint)seq_len + (uint)(x_t + 5)) * (uint)cin + (uint)ic;
+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 5)) * (uint)cin + (uint)ic;
acc5 += w * float(input[x_idx]);
}
if (valid_count > 6 && x_t + 6 >= 0 && x_t + 6 < seq_len) {
- const uint x_idx =
- (b * (uint)seq_len + (uint)(x_t + 6)) * (uint)cin + (uint)ic;
+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 6)) * (uint)cin + (uint)ic;
acc6 += w * float(input[x_idx]);
}
if (valid_count > 7 && x_t + 7 >= 0 && x_t + 7 < seq_len) {
- const uint x_idx =
- (b * (uint)seq_len + (uint)(x_t + 7)) * (uint)cin + (uint)ic;
+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 7)) * (uint)cin + (uint)ic;
acc7 += w * float(input[x_idx]);
}
}
@@ -327,34 +314,10 @@ PUBLIC KERNEL(AudioCausalConv1dGrouped)(
uint b AXIS(batch_size, 1)
) {
if (input_layout == AUDIO_LAYOUT_NCS) {
- causal_conv1d_grouped_ncs<T>(
- input,
- weight,
- bias,
- output,
- lengths,
- cin,
- cout,
- seq_len,
- kernel_size,
- dilation,
- groups,
- uint3(t, oc, b)
- );
+ causal_conv1d_grouped_ncs<
+ T>(input, weight, bias, output, lengths, cin, cout, seq_len, kernel_size, dilation, groups, uint3(t, oc, b));
} else {
- causal_conv1d_grouped_nsc<T>(
- input,
- weight,
- bias,
- output,
- lengths,
- cin,
- cout,
- seq_len,
- kernel_size,
- dilation,
- groups,
- uint3(t, oc, b)
- );
+ causal_conv1d_grouped_nsc<
+ T>(input, weight, bias, output, lengths, cin, cout, seq_len, kernel_size, dilation, groups, uint3(t, oc, b));
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal
index bfc694c9f..e472b03ac 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal
@@ -80,9 +80,7 @@ void causal_conv1d_grouped_residual(
for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {
const int ic = in_begin + ic_local;
const uint w_base =
- ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group +
- (uint)ic_local) *
- (uint)kernel_size;
+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group + (uint)ic_local) * (uint)kernel_size;
const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len;
for (int k = 0; k < kernel_size; ++k) {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal
index 016716aaa..6e1ee11f2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal
@@ -54,16 +54,14 @@ void causal_conv_transpose1d(
// Weight layout: [Cout, Cin_per_group, K]
const uint ic_local = (uint)(ic - ic_begin);
- const uint w_base =
- ((uint)oc * (uint)cin_per_group + ic_local) * (uint)(2 * stride);
+ const uint w_base = ((uint)oc * (uint)cin_per_group + ic_local) * (uint)(2 * stride);
// Contribution from input[q] with k=r
acc += float(weight[w_base + (uint)r]) * float(input[in_base + (uint)q]);
// Contribution from input[q-1] with k=stride+r
if (q > 0) {
- acc += float(weight[w_base + (uint)(stride + r)]) *
- float(input[in_base + (uint)(q - 1)]);
+ acc += float(weight[w_base + (uint)(stride + r)]) * float(input[in_base + (uint)(q - 1)]);
}
}
@@ -89,18 +87,6 @@ PUBLIC KERNEL(AudioCausalConvTranspose1d)(
uint oc AXIS(cout, 1),
uint b AXIS(batch_size, 1)
) {
- causal_conv_transpose1d<T>(
- input,
- weight,
- bias,
- output,
- lengths,
- cin,
- cout,
- seq_len_in,
- seq_len_out,
- stride,
- groups,
- uint3(t_out, oc, b)
- );
+ causal_conv_transpose1d<
+ T>(input, weight, bias, output, lengths, cin, cout, seq_len_in, seq_len_out, stride, groups, uint3(t_out, oc, b));
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal
index 46d3bed81..2481064e6 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal
@@ -40,9 +40,8 @@ void causal_conv_transpose1d_causal_pad(
if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {
return;
}
- if ((input_layout != AUDIO_LAYOUT_NCS && input_layout != AUDIO_LAYOUT_NSC) ||
- groups <= 0 || stride <= 0 || kernel_size <= 0 || (cin % groups) != 0 ||
- (cout % groups) != 0) {
+ if ((input_layout != AUDIO_LAYOUT_NCS && input_layout != AUDIO_LAYOUT_NSC) || groups <= 0 || stride <= 0 ||
+ kernel_size <= 0 || (cin % groups) != 0 || (cout % groups) != 0) {
return;
}
@@ -69,8 +68,7 @@ void causal_conv_transpose1d_causal_pad(
const int ic_begin = group_idx * cin_per_group;
const int ic_end = ic_begin + cin_per_group;
- const int seq_len_expanded =
- (seq_len_in > 0) ? (((seq_len_in - 1) * stride) + 1) : 0;
+ const int seq_len_expanded = (seq_len_in > 0) ? (((seq_len_in - 1) * stride) + 1) : 0;
const int left_pad = kernel_size - 1;
const bool fast_two_tap = (kernel_size == (stride * 2));
@@ -92,8 +90,7 @@ void causal_conv_transpose1d_causal_pad(
for (int ic = ic_begin; ic < ic_end; ++ic) {
const uint in_base = (b * (uint)cin + (uint)ic) * (uint)seq_len_in;
const uint ic_local = (uint)(ic - ic_begin);
- const uint w_base =
- ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;
+ const uint w_base = ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;
if (fast_two_tap) {
// When stride >= AUDIO_TIME_TILE (common for decoder blocks with
// strides 4,8), all 4 lanes share the same q value. This means
@@ -103,42 +100,30 @@ void causal_conv_transpose1d_causal_pad(
if (stride >= (int)AUDIO_TIME_TILE) {
// All lanes have same q since t_out+3 < t_out+stride means
// (t_out+3)/stride == t_out/stride.
- const float in_qm1 = (q0 > 0 && q0 - 1 < seq_len_in)
- ? float(input[in_base + (uint)(q0 - 1)])
- : 0.0f;
- const float in_q0 = (q0 >= 0 && q0 < seq_len_in)
- ? float(input[in_base + (uint)q0])
- : 0.0f;
+ const float in_qm1 = (q0 > 0 && q0 - 1 < seq_len_in) ? float(input[in_base + (uint)(q0 - 1)]) : 0.0f;
+ const float in_q0 = (q0 >= 0 && q0 < seq_len_in) ? float(input[in_base + (uint)q0]) : 0.0f;
// Lane 0: r = r0
if (valid_count > 0) {
- acc0 +=
- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);
- acc0 += in_q0 *
- float(weight[w_base + (uint)((kernel_size - 1) - r0)]);
+ acc0 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);
+ acc0 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r0)]);
}
// Lane 1: r = r0+1
if (valid_count > 1) {
const int r1 = r0 + 1;
- acc1 +=
- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);
- acc1 += in_q0 *
- float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
+ acc1 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);
+ acc1 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
}
// Lane 2: r = r0+2
if (valid_count > 2) {
const int r2 = r0 + 2;
- acc2 +=
- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);
- acc2 += in_q0 *
- float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
+ acc2 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);
+ acc2 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
}
// Lane 3: r = r0+3
if (valid_count > 3) {
const int r3 = r0 + 3;
- acc3 +=
- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);
- acc3 += in_q0 *
- float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
+ acc3 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);
+ acc3 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
}
} else {
// stride < 4 (e.g., stride=2 for upsampler blocks).
@@ -147,48 +132,40 @@ void causal_conv_transpose1d_causal_pad(
const int k_lo = (stride - 1) - r0;
const int k_hi = (kernel_size - 1) - r0;
if (q0 > 0 && q0 - 1 < seq_len_in) {
- acc0 += float(input[in_base + (uint)(q0 - 1)]) *
- float(weight[w_base + (uint)k_lo]);
+ acc0 += float(input[in_base + (uint)(q0 - 1)]) * float(weight[w_base + (uint)k_lo]);
}
if (q0 >= 0 && q0 < seq_len_in) {
- acc0 += float(input[in_base + (uint)q0]) *
- float(weight[w_base + (uint)k_hi]);
+ acc0 += float(input[in_base + (uint)q0]) * float(weight[w_base + (uint)k_hi]);
}
}
if (valid_count > 1) {
const int q1 = (t0 + 1) / stride;
const int r1 = (t0 + 1) % stride;
if (q1 > 0 && q1 - 1 < seq_len_in) {
- acc1 += float(input[in_base + (uint)(q1 - 1)]) *
- float(weight[w_base + (uint)((stride - 1) - r1)]);
+ acc1 += float(input[in_base + (uint)(q1 - 1)]) * float(weight[w_base + (uint)((stride - 1) - r1)]);
}
if (q1 >= 0 && q1 < seq_len_in) {
- acc1 += float(input[in_base + (uint)q1]) *
- float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
+ acc1 += float(input[in_base + (uint)q1]) * float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
}
}
if (valid_count > 2) {
const int q2 = (t0 + 2) / stride;
const int r2 = (t0 + 2) % stride;
if (q2 > 0 && q2 - 1 < seq_len_in) {
- acc2 += float(input[in_base + (uint)(q2 - 1)]) *
- float(weight[w_base + (uint)((stride - 1) - r2)]);
+ acc2 += float(input[in_base + (uint)(q2 - 1)]) * float(weight[w_base + (uint)((stride - 1) - r2)]);
}
if (q2 >= 0 && q2 < seq_len_in) {
- acc2 += float(input[in_base + (uint)q2]) *
- float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
+ acc2 += float(input[in_base + (uint)q2]) * float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
}
}
if (valid_count > 3) {
const int q3 = (t0 + 3) / stride;
const int r3 = (t0 + 3) % stride;
if (q3 > 0 && q3 - 1 < seq_len_in) {
- acc3 += float(input[in_base + (uint)(q3 - 1)]) *
- float(weight[w_base + (uint)((stride - 1) - r3)]);
+ acc3 += float(input[in_base + (uint)(q3 - 1)]) * float(weight[w_base + (uint)((stride - 1) - r3)]);
}
if (q3 >= 0 && q3 < seq_len_in) {
- acc3 += float(input[in_base + (uint)q3]) *
- float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
+ acc3 += float(input[in_base + (uint)q3]) * float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
}
}
}
@@ -198,8 +175,7 @@ void causal_conv_transpose1d_causal_pad(
// Lane 0
if (valid_count > 0) {
const int expanded_time0 = (int)t_out + k - left_pad;
- if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded &&
- (expanded_time0 % stride) == 0) {
+ if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded && (expanded_time0 % stride) == 0) {
const int src_time0 = expanded_time0 / stride;
if (src_time0 >= 0 && src_time0 < seq_len_in) {
acc0 += float(input[in_base + (uint)src_time0]) * w;
@@ -209,8 +185,7 @@ void causal_conv_transpose1d_causal_pad(
// Lane 1
if (valid_count > 1) {
const int expanded_time1 = (int)t_out + 1 + k - left_pad;
- if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded &&
- (expanded_time1 % stride) == 0) {
+ if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded && (expanded_time1 % stride) == 0) {
const int src_time1 = expanded_time1 / stride;
if (src_time1 >= 0 && src_time1 < seq_len_in) {
acc1 += float(input[in_base + (uint)src_time1]) * w;
@@ -220,8 +195,7 @@ void causal_conv_transpose1d_causal_pad(
// Lane 2
if (valid_count > 2) {
const int expanded_time2 = (int)t_out + 2 + k - left_pad;
- if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded &&
- (expanded_time2 % stride) == 0) {
+ if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded && (expanded_time2 % stride) == 0) {
const int src_time2 = expanded_time2 / stride;
if (src_time2 >= 0 && src_time2 < seq_len_in) {
acc2 += float(input[in_base + (uint)src_time2]) * w;
@@ -231,8 +205,7 @@ void causal_conv_transpose1d_causal_pad(
// Lane 3
if (valid_count > 3) {
const int expanded_time3 = (int)t_out + 3 + k - left_pad;
- if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded &&
- (expanded_time3 % stride) == 0) {
+ if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded && (expanded_time3 % stride) == 0) {
const int src_time3 = expanded_time3 / stride;
if (src_time3 >= 0 && src_time3 < seq_len_in) {
acc3 += float(input[in_base + (uint)src_time3]) * w;
@@ -246,117 +219,81 @@ void causal_conv_transpose1d_causal_pad(
// NSC layout
for (int ic = ic_begin; ic < ic_end; ++ic) {
const uint ic_local = (uint)(ic - ic_begin);
- const uint w_base =
- ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;
+ const uint w_base = ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;
if (fast_two_tap) {
if (stride >= (int)AUDIO_TIME_TILE) {
// All lanes share q0; load input once per (q-1, q) pair.
- const float in_qm1 =
- (q0 > 0 && q0 - 1 < seq_len_in)
- ? float(input
- [(b * (uint)seq_len_in + (uint)(q0 - 1)) *
- (uint)cin +
- (uint)ic])
- : 0.0f;
- const float in_q0 =
- (q0 >= 0 && q0 < seq_len_in)
- ? float(input
- [(b * (uint)seq_len_in + (uint)q0) * (uint)cin +
- (uint)ic])
- : 0.0f;
+ const float in_qm1 = (q0 > 0 && q0 - 1 < seq_len_in)
+ ? float(input[(b * (uint)seq_len_in + (uint)(q0 - 1)) * (uint)cin + (uint)ic])
+ : 0.0f;
+ const float in_q0 = (q0 >= 0 && q0 < seq_len_in)
+ ? float(input[(b * (uint)seq_len_in + (uint)q0) * (uint)cin + (uint)ic])
+ : 0.0f;
if (valid_count > 0) {
- acc0 +=
- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);
- acc0 += in_q0 *
- float(weight[w_base + (uint)((kernel_size - 1) - r0)]);
+ acc0 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);
+ acc0 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r0)]);
}
if (valid_count > 1) {
const int r1 = r0 + 1;
- acc1 +=
- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);
- acc1 += in_q0 *
- float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
+ acc1 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);
+ acc1 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
}
if (valid_count > 2) {
const int r2 = r0 + 2;
- acc2 +=
- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);
- acc2 += in_q0 *
- float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
+ acc2 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);
+ acc2 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
}
if (valid_count > 3) {
const int r3 = r0 + 3;
- acc3 +=
- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);
- acc3 += in_q0 *
- float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
+ acc3 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);
+ acc3 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
}
} else {
// stride < 4: lanes may have different q values.
if (valid_count > 0) {
if (q0 > 0 && q0 - 1 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)(q0 - 1)) * (uint)cin +
- (uint)ic;
- acc0 += float(input[in_idx]) *
- float(weight[w_base + (uint)((stride - 1) - r0)]);
+ const uint in_idx = (b * (uint)seq_len_in + (uint)(q0 - 1)) * (uint)cin + (uint)ic;
+ acc0 += float(input[in_idx]) * float(weight[w_base + (uint)((stride - 1) - r0)]);
}
if (q0 >= 0 && q0 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)q0) * (uint)cin + (uint)ic;
- acc0 += float(input[in_idx]) *
- float(weight[w_base + (uint)((kernel_size - 1) - r0)]);
+ const uint in_idx = (b * (uint)seq_len_in + (uint)q0) * (uint)cin + (uint)ic;
+ acc0 += float(input[in_idx]) * float(weight[w_base + (uint)((kernel_size - 1) - r0)]);
}
}
if (valid_count > 1) {
const int q1 = (t0 + 1) / stride;
const int r1 = (t0 + 1) % stride;
if (q1 > 0 && q1 - 1 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)(q1 - 1)) * (uint)cin +
- (uint)ic;
- acc1 += float(input[in_idx]) *
- float(weight[w_base + (uint)((stride - 1) - r1)]);
+ const uint in_idx = (b * (uint)seq_len_in + (uint)(q1 - 1)) * (uint)cin + (uint)ic;
+ acc1 += float(input[in_idx]) * float(weight[w_base + (uint)((stride - 1) - r1)]);
}
if (q1 >= 0 && q1 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)q1) * (uint)cin + (uint)ic;
- acc1 += float(input[in_idx]) *
- float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
+ const uint in_idx = (b * (uint)seq_len_in + (uint)q1) * (uint)cin + (uint)ic;
+ acc1 += float(input[in_idx]) * float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
}
}
if (valid_count > 2) {
const int q2 = (t0 + 2) / stride;
const int r2 = (t0 + 2) % stride;
if (q2 > 0 && q2 - 1 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)(q2 - 1)) * (uint)cin +
- (uint)ic;
- acc2 += float(input[in_idx]) *
- float(weight[w_base + (uint)((stride - 1) - r2)]);
+ const uint in_idx = (b * (uint)seq_len_in + (uint)(q2 - 1)) * (uint)cin + (uint)ic;
+ acc2 += float(input[in_idx]) * float(weight[w_base + (uint)((stride - 1) - r2)]);
}
if (q2 >= 0 && q2 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)q2) * (uint)cin + (uint)ic;
- acc2 += float(input[in_idx]) *
- float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
+ const uint in_idx = (b * (uint)seq_len_in + (uint)q2) * (uint)cin + (uint)ic;
+ acc2 += float(input[in_idx]) * float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
}
}
if (valid_count > 3) {
const int q3 = (t0 + 3) / stride;
const int r3 = (t0 + 3) % stride;
if (q3 > 0 && q3 - 1 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)(q3 - 1)) * (uint)cin +
- (uint)ic;
- acc3 += float(input[in_idx]) *
- float(weight[w_base + (uint)((stride - 1) - r3)]);
+ const uint in_idx = (b * (uint)seq_len_in + (uint)(q3 - 1)) * (uint)cin + (uint)ic;
+ acc3 += float(input[in_idx]) * float(weight[w_base + (uint)((stride - 1) - r3)]);
}
if (q3 >= 0 && q3 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)q3) * (uint)cin + (uint)ic;
- acc3 += float(input[in_idx]) *
- float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
+ const uint in_idx = (b * (uint)seq_len_in + (uint)q3) * (uint)cin + (uint)ic;
+ acc3 += float(input[in_idx]) * float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
}
}
}
@@ -366,13 +303,10 @@ void causal_conv_transpose1d_causal_pad(
// Lane 0
if (valid_count > 0) {
const int expanded_time0 = (int)t_out + k - left_pad;
- if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded &&
- (expanded_time0 % stride) == 0) {
+ if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded && (expanded_time0 % stride) == 0) {
const int src_time0 = expanded_time0 / stride;
if (src_time0 >= 0 && src_time0 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)src_time0) * (uint)cin +
- (uint)ic;
+ const uint in_idx = (b * (uint)seq_len_in + (uint)src_time0) * (uint)cin + (uint)ic;
acc0 += float(input[in_idx]) * w;
}
}
@@ -380,13 +314,10 @@ void causal_conv_transpose1d_causal_pad(
// Lane 1
if (valid_count > 1) {
const int expanded_time1 = (int)t_out + 1 + k - left_pad;
- if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded &&
- (expanded_time1 % stride) == 0) {
+ if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded && (expanded_time1 % stride) == 0) {
const int src_time1 = expanded_time1 / stride;
if (src_time1 >= 0 && src_time1 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)src_time1) * (uint)cin +
- (uint)ic;
+ const uint in_idx = (b * (uint)seq_len_in + (uint)src_time1) * (uint)cin + (uint)ic;
acc1 += float(input[in_idx]) * w;
}
}
@@ -394,13 +325,10 @@ void causal_conv_transpose1d_causal_pad(
// Lane 2
if (valid_count > 2) {
const int expanded_time2 = (int)t_out + 2 + k - left_pad;
- if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded &&
- (expanded_time2 % stride) == 0) {
+ if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded && (expanded_time2 % stride) == 0) {
const int src_time2 = expanded_time2 / stride;
if (src_time2 >= 0 && src_time2 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)src_time2) * (uint)cin +
- (uint)ic;
+ const uint in_idx = (b * (uint)seq_len_in + (uint)src_time2) * (uint)cin + (uint)ic;
acc2 += float(input[in_idx]) * w;
}
}
@@ -408,13 +336,10 @@ void causal_conv_transpose1d_causal_pad(
// Lane 3
if (valid_count > 3) {
const int expanded_time3 = (int)t_out + 3 + k - left_pad;
- if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded &&
- (expanded_time3 % stride) == 0) {
+ if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded && (expanded_time3 % stride) == 0) {
const int src_time3 = expanded_time3 / stride;
if (src_time3 >= 0 && src_time3 < seq_len_in) {
- const uint in_idx =
- (b * (uint)seq_len_in + (uint)src_time3) * (uint)cin +
- (uint)ic;
+ const uint in_idx = (b * (uint)seq_len_in + (uint)src_time3) * (uint)cin + (uint)ic;
acc3 += float(input[in_idx]) * w;
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_decode.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_decode.metal
index 95e3fb2db..486c4f9ea 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_decode.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_decode.metal
@@ -69,15 +69,5 @@ PUBLIC KERNEL(AudioFsqDecode)(
uint gd AXIS(num_groups * codebook_dim, 1),
uint b AXIS(batch_size, 1)
) {
- fsq_decode<T>(
- tokens,
- out,
- lengths,
- num_groups,
- seq_len,
- codebook_dim,
- num_levels,
- dim_base_index,
- uint3(t, gd, b)
- );
+ fsq_decode<T>(tokens, out, lengths, num_groups, seq_len, codebook_dim, num_levels, dim_base_index, uint3(t, gd, b));
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_encode.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_encode.metal
index 34320fcae..789bddeee 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_encode.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_encode.metal
@@ -44,10 +44,7 @@ void fsq_encode(
return;
}
- const uint in_base =
- (b * (uint)(num_groups * codebook_dim) + g * (uint)codebook_dim) *
- (uint)seq_len +
- t;
+ const uint in_base = (b * (uint)(num_groups * codebook_dim) + g * (uint)codebook_dim) * (uint)seq_len + t;
int token = 0;
for (int d = 0; d < codebook_dim; ++d) {
@@ -58,8 +55,7 @@ void fsq_encode(
const float input_shift = tan(output_offset / output_scale);
const float x = float(input[in_base + (uint)d * (uint)seq_len]);
- const float compressed =
- output_scale * tanh(x + input_shift) - output_offset;
+ const float compressed = output_scale * tanh(x + input_shift) - output_offset;
const float rounded = round_ties_to_even(compressed);
int code_nonneg = (int)rounded + scale_i;
@@ -87,16 +83,6 @@ PUBLIC KERNEL(AudioFsqEncode)(
uint g AXIS(num_groups, 1),
uint b AXIS(batch_size, 1)
) {
- fsq_encode<T>(
- input,
- tokens,
- lengths,
- num_groups,
- seq_len,
- codebook_dim,
- num_levels,
- dim_base_index,
- eps,
- uint3(t, g, b)
- );
+ fsq_encode<
+ T>(input, tokens, lengths, num_groups, seq_len, codebook_dim, num_levels, dim_base_index, eps, uint3(t, g, b));
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/half_snake.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/half_snake.metal
index 84e866842..5ac6454e3 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/half_snake.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/half_snake.metal
@@ -54,15 +54,5 @@ PUBLIC KERNEL(AudioHalfSnake)(
uint c AXIS(channels, 1),
uint b AXIS(batch_size, 1)
) {
- half_snake<T>(
- input,
- alpha,
- output,
- channels,
- seq_len,
- snake_channels,
- negative_slope,
- eps,
- uint3(t, c, b)
- );
+ half_snake<T>(input, alpha, output, channels, seq_len, snake_channels, negative_slope, eps, uint3(t, c, b));
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/norm_ncs.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/norm_ncs.metal
index 7650656d7..3baa8e414 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/norm_ncs.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/norm_ncs.metal
@@ -48,9 +48,11 @@ void norm_ncs(
}
}
- const float sum = threadgroup_cooperative_reduce<
- SimdReduceSum<float>,
- AUDIO_NORM_NCS_BLOCK_SIZE>(partial_sum, shared_mean, thread_context);
+ const float sum = threadgroup_cooperative_reduce<SimdReduceSum<float>, AUDIO_NORM_NCS_BLOCK_SIZE>(
+ partial_sum,
+ shared_mean,
+ thread_context
+ );
const float mean = (subtract_mean != 0) ? (sum / (float)channels) : 0.0f;
float partial_variance = 0.0f;
@@ -61,9 +63,7 @@ void norm_ncs(
partial_variance += centered * centered;
}
- const float variance_sum = threadgroup_cooperative_reduce<
- SimdReduceSum<float>,
- AUDIO_NORM_NCS_BLOCK_SIZE>(
+ const float variance_sum = threadgroup_cooperative_reduce<SimdReduceSum<float>, AUDIO_NORM_NCS_BLOCK_SIZE>(
partial_variance,
shared_variance,
thread_context
diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/quantizer_decode.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/quantizer_decode.metal
index ec147ca1d..78b0ebc11 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/audio/quantizer_decode.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/quantizer_decode.metal
@@ -5,16 +5,15 @@ using namespace metal;
template <typename T>
void quantizer_decode(
- device const uint* tokens, // [B, K, T]
- device const int* lengths, // [B]
- device const T* semantic_codebook, // [semantic_cardinality, codebook_dim]
- device const T* semantic_out_proj, // [input_dim, codebook_dim]
- device const T* semantic_out_bias, // [input_dim]
- device const T*
- residual_codebooks, // [R, residual_cardinality, codebook_dim]
- device const T* residual_out_proj, // [R, input_dim, codebook_dim]
- device const T* residual_out_bias, // [R, input_dim]
- device T* output, // [B, T, input_dim]
+ device const uint* tokens, // [B, K, T]
+ device const int* lengths, // [B]
+ device const T* semantic_codebook, // [semantic_cardinality, codebook_dim]
+ device const T* semantic_out_proj, // [input_dim, codebook_dim]
+ device const T* semantic_out_bias, // [input_dim]
+ device const T* residual_codebooks, // [R, residual_cardinality, codebook_dim]
+ device const T* residual_out_proj, // [R, input_dim, codebook_dim]
+ device const T* residual_out_bias, // [R, input_dim]
+ device T* output, // [B, T, input_dim]
const constant int& batch_size,
const constant int& total_codebooks,
const constant int& seq_len,
@@ -48,8 +47,7 @@ void quantizer_decode(
const uint semantic_code_base = (uint)semantic_token * (uint)codebook_dim;
const uint semantic_proj_base = d * (uint)codebook_dim;
for (uint k = 0; k < (uint)codebook_dim; ++k) {
- acc += float(semantic_out_proj[semantic_proj_base + k]) *
- float(semantic_codebook[semantic_code_base + k]);
+ acc += float(semantic_out_proj[semantic_proj_base + k]) * float(semantic_codebook[semantic_code_base + k]);
}
for (int r = 0; r < residual_quantizers; ++r) {
@@ -60,13 +58,10 @@ void quantizer_decode(
const uint bias_idx = (uint)r * (uint)input_dim + d;
acc += float(residual_out_bias[bias_idx]);
- const uint code_base =
- ((uint)r * (uint)residual_cardinality + (uint)residual_token) *
- (uint)codebook_dim;
+ const uint code_base = ((uint)r * (uint)residual_cardinality + (uint)residual_token) * (uint)codebook_dim;
const uint proj_base = ((uint)r * (uint)input_dim + d) * (uint)codebook_dim;
for (uint k = 0; k < (uint)codebook_dim; ++k) {
- acc += float(residual_out_proj[proj_base + k]) *
- float(residual_codebooks[code_base + k]);
+ acc += float(residual_out_proj[proj_base + k]) * float(residual_codebooks[code_base + k]);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h b/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h
index 11922d052..d61fe789d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h
@@ -13,8 +13,7 @@ using namespace metal;
#define DSL_STR(X) #X
#define DSL_XSTR(X) DSL_STR(X)
-#define VARIANTS(TYPENAME, ...) \
- DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)
+#define VARIANTS(TYPENAME, ...) DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)
#define CONSTRAINT(C) DSL_META("dsl.constraint", #C)
#define PUBLIC DSL_META("dsl.public")
#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h b/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h
index 4263ddeba..fa36461fc 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h
@@ -23,14 +23,11 @@ using false_type = bool_constant<false>;
template <int val>
using Int = integral_constant<int, val>;
-#define METAL_INTEGRAL_CONST_BINOP(op, fn) \
- template <typename T, T tv, typename U, U uv> \
- METAL_FUNC constexpr auto fn( \
- integral_constant<T, tv>, \
- integral_constant<U, uv> \
- ) { \
- constexpr auto res = tv op uv; \
- return integral_constant<decltype(res), res>{}; \
+#define METAL_INTEGRAL_CONST_BINOP(op, fn) \
+ template <typename T, T tv, typename U, U uv> \
+ METAL_FUNC constexpr auto fn(integral_constant<T, tv>, integral_constant<U, uv>) { \
+ constexpr auto res = tv op uv; \
+ return integral_constant<decltype(res), res>{}; \
}
METAL_INTEGRAL_CONST_BINOP(+, operator+)
@@ -41,10 +38,7 @@ METAL_INTEGRAL_CONST_BINOP(/, operator/)
#undef METAL_INTEGRAL_CONST_BINOP
template <int start, int step, typename F, int... Is>
-METAL_FUNC constexpr void const_for_loop_impl(
- F f,
- metal::integer_sequence<int, Is...>
-) {
+METAL_FUNC constexpr void const_for_loop_impl(F f, metal::integer_sequence<int, Is...>) {
(f(Int<start + Is * step>{}), ...);
}
@@ -52,10 +46,7 @@ template <int start, int stop, int step, typename F>
METAL_FUNC constexpr void const_for_loop(F f) {
static_assert(step > 0 && start <= stop);
constexpr int count = (stop - start + step - 1) / step;
- const_for_loop_impl<start, step>(
- f,
- metal::make_integer_sequence<int, count>{}
- );
+ const_for_loop_impl<start, step>(f, metal::make_integer_sequence<int, count>{});
}
template <typename F>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/tensor_view.h b/crates/backend-uzu/src/backends/metal/kernel/common/tensor_view.h
index 96030c488..76e67c935 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/common/tensor_view.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/tensor_view.h
@@ -40,16 +40,14 @@ struct TensorView3D {
TensorView3D(device T* ptr) : data(ptr) {}
- TensorView3D(device T* ptr, int d0, int d1, int d2)
- : data(ptr), dim0(d0), dim1(d1), dim2(d2) {
+ TensorView3D(device T* ptr, int d0, int d1, int d2) : data(ptr), dim0(d0), dim1(d1), dim2(d2) {
stride0 = d1 * d2;
stride1 = d2;
stride2 = 1;
}
TensorView3D(device T* ptr, int d0, int d1, int d2, int s0, int s1, int s2)
- : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1),
- stride2(s2) {}
+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1), stride2(s2) {}
thread TensorView3D& shaped(int d0, int d1, int d2) {
dim0 = d0;
@@ -61,9 +59,7 @@ struct TensorView3D {
return *this;
}
- device T& at(uint i, uint j, uint k) const {
- return data[i * stride0 + j * stride1 + k * stride2];
- }
+ device T& at(uint i, uint j, uint k) const { return data[i * stride0 + j * stride1 + k * stride2]; }
device T& operator()(uint i, uint j, uint k) const { return at(i, j, k); }
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h b/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h
index 4c0691786..b73d4048d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h
@@ -4,10 +4,10 @@
using namespace metal;
struct ThreadContext {
- uint simd_lane_id; // [[thread_index_in_simdgroup]] — 0..simdgroup_size
- uint simdgroup_index; // [[simdgroup_index_in_threadgroup]] —
- // 0..simdgroups_per_threadgroup
- uint simdgroup_size; // [[threads_per_simdgroup]] — typically 32
+ uint simd_lane_id; // [[thread_index_in_simdgroup]] — 0..simdgroup_size
+ uint simdgroup_index; // [[simdgroup_index_in_threadgroup]] —
+ // 0..simdgroups_per_threadgroup
+ uint simdgroup_size; // [[threads_per_simdgroup]] — typically 32
uint simdgroups_per_threadgroup; // [[simdgroups_per_threadgroup]]
uint3 threadgroup_position; // [[threadgroup_position_in_grid]]
uint3 threadgroup_size;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
index df37462bc..85756d118 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
@@ -15,18 +15,16 @@ struct SimdReduceSum {
template <typename T>
struct SimdReduceMax {
using value_type = T;
- static constant constexpr T identity = numeric_limits<T>::has_infinity
- ? -numeric_limits<T>::infinity()
- : numeric_limits<T>::lowest();
+ static constant constexpr T identity =
+ numeric_limits<T>::has_infinity ? -numeric_limits<T>::infinity() : numeric_limits<T>::lowest();
static T simd_reduce(T x) { return simd_max(x); }
};
template <typename T>
struct SimdReduceMin {
using value_type = T;
- static constant constexpr T identity = numeric_limits<T>::has_infinity
- ? numeric_limits<T>::infinity()
- : numeric_limits<T>::max();
+ static constant constexpr T identity =
+ numeric_limits<T>::has_infinity ? numeric_limits<T>::infinity() : numeric_limits<T>::max();
static T simd_reduce(T x) { return simd_min(x); }
};
@@ -47,9 +45,7 @@ static typename Op::value_type threadgroup_cooperative_reduce(
// Phase 2: first simdgroup reduces across simdgropus
if (thread_context.simdgroup_index == 0) {
typename Op::value_type total =
- thread_context.simd_lane_id <
- (BLOCK_SIZE + thread_context.simdgroup_size - 1) /
- thread_context.simdgroup_size
+ thread_context.simd_lane_id < (BLOCK_SIZE + thread_context.simdgroup_size - 1) / thread_context.simdgroup_size
? shared[thread_context.simd_lane_id]
: Op::identity;
total = Op::simd_reduce(total);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
index f8811bd0b..7933c475f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
@@ -42,18 +42,13 @@ PUBLIC KERNEL(DeltaNetNormGate)(
// RMSNorm: cooperative sum of squares
float o_sq = active ? o_i * o_i : 0.0f;
float o_sumsq =
- threadgroup_cooperative_reduce<SimdReduceSum<float>, HEAD_V_DIM>(
- o_sq,
- shared_scratch,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceSum<float>, HEAD_V_DIM>(o_sq, shared_scratch, thread_context);
float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);
// Apply norm + SiLU gate (in-place)
if (active) {
float nw = float(norm_weight[tid]);
- uint z_idx =
- token_idx * total_proj_dim + conv_dim + hv_idx * head_v_dim + tid;
+ uint z_idx = token_idx * total_proj_dim + conv_dim + hv_idx * head_v_dim + tid;
float z_silu = activate_silu(float(in_proj[z_idx]));
in_out[base + tid] = static_cast<T>(o_i * inv_rms * nw * z_silu);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal
index aebfcdfe6..822cf8918 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal
@@ -10,10 +10,7 @@ using namespace metal;
#define PREFILL_THREADS 128
#define DV_PER_SIMDGROUP 4
-static_assert(
- PREFILL_THREADS % METAL_SIMD_SIZE == 0,
- "PREFILL_THREADS must be a multiple of METAL_SIMD_SIZE"
-);
+static_assert(PREFILL_THREADS % METAL_SIMD_SIZE == 0, "PREFILL_THREADS must be a multiple of METAL_SIMD_SIZE");
template <typename T, uint HEAD_K_DIM>
VARIANTS(T, float, half, bfloat)
@@ -37,10 +34,7 @@ PUBLIC KERNEL(DeltaNetPrefill)(
const uint dv_group GROUPS(num_dv_groups),
const uint tid THREADS(PREFILL_THREADS)
) {
- static_assert(
- HEAD_K_DIM % METAL_SIMD_SIZE == 0,
- "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"
- );
+ static_assert(HEAD_K_DIM % METAL_SIMD_SIZE == 0, "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");
constexpr uint ELEMS = HEAD_K_DIM / METAL_SIMD_SIZE;
constexpr uint NUM_SG = PREFILL_THREADS / METAL_SIMD_SIZE;
static_assert(ELEMS == 4, "float4 prefill requires ELEMS == 4");
@@ -66,8 +60,7 @@ PUBLIC KERNEL(DeltaNetPrefill)(
// State layout: [Hv, Dv, Dk], contiguous along Dk (HEAD_K_DIM floats/row).
// Unrolled by hand: METAL_PRAGMA_UNROLL is ignored under metal4.0 -O2, so a
// loop over s[i] keeps the array in an alloca and reloads it every token.
- device const float* s_in =
- state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;
+ device const float* s_in = state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;
float4 s[DV_PER_SIMDGROUP];
s[0] = *reinterpret_cast<device const float4*>(s_in + 0 * HEAD_K_DIM);
s[1] = *reinterpret_cast<device const float4*>(s_in + 1 * HEAD_K_DIM);
@@ -84,16 +77,10 @@ PUBLIC KERNEL(DeltaNetPrefill)(
s[1] *= decay;
s[2] *= decay;
s[3] *= decay;
- float4 kv_mem =
- float4(dot(s[0], k), dot(s[1], k), dot(s[2], k), dot(s[3], k));
+ float4 kv_mem = float4(dot(s[0], k), dot(s[1], k), dot(s[2], k), dot(s[3], k));
kv_mem = simd_sum(kv_mem);
- float4 v_val = float4(
- float(v_row[0]),
- float(v_row[1]),
- float(v_row[2]),
- float(v_row[3])
- );
+ float4 v_val = float4(float(v_row[0]), float(v_row[1]), float(v_row[2]), float(v_row[3]));
float4 delta = beta * (v_val - kv_mem);
s[0] += k * delta[0];
s[1] += k * delta[1];
@@ -117,8 +104,7 @@ PUBLIC KERNEL(DeltaNetPrefill)(
out_row += value_dim;
}
- device float* s_out =
- state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;
+ device float* s_out = state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;
*reinterpret_cast<device float4*>(s_out + 0 * HEAD_K_DIM) = s[0];
*reinterpret_cast<device float4*>(s_out + 1 * HEAD_K_DIM) = s[1];
*reinterpret_cast<device float4*>(s_out + 2 * HEAD_K_DIM) = s[2];
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
index 937d73719..f78cf62e8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
@@ -29,10 +29,7 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(
const uint hk_idx GROUPS(num_k_heads),
const uint lane THREADS(METAL_SIMD_SIZE)
) {
- static_assert(
- HEAD_K_DIM % METAL_SIMD_SIZE == 0,
- "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"
- );
+ static_assert(HEAD_K_DIM % METAL_SIMD_SIZE == 0, "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");
constexpr uint elems_per_thread = HEAD_K_DIM / METAL_SIMD_SIZE;
const uint conv_dim = 2 * key_dim + value_dim;
@@ -86,8 +83,7 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(
float beta_raw = float(in_proj[tok_offset + conv_dim + value_dim + hv]);
float beta = 1.0f / (1.0f + fast::exp(-beta_raw));
- float a_raw =
- float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);
+ float a_raw = float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);
float sp = activate_softplus(a_raw + float(dt_bias[hv]));
float decay = fast::exp(-fast::exp(float(a_log[hv])) * sp);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal
index 9a6dc76bf..9790551f8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal
@@ -35,14 +35,9 @@ PUBLIC KERNEL(DeltaNetUpdate)(
const uint hv_idx GROUPS(num_v_heads),
const uint tid THREADS(UPDATE_THREADS)
) {
- static_assert(
- HEAD_K_DIM % METAL_SIMD_SIZE == 0,
- "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"
- );
- constexpr uint ELEMS =
- HEAD_K_DIM / METAL_SIMD_SIZE; // Dk per lane (128/32 = 4)
- constexpr uint NUM_SG =
- UPDATE_THREADS / METAL_SIMD_SIZE; // simd groups / tg (16)
+ static_assert(HEAD_K_DIM % METAL_SIMD_SIZE == 0, "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");
+ constexpr uint ELEMS = HEAD_K_DIM / METAL_SIMD_SIZE; // Dk per lane (128/32 = 4)
+ constexpr uint NUM_SG = UPDATE_THREADS / METAL_SIMD_SIZE; // simd groups / tg (16)
const uint lane = tid % METAL_SIMD_SIZE;
const uint sg = tid / METAL_SIMD_SIZE;
@@ -78,8 +73,7 @@ PUBLIC KERNEL(DeltaNetUpdate)(
// beta / decay (scalar per head)
const float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);
const float beta = 1.0f / (1.0f + fast::exp(-beta_raw));
- const float a_raw =
- float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);
+ const float a_raw = float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);
const float sp = activate_softplus(a_raw + float(dt_bias[hv_idx]));
const float decay = fast::exp(-fast::exp(float(a_log[hv_idx])) * sp);
@@ -122,11 +116,7 @@ PUBLIC KERNEL(DeltaNetUpdate)(
o_sq += o * o;
}
const float o_sumsq =
- threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(
- o_sq,
- shared_scratch,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(o_sq, shared_scratch, thread_context);
const float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);
for (uint dv = sg; dv < head_v_dim; dv += NUM_SG) {
@@ -134,8 +124,7 @@ PUBLIC KERNEL(DeltaNetUpdate)(
const float nw = float(norm_weight[dv]);
const float z_i = float(in_proj[conv_dim + hv_idx * head_v_dim + dv]);
const float z_silu = activate_silu(z_i);
- out[hv_idx * head_v_dim + dv] =
- static_cast<T>(shared_o[dv] * inv_rms * nw * z_silu);
+ out[hv_idx * head_v_dim + dv] = static_cast<T>(shared_o[dv] * inv_rms * nw * z_silu);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index b7521d1a2..5a885f0ce 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -37,14 +37,11 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
const uint group_idx = dim_idx / group_size;
const uint num_groups = (model_dim + group_size - 1) / group_size;
- const uint zp_stride = quantization_mode == QuantizationMode::U4
- ? (num_groups + 1) / 2
- : num_groups;
+ const uint zp_stride = quantization_mode == QuantizationMode::U4 ? (num_groups + 1) / 2 : num_groups;
const T scale = scales[token_id * num_groups + group_idx];
- const uint packing_divisor =
- quantization_mode == QuantizationMode::U4 ? 2 : 1;
+ const uint packing_divisor = quantization_mode == QuantizationMode::U4 ? 2 : 1;
const uint weights_stride = model_dim / packing_divisor;
int quantized_value = 0;
@@ -61,8 +58,7 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
};
case QuantizationMode::I8: {
const uint elem_idx = token_id * weights_stride + dim_idx;
- const device int8_t* weights_i8 =
- reinterpret_cast<const device int8_t*>(weights);
+ const device int8_t* weights_i8 = reinterpret_cast<const device int8_t*>(weights);
quantized_value = int(weights_i8[elem_idx]);
break;
};
@@ -80,8 +76,7 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
uint zero_point = 0;
if (quantization_mode == QuantizationMode::U4) {
const uint8_t packed = zero_points[token_id * zp_stride + group_idx / 2];
- zero_point =
- (group_idx & 1) == 0 ? (packed & 0x0F) : ((packed >> 4) & 0x0F);
+ zero_point = (group_idx & 1) == 0 ? (packed & 0x0F) : ((packed >> 4) & 0x0F);
} else {
zero_point = zero_points[token_id * zp_stride + group_idx];
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal b/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
index 8ba468d2d..ef6ca3413 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
@@ -29,10 +29,8 @@ PUBLIC KERNEL(GatedActMul) (
value = act_operand[base + gated_idx];
activated = activate(act_operand[base + gated_dim + gated_idx], act_type);
} else {
- activated =
- activate(act_operand[batch_idx * gated_dim + gated_idx], act_type);
- value =
- value_operand[batch_idx * value_row_stride + value_offset + gated_idx];
+ activated = activate(act_operand[batch_idx * gated_dim + gated_idx], act_type);
+ value = value_operand[batch_idx * value_row_stride + value_offset + gated_idx];
}
T result = value * activated;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
index a2e90ab48..05d191ae7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
@@ -20,12 +20,8 @@ struct GemmDTransform {
static constant constexpr uint ACCUMULATE = 1 << 1;
static constant constexpr uint BIAS = 1 << 2;
static constant constexpr uint RHT = 1 << 3;
- constexpr bool contains(uint flag) const thread {
- return (raw_value & flag) != 0;
- }
- constexpr bool contains(uint flag) const constant {
- return (raw_value & flag) != 0;
- }
+ constexpr bool contains(uint flag) const thread { return (raw_value & flag) != 0; }
+ constexpr bool contains(uint flag) const constant { return (raw_value & flag) != 0; }
constexpr uint bits() const thread { return raw_value; }
constexpr uint bits() const constant { return raw_value; }
};
@@ -49,12 +45,8 @@ struct GemmAlignment {
static constant constexpr uint M = 1 << 0;
static constant constexpr uint N = 1 << 1;
static constant constexpr uint K = 1 << 2;
- constexpr bool contains(uint flag) const thread {
- return (raw_value & flag) != 0;
- }
- constexpr bool contains(uint flag) const constant {
- return (raw_value & flag) != 0;
- }
+ constexpr bool contains(uint flag) const thread { return (raw_value & flag) != 0; }
+ constexpr bool contains(uint flag) const constant { return (raw_value & flag) != 0; }
constexpr uint bits() const thread { return raw_value; }
constexpr uint bits() const constant { return raw_value; }
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
index 10bc5ce2f..810c91953 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
@@ -6,39 +6,21 @@
using namespace metal;
-static METAL_FUNC float simdgroup_hadamard_transform(
- ushort lane_index,
- float lane_value
-) {
+static METAL_FUNC float simdgroup_hadamard_transform(ushort lane_index, float lane_value) {
for (ushort stride = 1; stride < METAL_SIMD_SIZE; stride <<= 1) {
float partner_lane_value = simd_shuffle_xor(lane_value, stride);
- lane_value = (lane_index & stride) ? (partner_lane_value - lane_value)
- : (partner_lane_value + lane_value);
+ lane_value = (lane_index & stride) ? (partner_lane_value - lane_value) : (partner_lane_value + lane_value);
}
return lane_value / sqrt((float)METAL_SIMD_SIZE);
}
template <typename T>
-static METAL_FUNC T simdgroup_input_random_hadamard_transform(
- ushort lane_index,
- T lane_value,
- int32_t lane_factor
-) {
- return T(simdgroup_hadamard_transform(
- lane_index,
- float(lane_value) * float(lane_factor)
- ));
+static METAL_FUNC T simdgroup_input_random_hadamard_transform(ushort lane_index, T lane_value, int32_t lane_factor) {
+ return T(simdgroup_hadamard_transform(lane_index, float(lane_value) * float(lane_factor)));
}
template <typename T>
-static METAL_FUNC T simdgroup_output_random_hadamard_transform(
- ushort lane_index,
- T lane_value,
- int32_t lane_factor
-) {
- return T(
- simdgroup_hadamard_transform(lane_index, float(lane_value)) *
- float(lane_factor)
- );
+static METAL_FUNC T simdgroup_output_random_hadamard_transform(ushort lane_index, T lane_value, int32_t lane_factor) {
+ return T(simdgroup_hadamard_transform(lane_index, float(lane_value)) * float(lane_factor));
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
index 6678f8060..f09eb4b60 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
@@ -23,16 +23,10 @@ PUBLIC KERNEL(HadamardTransform)(
uint element_index = batch_index * hidden_dim + factor_index;
if (transform_order == HadamardTransformOrder::Input) {
- data[element_index] = simdgroup_input_random_hadamard_transform(
- lane_index,
- data[element_index],
- factors[factor_index]
- );
+ data[element_index] =
+ simdgroup_input_random_hadamard_transform(lane_index, data[element_index], factors[factor_index]);
} else {
- data[element_index] = simdgroup_output_random_hadamard_transform(
- lane_index,
- data[element_index],
- factors[factor_index]
- );
+ data[element_index] =
+ simdgroup_output_random_hadamard_transform(lane_index, data[element_index], factors[factor_index]);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/backend-uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
index 9293e927a..76582bca9 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
@@ -27,10 +27,8 @@ PUBLIC KERNEL(KVCacheUpdate) (
// Offset = token_idx * num_heads * head_dim +
// head_idx * head_dim + channel_idx
const uint head_offset = head_idx * head_dim;
- const uint sourceIdx =
- swaps[i].source * num_heads * head_dim + head_offset + channel_idx;
- const uint destIdx =
- swaps[i].destination * num_heads * head_dim + head_offset + channel_idx;
+ const uint sourceIdx = swaps[i].source * num_heads * head_dim + head_offset + channel_idx;
+ const uint destIdx = swaps[i].destination * num_heads * head_dim + head_offset + channel_idx;
swap(in_place_keys, sourceIdx, destIdx);
swap(in_place_values, sourceIdx, destIdx);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/backend-uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
index 92c946f22..902c624c2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
@@ -25,8 +25,7 @@ void layer_norm_core(
AccumT partial_sum = static_cast<AccumT>(0.0f);
// Compute mean: sum all elements
- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
- base_i += BLOCK_SIZE * GRAIN_SIZE) {
+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count; base_i += BLOCK_SIZE * GRAIN_SIZE) {
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
if (i < element_count) {
@@ -37,17 +36,12 @@ void layer_norm_core(
// Reduce to get total mean
AccumT total_sum =
- threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(
- partial_sum,
- shared_mean,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(partial_sum, shared_mean, thread_context);
AccumT mean = total_sum / static_cast<AccumT>(element_count);
// Compute variance: sum of squared deviations
AccumT partial_var_sum = static_cast<AccumT>(0.0f);
- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
- base_i += BLOCK_SIZE * GRAIN_SIZE) {
+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count; base_i += BLOCK_SIZE * GRAIN_SIZE) {
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
if (i < element_count) {
@@ -58,27 +52,23 @@ void layer_norm_core(
}
// Reduce to get total variance
- AccumT total_var =
- threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(
- partial_var_sum,
- shared_variance,
- thread_context
- );
+ AccumT total_var = threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(
+ partial_var_sum,
+ shared_variance,
+ thread_context
+ );
AccumT variance = total_var / static_cast<AccumT>(element_count);
AccumT inv_std = rsqrt(variance + static_cast<AccumT>(epsilon));
// Apply normalization and scaling
- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
- base_i += BLOCK_SIZE * GRAIN_SIZE) {
+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count; base_i += BLOCK_SIZE * GRAIN_SIZE) {
AccumT vals[GRAIN_SIZE];
AccumT scaled_vals[GRAIN_SIZE];
// Load and center
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
- vals[j] = (i < element_count)
- ? (static_cast<AccumT>(input_data[i]) - mean)
- : 0.0f;
+ vals[j] = (i < element_count) ? (static_cast<AccumT>(input_data[i]) - mean) : 0.0f;
}
// Normalize and scale
@@ -91,15 +81,12 @@ void layer_norm_core(
AccumT normalized_high = vals[j] * inv_std;
if (full_layer) {
- AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +
- static_cast<AccumT>(scale_offset);
+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset);
scaled_vals[j] = normalized_high * scale_value_high;
} else {
OutputT normalized_low = static_cast<OutputT>(normalized_high);
- OutputT scale_value_low = static_cast<OutputT>(
- static_cast<AccumT>(scales_data[i]) +
- static_cast<AccumT>(scale_offset)
- );
+ OutputT scale_value_low =
+ static_cast<OutputT>(static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset));
OutputT product_low = normalized_low * scale_value_low;
scaled_vals[j] = static_cast<AccumT>(product_low);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h
index 5781cdf8c..93691392c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h
@@ -35,12 +35,8 @@ template <typename OutputType, typename InputType>
struct TransformNone {
TransformNone(float = 1.0f, float = 0.0f) {}
- static METAL_FUNC OutputType apply(InputType x) {
- return static_cast<OutputType>(x);
- }
- METAL_FUNC OutputType apply(InputType x, OutputType) const {
- return static_cast<OutputType>(x);
- }
+ static METAL_FUNC OutputType apply(InputType x) { return static_cast<OutputType>(x); }
+ METAL_FUNC OutputType apply(InputType x, OutputType) const { return static_cast<OutputType>(x); }
};
template <typename OutputType, typename InputType>
@@ -48,18 +44,14 @@ struct TransformScaleAccumulate {
const float alpha;
const float beta;
- TransformScaleAccumulate(float alpha, float beta)
- : alpha(alpha), beta(beta) {}
+ TransformScaleAccumulate(float alpha, float beta) : alpha(alpha), beta(beta) {}
- static METAL_FUNC OutputType apply(InputType x) {
- return static_cast<OutputType>(x);
- }
+ static METAL_FUNC OutputType apply(InputType x) { return static_cast<OutputType>(x); }
METAL_FUNC OutputType apply(InputType x, OutputType c) const {
return static_cast<OutputType>(
x * static_cast<InputType>(alpha) +
- (beta != 0.0f ? (static_cast<OutputType>(beta) * c)
- : static_cast<OutputType>(0.0))
+ (beta != 0.0f ? (static_cast<OutputType>(beta) * c) : static_cast<OutputType>(0.0))
);
}
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
index 3f2973d06..6c3f08682 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
@@ -28,8 +28,7 @@ struct Fragment {
"Ops must expose positive FRAGMENT_ROWS/FRAGMENT_COLS"
);
static_assert(
- Ops::ELEMENTS_PER_THREAD ==
- (Ops::FRAGMENT_ROWS * Ops::FRAGMENT_COLS) / METAL_SIMD_SIZE,
+ Ops::ELEMENTS_PER_THREAD == (Ops::FRAGMENT_ROWS * Ops::FRAGMENT_COLS) / METAL_SIMD_SIZE,
"Ops::ELEMENTS_PER_THREAD must equal "
"(FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE"
);
@@ -51,26 +50,20 @@ struct Fragment {
ThreadVectorType fragment_data[NUM_FRAGS];
ThreadContext thread_context;
- METAL_FUNC Fragment(const thread ThreadContext& thread_context) thread
- : thread_context(thread_context) {}
+ METAL_FUNC Fragment(const thread ThreadContext& thread_context) thread : thread_context(thread_context) {}
// Lane origin (row, col) within a single FRAGMENT_ROWS x FRAGMENT_COLS
// sub-tile. The mapping depends only on Ops::THREAD_ELEMENT_COLS, so it is
// identical for every Fragment sharing that Ops type.
- METAL_FUNC static constexpr short2 get_position(
- const thread ThreadContext& thread_context
- ) {
+ METAL_FUNC static constexpr short2 get_position(const thread ThreadContext& thread_context) {
const ushort simd_lane_id = ushort(thread_context.simd_lane_id);
const short quad = simd_lane_id / 4;
const short row = (quad & 4) + (simd_lane_id / 2) % 4;
- const short col =
- ((quad & 2) + simd_lane_id % 2) * Ops::THREAD_ELEMENT_COLS;
+ const short col = ((quad & 2) + simd_lane_id % 2) * Ops::THREAD_ELEMENT_COLS;
return short2{col, row};
}
- METAL_FUNC short2 get_position() const thread {
- return get_position(thread_context);
- }
+ METAL_FUNC short2 get_position() const thread { return get_position(thread_context); }
METAL_FUNC constexpr void clear() {
METAL_PRAGMA_UNROLL
@@ -79,10 +72,7 @@ struct Fragment {
}
}
- METAL_FUNC constexpr thread ThreadVectorType& fragment_at(
- const ushort row_index,
- const ushort col_index
- ) {
+ METAL_FUNC constexpr thread ThreadVectorType& fragment_at(const ushort row_index, const ushort col_index) {
return fragment_data[row_index * TILE_COLS + col_index];
}
@@ -99,9 +89,7 @@ struct Fragment {
}
}
- METAL_FUNC thread ElementType* elements() {
- return reinterpret_cast<thread ElementType*>(fragment_data);
- }
+ METAL_FUNC thread ElementType* elements() { return reinterpret_cast<thread ElementType*>(fragment_data); }
// Unsafe load: copy a (TILE_ROWS * FRAGMENT_ROWS) x (TILE_COLS *
// FRAGMENT_COLS) block from device memory into fragment registers. ColStride
@@ -121,22 +109,11 @@ struct Fragment {
TileRowStride tile_row_stride = {},
TileColStride tile_col_stride = {}
) thread {
- transfer<LOAD, UNSAFE>(
- source,
- row_stride,
- col_stride,
- Int<0>{},
- Int<0>{},
- tile_row_stride,
- tile_col_stride
- );
+ transfer<LOAD, UNSAFE>(source, row_stride, col_stride, Int<0>{}, Int<0>{}, tile_row_stride, tile_col_stride);
}
// Safe load: col_stride is implicitly 1; out-of-bounds elements become T(0).
- template <
- class Ptr,
- class TileRowStride = Int<1>,
- class TileColStride = Int<1>>
+ template <class Ptr, class TileRowStride = Int<1>, class TileColStride = Int<1>>
METAL_FUNC void load_safe(
Ptr source,
const int leading_dimension,
@@ -169,22 +146,11 @@ struct Fragment {
TileRowStride tile_row_stride = {},
TileColStride tile_col_stride = {}
) thread {
- transfer<STORE, UNSAFE>(
- destination,
- row_stride,
- col_stride,
- Int<0>{},
- Int<0>{},
- tile_row_stride,
- tile_col_stride
- );
+ transfer<STORE, UNSAFE>(destination, row_stride, col_stride, Int<0>{}, Int<0>{}, tile_row_stride, tile_col_stride);
}
// Safe store: col_stride is implicitly 1; out-of-bounds elements are skipped.
- template <
- class Ptr,
- class TileRowStride = Int<1>,
- class TileColStride = Int<1>>
+ template <class Ptr, class TileRowStride = Int<1>, class TileColStride = Int<1>>
METAL_FUNC void store_safe(
Ptr destination,
const int leading_dimension,
@@ -212,9 +178,7 @@ private:
template <class Fn>
METAL_FUNC void for_each_fragment(Fn fn) thread {
const_for_loop<0, TILE_ROWS, 1>([&](auto row_index) {
- const_for_loop<0, TILE_COLS, 1>([&](auto col_index) {
- fn(row_index, col_index);
- });
+ const_for_loop<0, TILE_COLS, 1>([&](auto col_index) { fn(row_index, col_index); });
});
}
@@ -266,16 +230,12 @@ private:
for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; j++) {
const ushort element_index = i * Ops::THREAD_ELEMENT_COLS + j;
const auto col = col_base + j;
- const auto offset = col_stride_is_one
- ? (row * row_stride + col)
- : (row * row_stride + col * col_stride);
+ const auto offset = col_stride_is_one ? (row * row_stride + col) : (row * row_stride + col * col_stride);
if constexpr (IS_LOAD) {
if constexpr (IS_SAFE) {
- const bool in_bounds =
- (row < local_row_limit) && (col < local_col_limit);
- frag[element_index] =
- in_bounds ? static_cast<T>(ptr[offset]) : T(0);
+ const bool in_bounds = (row < local_row_limit) && (col < local_col_limit);
+ frag[element_index] = in_bounds ? static_cast<T>(ptr[offset]) : T(0);
} else {
frag[element_index] = static_cast<T>(ptr[offset]);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
index a1a2f1a9b..abf7c3c6a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
@@ -20,13 +20,11 @@ template <
ushort REDUCTION_DIMENSION,
ushort THREADGROUP_SIZE,
ushort ALIGNMENT = 1,
- ushort READS_PER_THREAD =
- (THREADGROUP_TILE_COLS * THREADGROUP_TILE_ROWS) / (THREADGROUP_SIZE),
+ ushort READS_PER_THREAD = (THREADGROUP_TILE_COLS * THREADGROUP_TILE_ROWS) / (THREADGROUP_SIZE),
ushort THREAD_COLS = THREADGROUP_TILE_COLS / READS_PER_THREAD,
ushort THREAD_ROWS = THREADGROUP_SIZE / THREAD_COLS>
struct ThreadgroupLoader {
- METAL_CONST ushort ROW_ITERATIONS =
- (THREADGROUP_TILE_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;
+ METAL_CONST ushort ROW_ITERATIONS = (THREADGROUP_TILE_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;
const int source_leading_dimension;
const int tile_stride;
@@ -49,23 +47,11 @@ struct ThreadgroupLoader {
const thread ThreadContext& thread_context
)
: source_leading_dimension(source_leading_dim),
- tile_stride(
- REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS
- : THREADGROUP_TILE_ROWS * source_leading_dim
- ),
- thread_index(
- thread_context.simdgroup_index * METAL_SIMD_SIZE +
- thread_context.simd_lane_id
- ),
- tile_row_index(thread_index / THREAD_COLS),
- tile_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),
- destination(
- destination_ptr + tile_row_index * DESTINATION_LEADING_DIMENSION +
- tile_col_index
- ),
- source(
- source_ptr + tile_row_index * source_leading_dim + tile_col_index
- ) {}
+ tile_stride(REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS : THREADGROUP_TILE_ROWS * source_leading_dim),
+ thread_index(thread_context.simdgroup_index * METAL_SIMD_SIZE + thread_context.simd_lane_id),
+ tile_row_index(thread_index / THREAD_COLS), tile_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),
+ destination(destination_ptr + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index),
+ source(source_ptr + tile_row_index * source_leading_dim + tile_col_index) {}
template <typename UnaryOp>
METAL_FUNC void apply_inplace_op(thread const UnaryOp& operation) const {
@@ -82,18 +68,13 @@ struct ThreadgroupLoader {
METAL_FUNC void load_unsafe() const {
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {
- *reinterpret_cast<threadgroup ReadVector*>(
- &destination[i * DESTINATION_LEADING_DIMENSION]
- ) =
- *reinterpret_cast<const device ReadVector*>(
- &source[i * source_leading_dimension]
- );
+ *reinterpret_cast<threadgroup ReadVector*>(&destination[i * DESTINATION_LEADING_DIMENSION]) =
+ *reinterpret_cast<const device ReadVector*>(&source[i * source_leading_dimension]);
}
}
METAL_FUNC void load_safe(short2 source_tile_dimensions) const {
- source_tile_dimensions =
- source_tile_dimensions - short2(tile_col_index, tile_row_index);
+ source_tile_dimensions = source_tile_dimensions - short2(tile_col_index, tile_row_index);
if (source_tile_dimensions.x <= 0 || source_tile_dimensions.y <= 0) {
METAL_PRAGMA_UNROLL
@@ -113,14 +94,12 @@ struct ThreadgroupLoader {
for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {
METAL_PRAGMA_UNROLL
for (ushort j = 0; j < READS_PER_THREAD; j++) {
- valid_mask[j] =
- (i < source_tile_dimensions.y) && (j < source_tile_dimensions.x);
+ valid_mask[j] = (i < source_tile_dimensions.y) && (j < source_tile_dimensions.x);
}
METAL_PRAGMA_UNROLL
for (ushort j = 0; j < READS_PER_THREAD; j++) {
- loaded_values[j] =
- source[(valid_mask[j] ? i * source_leading_dimension + j : 0)];
+ loaded_values[j] = source[(valid_mask[j] ? i * source_leading_dimension + j : 0)];
}
METAL_PRAGMA_UNROLL
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
index 9b31e884a..cce0638d8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
@@ -21,8 +21,7 @@ struct MxuFragmentOps {
METAL_CONST ushort FRAGMENT_ROWS = 16;
METAL_CONST ushort FRAGMENT_COLS = 16;
- METAL_CONST ushort ELEMENTS_PER_THREAD =
- (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
+ METAL_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
METAL_CONST ushort THREAD_ELEMENT_ROWS = 2;
METAL_CONST ushort THREAD_ELEMENT_COLS = 4;
@@ -41,13 +40,7 @@ struct MxuFragmentOps {
// supplies a `marshal_inputs` callable that copies its fragments into the
// cooperative left/right tensors; the descriptor and output marshalling are
// shared between all call shapes.
- template <
- typename CType,
- typename AType,
- typename BType,
- bool transpose_a,
- bool transpose_b,
- typename MarshalInputs>
+ template <typename CType, typename AType, typename BType, bool transpose_a, bool transpose_b, typename MarshalInputs>
METAL_FUNC static void mma_impl(
thread ThreadVector<CType>& output_0,
thread ThreadVector<CType>& output_1,
@@ -65,17 +58,12 @@ struct MxuFragmentOps {
mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;
- auto cooperative_left =
- matmul_op
- .template get_left_input_cooperative_tensor<AType, BType, CType>();
- auto cooperative_right =
- matmul_op
- .template get_right_input_cooperative_tensor<AType, BType, CType>();
- auto cooperative_output =
- matmul_op.template get_destination_cooperative_tensor<
- decltype(cooperative_left),
- decltype(cooperative_right),
- CType>();
+ auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<AType, BType, CType>();
+ auto cooperative_right = matmul_op.template get_right_input_cooperative_tensor<AType, BType, CType>();
+ auto cooperative_output = matmul_op.template get_destination_cooperative_tensor<
+ decltype(cooperative_left),
+ decltype(cooperative_right),
+ CType>();
marshal_inputs(cooperative_left, cooperative_right);
@@ -95,12 +83,7 @@ struct MxuFragmentOps {
}
// N-paired: output is two columns, left is one fragment, right is two.
- template <
- typename CType,
- typename AType,
- typename BType,
- bool transpose_a = false,
- bool transpose_b = false>
+ template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>
METAL_FUNC static void mma(
thread ThreadVector<CType>& output_col_0,
thread ThreadVector<CType>& output_col_1,
@@ -125,12 +108,7 @@ struct MxuFragmentOps {
}
// M-paired: output is two rows, left is two fragments, right is one.
- template <
- typename CType,
- typename AType,
- typename BType,
- bool transpose_a = false,
- bool transpose_b = false>
+ template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>
METAL_FUNC static void mma(
thread ThreadVector<CType>& output_row_0,
thread ThreadVector<CType>& output_row_1,
@@ -154,41 +132,19 @@ struct MxuFragmentOps {
);
}
- template <
- bool transpose_a,
- bool transpose_b,
- class OutputTile,
- class LeftTile,
- class RightTile>
- METAL_FUNC static void tile_matmul(
- thread OutputTile& output,
- thread LeftTile& left,
- thread RightTile& right
- ) {
- constexpr ushort left_tile_m =
- transpose_a ? LeftTile::TILE_COLS : LeftTile::TILE_ROWS;
+ template <bool transpose_a, bool transpose_b, class OutputTile, class LeftTile, class RightTile>
+ METAL_FUNC static void tile_matmul(thread OutputTile& output, thread LeftTile& left, thread RightTile& right) {
+ constexpr ushort left_tile_m = transpose_a ? LeftTile::TILE_COLS : LeftTile::TILE_ROWS;
constexpr ushort tile_m = OutputTile::TILE_ROWS;
- static_assert(
- left_tile_m == tile_m,
- "tile matmul: M dimensions do not match"
- );
+ static_assert(left_tile_m == tile_m, "tile matmul: M dimensions do not match");
- constexpr ushort right_tile_n =
- transpose_b ? RightTile::TILE_ROWS : RightTile::TILE_COLS;
+ constexpr ushort right_tile_n = transpose_b ? RightTile::TILE_ROWS : RightTile::TILE_COLS;
constexpr ushort tile_n = OutputTile::TILE_COLS;
- static_assert(
- right_tile_n == tile_n,
- "tile matmul: N dimensions do not match"
- );
+ static_assert(right_tile_n == tile_n, "tile matmul: N dimensions do not match");
- constexpr ushort left_tile_k =
- transpose_a ? LeftTile::TILE_ROWS : LeftTile::TILE_COLS;
- constexpr ushort tile_k =
- transpose_b ? RightTile::TILE_COLS : RightTile::TILE_ROWS;
- static_assert(
- left_tile_k == tile_k,
- "tile matmul: K dimensions do not match"
- );
+ constexpr ushort left_tile_k = transpose_a ? LeftTile::TILE_ROWS : LeftTile::TILE_COLS;
+ constexpr ushort tile_k = transpose_b ? RightTile::TILE_COLS : RightTile::TILE_ROWS;
+ static_assert(left_tile_k == tile_k, "tile matmul: K dimensions do not match");
constexpr auto transpose_left = metal::bool_constant<transpose_a>{};
constexpr auto transpose_right = metal::bool_constant<transpose_b>{};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
index a62acb567..4fd31c489 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
@@ -43,9 +43,7 @@ METAL_FUNC auto gemm_loop(
constexpr ushort RIGHT_TILE_ROWS = transpose_b ? TILES_N : TILES_K;
constexpr ushort RIGHT_TILE_COLS = transpose_b ? TILES_K : TILES_N;
- Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator(
- thread_context
- );
+ Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator(thread_context);
accumulator.clear();
METAL_PRAGMA_NO_UNROLL
@@ -54,53 +52,31 @@ METAL_FUNC auto gemm_loop(
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < BLOCK_K; inner_k += SIMDGROUP_BLOCK_K) {
- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(
- thread_context
- );
- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(
- thread_context
- );
+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(thread_context);
+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(thread_context);
volatile int mxu_iteration_fence;
- const int left_offset =
- transpose_a ? inner_k * leading_dimension_a : inner_k;
- const int right_offset =
- transpose_b ? inner_k : inner_k * leading_dimension_b;
+ const int left_offset = transpose_a ? inner_k * leading_dimension_a : inner_k;
+ const int right_offset = transpose_b ? inner_k : inner_k * leading_dimension_b;
if constexpr (aligned_m) {
left_tile.load(left_ptr + left_offset, leading_dimension_a);
} else {
- const short row_limit =
- transpose_a ? SIMDGROUP_BLOCK_K : simdgroup_limit_m;
- const short col_limit =
- transpose_a ? simdgroup_limit_m : SIMDGROUP_BLOCK_K;
- left_tile.load_safe(
- left_ptr + left_offset,
- leading_dimension_a,
- short2(col_limit, row_limit)
- );
+ const short row_limit = transpose_a ? SIMDGROUP_BLOCK_K : simdgroup_limit_m;
+ const short col_limit = transpose_a ? simdgroup_limit_m : SIMDGROUP_BLOCK_K;
+ left_tile.load_safe(left_ptr + left_offset, leading_dimension_a, short2(col_limit, row_limit));
}
if constexpr (aligned_n) {
right_tile.load(right_ptr + right_offset, leading_dimension_b);
} else {
- const short row_limit =
- transpose_b ? simdgroup_limit_n : SIMDGROUP_BLOCK_K;
- const short col_limit =
- transpose_b ? SIMDGROUP_BLOCK_K : simdgroup_limit_n;
- right_tile.load_safe(
- right_ptr + right_offset,
- leading_dimension_b,
- short2(col_limit, row_limit)
- );
+ const short row_limit = transpose_b ? simdgroup_limit_n : SIMDGROUP_BLOCK_K;
+ const short col_limit = transpose_b ? SIMDGROUP_BLOCK_K : simdgroup_limit_n;
+ right_tile.load_safe(right_ptr + right_offset, leading_dimension_b, short2(col_limit, row_limit));
}
- MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(
- accumulator,
- left_tile,
- right_tile
- );
+ MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(accumulator, left_tile, right_tile);
(void)mxu_iteration_fence;
}
@@ -116,40 +92,21 @@ METAL_FUNC auto gemm_loop(
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < remaining_k; inner_k += SIMDGROUP_BLOCK_K) {
- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(
- thread_context
- );
- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(
- thread_context
- );
+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(thread_context);
+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(thread_context);
const short safe_k = max(short(0), short(remaining_k - inner_k));
- const short2 left_limits = transpose_a
- ? short2(simdgroup_limit_m, safe_k)
- : short2(safe_k, simdgroup_limit_m);
- const short2 right_limits = transpose_b
- ? short2(safe_k, simdgroup_limit_n)
- : short2(simdgroup_limit_n, safe_k);
-
- const int left_offset =
- transpose_a ? inner_k * leading_dimension_a : inner_k;
- const int right_offset =
- transpose_b ? inner_k : inner_k * leading_dimension_b;
-
- left_tile
- .load_safe(left_ptr + left_offset, leading_dimension_a, left_limits);
- right_tile.load_safe(
- right_ptr + right_offset,
- leading_dimension_b,
- right_limits
- );
-
- MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(
- accumulator,
- left_tile,
- right_tile
- );
+ const short2 left_limits = transpose_a ? short2(simdgroup_limit_m, safe_k) : short2(safe_k, simdgroup_limit_m);
+ const short2 right_limits = transpose_b ? short2(safe_k, simdgroup_limit_n) : short2(simdgroup_limit_n, safe_k);
+
+ const int left_offset = transpose_a ? inner_k * leading_dimension_a : inner_k;
+ const int right_offset = transpose_b ? inner_k : inner_k * leading_dimension_b;
+
+ left_tile.load_safe(left_ptr + left_offset, leading_dimension_a, left_limits);
+ right_tile.load_safe(right_ptr + right_offset, leading_dimension_b, right_limits);
+
+ MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(accumulator, left_tile, right_tile);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h
index 7f1b2f305..f98167440 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h
@@ -18,11 +18,7 @@ namespace gemm {
template <typename T, typename U, int VALUES_PER_THREAD, int BITS>
METAL_FUNC U load_vector(const device T* x, thread U* x_thread) {
using U4 = vec<U, 4>;
- const U4 inv =
- U4(U(1),
- U(1) / U(1u << BITS),
- U(1) / U(1u << (2u * BITS)),
- U(1) / U(1u << (3u * BITS)));
+ const U4 inv = U4(U(1), U(1) / U(1u << BITS), U(1) / U(1u << (2u * BITS)), U(1) / U(1u << (3u * BITS)));
U sum = 0;
thread U4* x_vec4 = reinterpret_cast<thread U4*>(x_thread);
METAL_PRAGMA_UNROLL
@@ -50,30 +46,21 @@ METAL_FUNC U load_vector_safe(const device T* x, thread U* x_thread, int N) {
}
template <typename U, int VALUES_PER_THREAD, int BITS>
-METAL_FUNC U qdot(
- const device uint8_t* w,
- const thread U* x_thread,
- U scale,
- U bias,
- U sum
-) {
+METAL_FUNC U qdot(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum) {
static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
U accumulator = 0;
if constexpr (BITS == 4) {
using U4 = vec<U, 4>;
- const device ushort* weight_words =
- reinterpret_cast<const device ushort*>(w);
+ const device ushort* weight_words = reinterpret_cast<const device ushort*>(w);
const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);
METAL_PRAGMA_UNROLL
for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {
// Mask each nibble in place (no shifts); value of lane k is n_k << (4*k),
// i.e. n_k * 16^k, which is < 2^23 so the magic-number convert is valid.
// The matching x lane was pre-divided by 16^k in load_vector.
- const uint4 lanes =
- uint4(weight_words[i]) & uint4(0x000fu, 0x00f0u, 0x0f00u, 0xf000u);
- const U4 weight_vec4 =
- U4(as_type<float4>(lanes | uint4(0x4b000000u)) - float4(8388608.0f));
+ const uint4 lanes = uint4(weight_words[i]) & uint4(0x000fu, 0x00f0u, 0x0f00u, 0xf000u);
+ const U4 weight_vec4 = U4(as_type<float4>(lanes | uint4(0x4b000000u)) - float4(8388608.0f));
accumulator += dot(x_vec4[i], weight_vec4);
}
} else if constexpr (BITS == 8) {
@@ -85,9 +72,7 @@ METAL_FUNC U qdot(
// Mask each byte in place (no shifts); lane k value is b_k * 256^k. This
// exceeds the magic-number range for k=3, so use the hardware convert
// (exact for b_k * 256^k). x lane k was pre-divided by 256^k.
- const uint4 lanes =
- uint4(weight_words[i]) &
- uint4(0x000000ffu, 0x0000ff00u, 0x00ff0000u, 0xff000000u);
+ const uint4 lanes = uint4(weight_words[i]) & uint4(0x000000ffu, 0x0000ff00u, 0x00ff0000u, 0xff000000u);
const U4 weight_vec4 = U4(float4(lanes));
accumulator += dot(x_vec4[i], weight_vec4);
}
@@ -96,32 +81,19 @@ METAL_FUNC U qdot(
}
template <typename U, int VALUES_PER_THREAD, int BITS>
-METAL_FUNC U qdot_safe(
- const device uint8_t* w,
- const thread U* x_thread,
- U scale,
- U bias,
- U sum,
- int N
-) {
+METAL_FUNC U qdot_safe(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum, int N) {
static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
U accumulator = 0;
if constexpr (BITS == 4) {
using U4 = vec<U, 4>;
- const device uint16_t* weight_words =
- reinterpret_cast<const device uint16_t*>(w);
+ const device uint16_t* weight_words = reinterpret_cast<const device uint16_t*>(w);
const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);
int full_chunks = N / 4;
for (int i = 0; i < full_chunks; i++) {
uint16_t weight_word = weight_words[i];
- U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(
- weight_word,
- weight_word >> 4,
- weight_word >> 8,
- weight_word >> 12
- ));
+ U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(weight_word, weight_word >> 4, weight_word >> 8, weight_word >> 12));
accumulator += dot(x_vec4[i], weight_vec4);
}
@@ -131,11 +103,9 @@ METAL_FUNC U qdot_safe(
int base_index = 4 * full_chunks;
accumulator += x_thread[base_index] * uint_to_fp<U>(weight_word & 0xf);
if (remainder > 1)
- accumulator +=
- x_thread[base_index + 1] * uint_to_fp<U>((weight_word >> 4) & 0xf);
+ accumulator += x_thread[base_index + 1] * uint_to_fp<U>((weight_word >> 4) & 0xf);
if (remainder > 2)
- accumulator +=
- x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);
+ accumulator += x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);
}
} else if constexpr (BITS == 8) {
for (int i = 0; i < N; i++) {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h
index 4735576f3..137cdb4fd 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h
@@ -51,12 +51,7 @@ METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {
}
template <typename U, int N, int bits>
-METAL_FUNC void dequantize(
- const device uint8_t* w,
- U scale,
- U bias,
- threadgroup U* w_local
-) {
+METAL_FUNC void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
if constexpr (bits == 4) {
@@ -100,8 +95,7 @@ METAL_FUNC void dequantize<bfloat, 8, 4>(
v0 = v0 * scale + bias;
v1 = v1 * scale + bias;
- threadgroup bfloat4* out_ptr =
- reinterpret_cast<threadgroup bfloat4*>(w_local);
+ threadgroup bfloat4* out_ptr = reinterpret_cast<threadgroup bfloat4*>(w_local);
out_ptr[0] = v0;
out_ptr[1] = v1;
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h
index 2db44e7a1..cae407263 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h
@@ -15,17 +15,13 @@ template <
typename T,
int GRID_ROWS_,
int GRID_COLS_,
- class SimdgroupMultiplyAccumulateOps_ =
- SimdgroupMultiplyAccumulate<T, 8, 8>>
+ class SimdgroupMultiplyAccumulateOps_ = SimdgroupMultiplyAccumulate<T, 8, 8>>
struct SimdgroupFragment {
using SimdgroupMultiplyAccumulateOpsType = SimdgroupMultiplyAccumulateOps_;
using ElementType = T;
- METAL_CONST int MULTIPLY_ACCUMULATE_ROWS =
- SimdgroupMultiplyAccumulateOpsType::ROWS;
- METAL_CONST int MULTIPLY_ACCUMULATE_COLS =
- SimdgroupMultiplyAccumulateOpsType::COLS;
- METAL_CONST int ELEMENTS_PER_MULTIPLY_ACCUMULATE =
- SimdgroupMultiplyAccumulateOpsType::THREAD_ELEMENT_COUNT;
+ METAL_CONST int MULTIPLY_ACCUMULATE_ROWS = SimdgroupMultiplyAccumulateOpsType::ROWS;
+ METAL_CONST int MULTIPLY_ACCUMULATE_COLS = SimdgroupMultiplyAccumulateOpsType::COLS;
+ METAL_CONST int ELEMENTS_PER_MULTIPLY_ACCUMULATE = SimdgroupMultiplyAccumulateOpsType::THREAD_ELEMENT_COUNT;
METAL_CONST int GRID_ROWS = GRID_ROWS_;
METAL_CONST int GRID_COLS = GRID_COLS_;
@@ -34,17 +30,12 @@ struct SimdgroupFragment {
METAL_CONST int TOTAL_COLS = GRID_COLS * MULTIPLY_ACCUMULATE_COLS;
METAL_CONST int MULTIPLY_ACCUMULATE_COUNT = GRID_ROWS * GRID_COLS;
- METAL_CONST int ELEMENTS_PER_FRAGMENT =
- MULTIPLY_ACCUMULATE_COUNT * ELEMENTS_PER_MULTIPLY_ACCUMULATE;
+ METAL_CONST int ELEMENTS_PER_FRAGMENT = MULTIPLY_ACCUMULATE_COUNT * ELEMENTS_PER_MULTIPLY_ACCUMULATE;
- typedef typename SimdgroupMultiplyAccumulateOpsType::SimdgroupMatrixType
- SimdgroupMatrixType;
- typedef typename SimdgroupMultiplyAccumulateOpsType::ThreadDataType
- ThreadDataType;
+ typedef typename SimdgroupMultiplyAccumulateOpsType::SimdgroupMatrixType SimdgroupMatrixType;
+ typedef typename SimdgroupMultiplyAccumulateOpsType::ThreadDataType ThreadDataType;
- ThreadDataType multiply_accumulate_data[MULTIPLY_ACCUMULATE_COUNT] = {
- ThreadDataType(0)
- };
+ ThreadDataType multiply_accumulate_data[MULTIPLY_ACCUMULATE_COUNT] = {ThreadDataType(0)};
METAL_FUNC SimdgroupFragment() thread {}
@@ -55,36 +46,21 @@ struct SimdgroupFragment {
}
}
- METAL_FUNC constexpr thread ThreadDataType& multiply_accumulate_at(
- const ushort i,
- const ushort j
- ) {
+ METAL_FUNC constexpr thread ThreadDataType& multiply_accumulate_at(const ushort i, const ushort j) {
return multiply_accumulate_data[i * GRID_COLS + j];
}
- METAL_FUNC constexpr const thread ThreadDataType& multiply_accumulate_at(
- const ushort i,
- const ushort j
- ) const {
+ METAL_FUNC constexpr const thread ThreadDataType& multiply_accumulate_at(const ushort i, const ushort j) const {
return multiply_accumulate_data[i * GRID_COLS + j];
}
- METAL_FUNC thread ElementType* elements() {
- return reinterpret_cast<thread ElementType*>(multiply_accumulate_data);
- }
+ METAL_FUNC thread ElementType* elements() { return reinterpret_cast<thread ElementType*>(multiply_accumulate_data); }
METAL_FUNC const thread ElementType* elements() const {
- return reinterpret_cast<const thread ElementType*>(
- multiply_accumulate_data
- );
+ return reinterpret_cast<const thread ElementType*>(multiply_accumulate_data);
}
- template <
- typename U,
- int SIMDGROUP_STRIDE_X,
- int SIMDGROUP_STRIDE_Y,
- int STRIDE_X,
- int STRIDE_Y>
+ template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, int STRIDE_X, int STRIDE_Y>
METAL_FUNC void load(const threadgroup U* source) {
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < GRID_ROWS; ++i) {
@@ -93,10 +69,8 @@ struct SimdgroupFragment {
SimdgroupMultiplyAccumulateOpsType::load(
multiply_accumulate_at(i, j),
&(source
- [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *
- STRIDE_X +
- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y *
- STRIDE_Y]),
+ [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X * STRIDE_X +
+ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y * STRIDE_Y]),
STRIDE_X,
STRIDE_Y
);
@@ -105,10 +79,7 @@ struct SimdgroupFragment {
}
template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>
- METAL_FUNC void store(
- device U* destination,
- const int leading_dimension
- ) const {
+ METAL_FUNC void store(device U* destination, const int leading_dimension) const {
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < GRID_ROWS; ++i) {
METAL_PRAGMA_UNROLL
@@ -116,8 +87,7 @@ struct SimdgroupFragment {
SimdgroupMultiplyAccumulateOpsType::store(
multiply_accumulate_at(i, j),
&(destination
- [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *
- leading_dimension +
+ [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X * leading_dimension +
(j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y]),
leading_dimension,
1
@@ -169,13 +139,12 @@ METAL_FUNC void tile_multiply_accumulate(
const ushort column_serpentine = (m % 2) ? (N - 1 - n) : n;
METAL_PRAGMA_UNROLL
for (ushort k = 0; k < K; ++k) {
- SimdgroupFragment<T, M, N>::SimdgroupMultiplyAccumulateOpsType::
- multiply_accumulate(
- D.multiply_accumulate_at(m, column_serpentine),
- A.multiply_accumulate_at(m, k),
- B.multiply_accumulate_at(k, column_serpentine),
- C.multiply_accumulate_at(m, column_serpentine)
- );
+ SimdgroupFragment<T, M, N>::SimdgroupMultiplyAccumulateOpsType::multiply_accumulate(
+ D.multiply_accumulate_at(m, column_serpentine),
+ A.multiply_accumulate_at(m, k),
+ B.multiply_accumulate_at(k, column_serpentine),
+ C.multiply_accumulate_at(m, column_serpentine)
+ );
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
index 27860a3ac..939635b80 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
@@ -13,14 +13,8 @@ namespace matmul {
template <typename T, int ROWS_, int COLS_>
struct SimdgroupMultiplyAccumulate {
- static_assert(
- ROWS_ == 8,
- "Only 8 x 8 fragment matrices are currently supported"
- );
- static_assert(
- COLS_ == 8,
- "Only 8 x 8 fragment matrices are currently supported"
- );
+ static_assert(ROWS_ == 8, "Only 8 x 8 fragment matrices are currently supported");
+ static_assert(COLS_ == 8, "Only 8 x 8 fragment matrices are currently supported");
};
template <typename T>
@@ -39,9 +33,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {
typedef metal::simdgroup_matrix<T, ROWS, COLS> SimdgroupMatrixType;
typedef metal::vec<T, THREAD_ELEMENT_COUNT> ThreadDataType;
- METAL_FUNC static constexpr short2 get_lane_coordinates(
- ushort simd_lane_id [[thread_index_in_simdgroup]]
- ) {
+ METAL_FUNC static constexpr short2 get_lane_coordinates(ushort simd_lane_id [[thread_index_in_simdgroup]]) {
const short quad_index = simd_lane_id / 4;
const short lane_row = (quad_index & 4) + ((simd_lane_id / 2) % 4);
const short lane_col = (quad_index & 2) * 2 + (simd_lane_id % 2) * 2;
@@ -59,8 +51,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {
for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {
METAL_PRAGMA_UNROLL
for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {
- destination[i * THREAD_ELEMENT_COLS + j] =
- static_cast<T>(source[i * row_stride + j * col_stride]);
+ destination[i * THREAD_ELEMENT_COLS + j] = static_cast<T>(source[i * row_stride + j * col_stride]);
}
}
}
@@ -89,9 +80,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {
for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {
if ((row_offset + i) < row_limit && (col_offset + j) < col_limit) {
destination[i * THREAD_ELEMENT_COLS + j] =
- static_cast<T>(source
- [(row_offset + i) * row_stride +
- (col_offset + j) * col_stride]);
+ static_cast<T>(source[(row_offset + i) * row_stride + (col_offset + j) * col_stride]);
} else {
destination[i * THREAD_ELEMENT_COLS + j] = T(0);
}
@@ -99,10 +88,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {
}
}
- template <
- typename DestinationPointerType,
- typename RowStride,
- typename ColStride>
+ template <typename DestinationPointerType, typename RowStride, typename ColStride>
METAL_FUNC static constexpr void store(
const thread ThreadDataType& source,
DestinationPointerType destination,
@@ -115,8 +101,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {
for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {
METAL_PRAGMA_UNROLL
for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {
- destination[i * row_stride + j * col_stride] =
- static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);
+ destination[i * row_stride + j * col_stride] = static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);
}
}
}
@@ -146,9 +131,8 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {
METAL_PRAGMA_UNROLL
for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {
if ((row_offset + i) < row_limit && (col_offset + j) < col_limit) {
- destination
- [(row_offset + i) * row_stride + (col_offset + j) * col_stride] =
- static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);
+ destination[(row_offset + i) * row_stride + (col_offset + j) * col_stride] =
+ static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
index 2552c3a0a..451bc066b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
@@ -29,52 +29,27 @@ template <
typename Epilogue = TransformNone<DT, AccumulatorType>>
struct ThreadgroupTile {
METAL_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;
- using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<
- AccumulatorType,
- SIMDGROUP_BLOCK_SIZE,
- SIMDGROUP_BLOCK_SIZE>;
-
- METAL_CONST ushort TILE_ROW_STRIDE =
- SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
- METAL_CONST ushort TILE_COL_STRIDE =
- SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
-
- METAL_CONST ushort TILE_ROWS =
- BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);
- METAL_CONST ushort TILE_COLS =
- BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);
-
- METAL_CONST ushort A_STRIDE_ROW =
- transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;
- METAL_CONST ushort A_STRIDE_INNER =
- transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;
-
- METAL_CONST ushort B_STRIDE_INNER =
- transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;
- METAL_CONST ushort B_STRIDE_COL =
- transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;
+ using SimdgroupMultiplyAccumulateType =
+ SimdgroupMultiplyAccumulate<AccumulatorType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;
+
+ METAL_CONST ushort TILE_ROW_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
+ METAL_CONST ushort TILE_COL_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
+
+ METAL_CONST ushort TILE_ROWS = BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);
+ METAL_CONST ushort TILE_COLS = BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);
+
+ METAL_CONST ushort A_STRIDE_ROW = transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;
+ METAL_CONST ushort A_STRIDE_INNER = transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;
+
+ METAL_CONST ushort B_STRIDE_INNER = transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;
+ METAL_CONST ushort B_STRIDE_COL = transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;
METAL_CONST ushort TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;
METAL_CONST ushort TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;
- SimdgroupFragment<
- AccumulatorType,
- TILE_ROWS,
- 1,
- SimdgroupMultiplyAccumulateType>
- a_fragment;
- SimdgroupFragment<
- AccumulatorType,
- 1,
- TILE_COLS,
- SimdgroupMultiplyAccumulateType>
- b_fragment;
- SimdgroupFragment<
- AccumulatorType,
- TILE_ROWS,
- TILE_COLS,
- SimdgroupMultiplyAccumulateType>
- c_fragment;
+ SimdgroupFragment<AccumulatorType, TILE_ROWS, 1, SimdgroupMultiplyAccumulateType> a_fragment;
+ SimdgroupFragment<AccumulatorType, 1, TILE_COLS, SimdgroupMultiplyAccumulateType> b_fragment;
+ SimdgroupFragment<AccumulatorType, TILE_ROWS, TILE_COLS, SimdgroupMultiplyAccumulateType> c_fragment;
ushort simdgroup_row_offset;
ushort simdgroup_col_offset;
@@ -83,53 +58,30 @@ struct ThreadgroupTile {
ushort b_shared_offset;
METAL_FUNC ThreadgroupTile(const thread ThreadContext& thread_context) {
- ushort tile_row_base =
- SIMDGROUP_BLOCK_SIZE *
- (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);
- ushort tile_col_base =
- SIMDGROUP_BLOCK_SIZE *
- (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
-
- const ushort2 simdgroup_coordinates = ushort2(
- SimdgroupMultiplyAccumulateType::get_lane_coordinates(
- thread_context.simd_lane_id
- )
- );
+ ushort tile_row_base = SIMDGROUP_BLOCK_SIZE * (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);
+ ushort tile_col_base = SIMDGROUP_BLOCK_SIZE * (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
+
+ const ushort2 simdgroup_coordinates =
+ ushort2(SimdgroupMultiplyAccumulateType::get_lane_coordinates(thread_context.simd_lane_id));
simdgroup_row_offset = simdgroup_coordinates.y;
simdgroup_col_offset = simdgroup_coordinates.x;
- a_shared_offset = (tile_row_base + simdgroup_row_offset) * A_STRIDE_ROW +
- (simdgroup_col_offset)*A_STRIDE_INNER;
- b_shared_offset = (simdgroup_row_offset)*B_STRIDE_INNER +
- (tile_col_base + simdgroup_col_offset) * B_STRIDE_COL;
+ a_shared_offset = (tile_row_base + simdgroup_row_offset) * A_STRIDE_ROW + (simdgroup_col_offset)*A_STRIDE_INNER;
+ b_shared_offset = (simdgroup_row_offset)*B_STRIDE_INNER + (tile_col_base + simdgroup_col_offset) * B_STRIDE_COL;
simdgroup_row_offset += tile_row_base;
simdgroup_col_offset += tile_col_base;
}
- METAL_FUNC void multiply_accumulate(
- const threadgroup AT* a_shared,
- const threadgroup BT* b_shared
- ) {
+ METAL_FUNC void multiply_accumulate(const threadgroup AT* a_shared, const threadgroup BT* b_shared) {
a_shared += a_shared_offset;
b_shared += b_shared_offset;
METAL_PRAGMA_UNROLL
- for (ushort k_block_index = 0; k_block_index < BLOCK_DEPTH;
- k_block_index += SIMDGROUP_BLOCK_SIZE) {
- a_fragment.template load<
- AT,
- SIMDGROUPS_PER_ROW,
- 1,
- A_STRIDE_ROW,
- A_STRIDE_INNER>(a_shared);
-
- b_fragment.template load<
- BT,
- 1,
- SIMDGROUPS_PER_COLUMN,
- B_STRIDE_INNER,
- B_STRIDE_COL>(b_shared);
+ for (ushort k_block_index = 0; k_block_index < BLOCK_DEPTH; k_block_index += SIMDGROUP_BLOCK_SIZE) {
+ a_fragment.template load<AT, SIMDGROUPS_PER_ROW, 1, A_STRIDE_ROW, A_STRIDE_INNER>(a_shared);
+
+ b_fragment.template load<BT, 1, SIMDGROUPS_PER_COLUMN, B_STRIDE_INNER, B_STRIDE_COL>(b_shared);
tile_multiply_accumulate(c_fragment, a_fragment, b_fragment, c_fragment);
@@ -146,36 +98,26 @@ struct ThreadgroupTile {
D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;
- c_fragment.template store<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
- D,
- leading_dimension_d
- );
+ c_fragment.template store<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(D, leading_dimension_d);
}
- METAL_FUNC void store_result_safe(
- device DT* D,
- const int leading_dimension_d,
- short2 destination_tile_dimensions
- ) {
+ METAL_FUNC void store_result_safe(device DT* D, const int leading_dimension_d, short2 destination_tile_dimensions) {
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {
c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);
}
D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;
- destination_tile_dimensions -=
- short2(simdgroup_col_offset, simdgroup_row_offset);
+ destination_tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
- if (destination_tile_dimensions.x <= 0 ||
- destination_tile_dimensions.y <= 0)
+ if (destination_tile_dimensions.x <= 0 || destination_tile_dimensions.y <= 0)
return;
- c_fragment
- .template store_safe<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
- D,
- leading_dimension_d,
- destination_tile_dimensions
- );
+ c_fragment.template store_safe<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
+ D,
+ leading_dimension_d,
+ destination_tile_dimensions
+ );
}
template <typename EpilogueOp>
@@ -185,9 +127,8 @@ struct ThreadgroupTile {
const int column_stride_c,
thread const EpilogueOp& epilogue_operation
) {
- const device DT* c_pointer = C +
- simdgroup_row_offset * leading_dimension_c +
- simdgroup_col_offset * column_stride_c;
+ const device DT* c_pointer =
+ C + simdgroup_row_offset * leading_dimension_c + simdgroup_col_offset * column_stride_c;
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < TILE_ROWS; i++) {
@@ -195,20 +136,11 @@ struct ThreadgroupTile {
for (ushort j = 0; j < TILE_COLS; j++) {
thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
METAL_PRAGMA_UNROLL
- for (ushort k = 0;
- k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;
- k++) {
- const ushort row_offset =
- i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
- const ushort col_offset =
- j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
- DT c_value = c_pointer
- [row_offset * leading_dimension_c + col_offset * column_stride_c +
- k];
- block_data[k] = epilogue_operation.apply(
- block_data[k],
- static_cast<AccumulatorType>(c_value)
- );
+ for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {
+ const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
+ const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
+ DT c_value = c_pointer[row_offset * leading_dimension_c + col_offset * column_stride_c + k];
+ block_data[k] = epilogue_operation.apply(block_data[k], static_cast<AccumulatorType>(c_value));
}
}
}
@@ -222,9 +154,8 @@ struct ThreadgroupTile {
short2 tile_dimensions,
thread const EpilogueOp& epilogue_operation
) {
- const device DT* c_pointer = C +
- simdgroup_row_offset * leading_dimension_c +
- simdgroup_col_offset * column_stride_c;
+ const device DT* c_pointer =
+ C + simdgroup_row_offset * leading_dimension_c + simdgroup_col_offset * column_stride_c;
tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
METAL_PRAGMA_UNROLL
@@ -233,21 +164,12 @@ struct ThreadgroupTile {
for (ushort j = 0; j < TILE_COLS; j++) {
thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
- const ushort col_offset =
- j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
+ const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
METAL_PRAGMA_UNROLL
- for (ushort k = 0;
- k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;
- k++) {
- if (row_offset < tile_dimensions.y &&
- col_offset + k < tile_dimensions.x) {
- DT c_value = c_pointer
- [row_offset * leading_dimension_c +
- col_offset * column_stride_c + k];
- block_data[k] = epilogue_operation.apply(
- block_data[k],
- static_cast<AccumulatorType>(c_value)
- );
+ for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {
+ if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x) {
+ DT c_value = c_pointer[row_offset * leading_dimension_c + col_offset * column_stride_c + k];
+ block_data[k] = epilogue_operation.apply(block_data[k], static_cast<AccumulatorType>(c_value));
}
}
}
@@ -262,23 +184,16 @@ struct ThreadgroupTile {
METAL_PRAGMA_UNROLL
for (ushort j = 0; j < TILE_COLS; j++) {
thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
- const ushort col_offset =
- j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
+ const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
METAL_PRAGMA_UNROLL
- for (ushort k = 0;
- k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;
- k++) {
- block_data[k] +=
- static_cast<AccumulatorType>(bias_pointer[col_offset + k]);
+ for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {
+ block_data[k] += static_cast<AccumulatorType>(bias_pointer[col_offset + k]);
}
}
}
}
- METAL_FUNC void apply_bias_safe(
- const device BT* bias,
- short2 tile_dimensions
- ) {
+ METAL_FUNC void apply_bias_safe(const device BT* bias, short2 tile_dimensions) {
const device BT* bias_pointer = bias + simdgroup_col_offset;
tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
@@ -288,16 +203,11 @@ struct ThreadgroupTile {
for (ushort j = 0; j < TILE_COLS; j++) {
thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
- const ushort col_offset =
- j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
+ const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
METAL_PRAGMA_UNROLL
- for (ushort k = 0;
- k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;
- k++) {
- if (row_offset < tile_dimensions.y &&
- col_offset + k < tile_dimensions.x) {
- block_data[k] +=
- static_cast<AccumulatorType>(bias_pointer[col_offset + k]);
+ for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {
+ if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x) {
+ block_data[k] += static_cast<AccumulatorType>(bias_pointer[col_offset + k]);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h
index 84b37753a..19ba79a27 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h
@@ -20,16 +20,10 @@ static METAL_FUNC uint morton_expand_bits(uint x) {
// Morton-unpacks a 1D dispatch index (`fp.rs` sets `group_count_y = 1`) into
// 2D.
static METAL_FUNC uint2 morton_tile_id(uint2 threadgroup_position) {
- return uint2(
- morton_expand_bits(threadgroup_position.x),
- morton_expand_bits(threadgroup_position.x >> 1)
- );
+ return uint2(morton_expand_bits(threadgroup_position.x), morton_expand_bits(threadgroup_position.x >> 1));
}
-static METAL_FUNC uint2 tile_id(
- uint2 threadgroup_position,
- const constant uzu::matmul::GemmParams* params
-) {
+static METAL_FUNC uint2 tile_id(uint2 threadgroup_position, const constant uzu::matmul::GemmParams* params) {
if (params->use_morton) {
return morton_tile_id(threadgroup_position);
}
@@ -44,12 +38,11 @@ struct ThreadgroupTileGeometry {
uint block_col_start;
bool out_of_bounds;
- static METAL_FUNC ThreadgroupTileGeometry
- compute(uint2 tile_id, const constant uzu::matmul::GemmParams* params) {
+ static METAL_FUNC ThreadgroupTileGeometry compute(uint2 tile_id, const constant uzu::matmul::GemmParams* params) {
ThreadgroupTileGeometry geometry;
geometry.tile_id = tile_id;
- geometry.out_of_bounds = (tile_id.x >= params->threadgroups_per_row) ||
- (tile_id.y >= params->threadgroups_per_column);
+ geometry.out_of_bounds =
+ (tile_id.x >= params->threadgroups_per_row) || (tile_id.y >= params->threadgroups_per_column);
geometry.block_row_start = tile_id.y * THREADGROUP_BLOCK_M;
geometry.block_col_start = tile_id.x * THREADGROUP_BLOCK_N;
return geometry;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
index 864a0af1c..35ba6d7c7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
@@ -18,17 +18,12 @@ static METAL_FUNC void apply_output_random_hadamard_transform(
const ushort stripes_per_row = tile_block_cols / METAL_SIMD_SIZE;
const ushort simd_lane = thread_context.simd_lane_id;
const uint total_work = uint(tile_block_rows) * uint(stripes_per_row);
- for (uint cell = thread_context.simdgroup_index; cell < total_work;
- cell += simdgroup_count) {
+ for (uint cell = thread_context.simdgroup_index; cell < total_work; cell += simdgroup_count) {
const ushort row_local = ushort(cell / stripes_per_row);
const ushort stripe = ushort(cell % stripes_per_row);
const ushort col_local = stripe * ushort(METAL_SIMD_SIZE) + simd_lane;
- const size_t output_index =
- size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);
- output_block[output_index] = simdgroup_output_random_hadamard_transform(
- simd_lane,
- output_block[output_index],
- rht_factors_block[col_local]
- );
+ const size_t output_index = size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);
+ output_block[output_index] =
+ simdgroup_output_random_hadamard_transform(simd_lane, output_block[output_index], rht_factors_block[col_local]);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 82202985f..17112a7fb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -32,42 +32,30 @@ template <
struct MxuMmaCore {
METAL_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
METAL_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
- METAL_CONST ushort SIMDGROUPS_PER_ROW =
- gemm_tiling_simdgroups_per_row(GEMM_TILING);
- METAL_CONST ushort SIMDGROUPS_PER_COLUMN =
- gemm_tiling_simdgroups_per_column(GEMM_TILING);
- METAL_CONST ushort SIMDGROUP_BLOCK_M =
- THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
- METAL_CONST ushort SIMDGROUP_BLOCK_N =
- THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
+ METAL_CONST ushort SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);
+ METAL_CONST ushort SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);
+ METAL_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
+ METAL_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;
METAL_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);
static_assert(
THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,
"FP THREADGROUP_BLOCK_K must be a multiple of SIMDGROUP_BLOCK_K"
);
- METAL_CONST ushort TILES_M =
- SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;
- METAL_CONST ushort TILES_N =
- SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps::FRAGMENT_COLS;
- METAL_CONST ushort TILES_K =
- SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;
-
- METAL_CONST ushort QUANT_BK =
- (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;
+ METAL_CONST ushort TILES_M = SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;
+ METAL_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps::FRAGMENT_COLS;
+ METAL_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;
+
+ METAL_CONST ushort QUANT_BK = (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;
METAL_CONST ushort PADDING_B = 16 / sizeof(BT);
- METAL_CONST ushort SHARED_STRIDE_B =
- (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;
- METAL_CONST ushort THREADGROUP_THREADS =
- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
+ METAL_CONST ushort SHARED_STRIDE_B = (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;
+ METAL_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
static_assert(
- B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
- QUANT_BK % SIMDGROUP_BLOCK_K == 0,
+ B_PROLOGUE == GemmBPrologueKind::FullPrecision || QUANT_BK % SIMDGROUP_BLOCK_K == 0,
"QUANT_BK must be a multiple of SIMDGROUP_BLOCK_K"
);
static_assert(
- B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
- THREADGROUP_BLOCK_K_FP % QUANT_BK == 0,
+ B_PROLOGUE == GemmBPrologueKind::FullPrecision || THREADGROUP_BLOCK_K_FP % QUANT_BK == 0,
"Tile block_k must be a multiple of QUANT_BK"
);
@@ -102,8 +90,7 @@ struct MxuMmaCore {
(BITS > 0) ? BITS : 4,
true>;
- using AccumFragment = uzu::matmul::
- Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;
+ using AccumFragment = uzu::matmul::Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;
template <bool ALIGNED_M, bool ALIGNED_N, typename Loader>
static METAL_FUNC AccumFragment quant_k_loop(
@@ -121,8 +108,7 @@ struct MxuMmaCore {
AccumFragment accumulator(thread_context);
accumulator.clear();
- threadgroup BT* b_shared_simdgroup =
- b_shared + tile_col_offset * SHARED_STRIDE_B;
+ threadgroup BT* b_shared_simdgroup = b_shared + tile_col_offset * SHARED_STRIDE_B;
const short2 tile_dimensions_b = short2(QUANT_BK, tile_block_cols);
METAL_PRAGMA_NO_UNROLL
@@ -137,29 +123,20 @@ struct MxuMmaCore {
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < QUANT_BK; inner_k += SIMDGROUP_BLOCK_K) {
- uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps>
- left_tile(thread_context);
- uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps>
- right_tile(thread_context);
+ uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps> left_tile(thread_context);
+ uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps> right_tile(thread_context);
const int left_offset = inner_k;
if constexpr (ALIGNED_M) {
left_tile.load(a_simdgroup + left_offset, leading_dimension_a);
} else {
- left_tile.load_safe(
- a_simdgroup + left_offset,
- leading_dimension_a,
- short2(SIMDGROUP_BLOCK_K, simdgroup_limit_m)
- );
+ left_tile
+ .load_safe(a_simdgroup + left_offset, leading_dimension_a, short2(SIMDGROUP_BLOCK_K, simdgroup_limit_m));
}
right_tile.load(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B));
- uzu::matmul::MxuFragmentOps::template tile_matmul<false, true>(
- accumulator,
- left_tile,
- right_tile
- );
+ uzu::matmul::MxuFragmentOps::template tile_matmul<false, true>(accumulator, left_tile, right_tile);
}
a_simdgroup += QUANT_BK;
@@ -187,11 +164,8 @@ struct MxuMmaCore {
const uint partition = thread_context.threadgroup_position.z;
const uint tile_y = thread_context.threadgroup_position.y;
- const uint2 tile =
- tile_id(uint2(thread_context.threadgroup_position.x, tile_y), params);
- const auto geometry =
- ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::
- compute(tile, params);
+ const uint2 tile = tile_id(uint2(thread_context.threadgroup_position.x, tile_y), params);
+ const auto geometry = ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::compute(tile, params);
if (geometry.out_of_bounds) {
return;
}
@@ -200,69 +174,43 @@ struct MxuMmaCore {
const size_t block_col = size_t(geometry.block_col_start);
const uint k_offset_per_block =
- (B_PROLOGUE == GemmBPrologueKind::FullPrecision)
- ? uint(THREADGROUP_BLOCK_K_FP)
- : uint(QUANT_BK);
- const uint k_offset =
- partition * params->aligned_inner_iterations * k_offset_per_block;
-
- const device AT* a_block =
- a + block_row * params->leading_dimension_a + k_offset;
- const device BT* b_block_fp =
- b +
- (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) +
- (TRANSPOSE_B ? k_offset : k_offset * uint(params->leading_dimension_b));
-
- const ushort tile_row_offset =
- SIMDGROUP_BLOCK_M *
- (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);
- const ushort tile_col_offset =
- SIMDGROUP_BLOCK_N *
- (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
-
- device DT* d_simdgroup =
- d + size_t(partition) * size_t(params->M) * size_t(params->N) +
- block_row * params->leading_dimension_d + block_col +
- tile_row_offset * params->leading_dimension_d + tile_col_offset;
+ (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? uint(THREADGROUP_BLOCK_K_FP) : uint(QUANT_BK);
+ const uint k_offset = partition * params->aligned_inner_iterations * k_offset_per_block;
+
+ const device AT* a_block = a + block_row * params->leading_dimension_a + k_offset;
+ const device BT* b_block_fp = b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) +
+ (TRANSPOSE_B ? k_offset : k_offset * uint(params->leading_dimension_b));
+
+ const ushort tile_row_offset = SIMDGROUP_BLOCK_M * (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);
+ const ushort tile_col_offset = SIMDGROUP_BLOCK_N * (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
+
+ device DT* d_simdgroup = d + size_t(partition) * size_t(params->M) * size_t(params->N) +
+ block_row * params->leading_dimension_d + block_col +
+ tile_row_offset * params->leading_dimension_d + tile_col_offset;
const short simdgroup_limit_m =
alignment.contains(GemmAlignment::M)
? SIMDGROUP_BLOCK_M
- : short(
- min(int(SIMDGROUP_BLOCK_M),
- int(params->M) -
- int(geometry.block_row_start + tile_row_offset))
- );
+ : short(min(int(SIMDGROUP_BLOCK_M), int(params->M) - int(geometry.block_row_start + tile_row_offset)));
const short simdgroup_limit_n =
alignment.contains(GemmAlignment::N)
? SIMDGROUP_BLOCK_N
- : short(
- min(int(SIMDGROUP_BLOCK_N),
- int(params->N) -
- int(geometry.block_col_start + tile_col_offset))
- );
-
- const device AT* a_simdgroup =
- a_block + size_t(tile_row_offset) * params->leading_dimension_a;
+ : short(min(int(SIMDGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start + tile_col_offset)));
+
+ const device AT* a_simdgroup = a_block + size_t(tile_row_offset) * params->leading_dimension_a;
const device BT* b_simdgroup_fp =
- b_block_fp + (TRANSPOSE_B ? size_t(tile_col_offset) *
- int(params->leading_dimension_b)
- : size_t(tile_col_offset));
+ b_block_fp +
+ (TRANSPOSE_B ? size_t(tile_col_offset) * int(params->leading_dimension_b) : size_t(tile_col_offset));
- const ushort tile_block_cols = ushort(
- min(int(THREADGROUP_BLOCK_N),
- int(params->N) - int(geometry.block_col_start))
- );
+ const ushort tile_block_cols =
+ ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start)));
const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);
- const bool apply_accumulate =
- output_transform.contains(GemmDTransform::ACCUMULATE);
+ const bool apply_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);
const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);
- const device BT* bias_simdgroup =
- output_bias + size_t(block_col) + size_t(tile_col_offset);
- using FragType = uzu::matmul::
- Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;
+ const device BT* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);
+ using FragType = uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;
const short2 thread_position = FragType::get_position(thread_context);
auto dispatch_aligned_k = [&](auto body) {
@@ -274,19 +222,14 @@ struct MxuMmaCore {
};
dispatch_aligned_k([&](auto aligned_k) {
dispatch_bool(
- alignment.contains(GemmAlignment::M) ||
- (simdgroup_limit_m == SIMDGROUP_BLOCK_M),
+ alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),
[&](auto aligned_m) {
dispatch_bool(
- alignment.contains(GemmAlignment::N) ||
- (simdgroup_limit_n == SIMDGROUP_BLOCK_N),
+ alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),
[&](auto aligned_n) {
auto accumulator_tile = [&]() {
- if constexpr (
- B_PROLOGUE == GemmBPrologueKind::FullPrecision
- ) {
- const int aligned_k_iterations_fp =
- int(params->aligned_inner_iterations);
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
+ const int aligned_k_iterations_fp = int(params->aligned_inner_iterations);
return uzu::matmul::gemm_loop<
AT,
BT,
@@ -311,30 +254,21 @@ struct MxuMmaCore {
thread_context
);
} else {
- const int aligned_k_iterations_q =
- int(params->aligned_inner_iterations);
+ const int aligned_k_iterations_q = int(params->aligned_inner_iterations);
constexpr int pack_factor = get_pack_factor<BITS, 8>();
constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
const int k_elements = int(params->K);
- const int weights_row_stride_bytes =
- k_elements * bytes_per_pack / pack_factor;
- const int groups_per_row =
- (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
+ const int weights_row_stride_bytes = k_elements * bytes_per_pack / pack_factor;
+ const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
const int k_offset_groups = int(k_offset) / GROUP_SIZE;
- const device uint8_t* weights_block =
- reinterpret_cast<const device uint8_t*>(b) +
- block_col * weights_row_stride_bytes +
- int(k_offset) * bytes_per_pack / pack_factor;
- const device BT* scales_offset =
- scales + block_col * groups_per_row + k_offset_groups;
+ const device uint8_t* weights_block = reinterpret_cast<const device uint8_t*>(b) +
+ block_col * weights_row_stride_bytes +
+ int(k_offset) * bytes_per_pack / pack_factor;
+ const device BT* scales_offset = scales + block_col * groups_per_row + k_offset_groups;
auto loader_b = [&]() {
- if constexpr (
- B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant
- ) {
- const device BT* biases_offset =
- biases + block_col * groups_per_row +
- k_offset_groups;
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
+ const device BT* biases_offset = biases + block_col * groups_per_row + k_offset_groups;
return BLoaderScaleBias(
weights_block,
scales_offset,
@@ -344,18 +278,12 @@ struct MxuMmaCore {
thread_context.simdgroup_index,
thread_context.simd_lane_id
);
- } else if constexpr (
- B_PROLOGUE ==
- GemmBPrologueKind::ScaleZeroPointDequant
- ) {
+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {
const int zero_point_stride_per_row =
- (BITS == 4) ? ((groups_per_row + 1) / 2)
- : groups_per_row;
+ (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;
const device uint8_t* zero_points_row_start =
- zero_points +
- block_col * zero_point_stride_per_row +
- ((BITS == 4) ? (k_offset_groups / 2)
- : k_offset_groups);
+ zero_points + block_col * zero_point_stride_per_row +
+ ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);
return BLoaderScaleZeroPoint(
weights_block,
scales_offset,
@@ -396,27 +324,19 @@ struct MxuMmaCore {
}();
if (apply_scale) {
- const AccumulatorType scale =
- AccumulatorType(params->ab_scale);
+ const AccumulatorType scale = AccumulatorType(params->ab_scale);
METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;
- i++) {
+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE; i++) {
accumulator_tile.elements()[i] *= scale;
}
}
if (apply_accumulate) {
- uzu::matmul::Fragment<
- DT,
- TILES_M,
- TILES_N,
- uzu::matmul::MxuFragmentOps>
- existing_output(thread_context);
+ uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps> existing_output(
+ thread_context
+ );
if constexpr (aligned_m.value && aligned_n.value) {
- existing_output.load(
- d_simdgroup,
- int(params->leading_dimension_d)
- );
+ existing_output.load(d_simdgroup, int(params->leading_dimension_d));
} else {
existing_output.load_safe(
d_simdgroup,
@@ -425,10 +345,8 @@ struct MxuMmaCore {
);
}
METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;
- i++) {
- accumulator_tile.elements()[i] +=
- AccumulatorType(existing_output.elements()[i]);
+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE; i++) {
+ accumulator_tile.elements()[i] += AccumulatorType(existing_output.elements()[i]);
}
}
@@ -436,37 +354,21 @@ struct MxuMmaCore {
METAL_PRAGMA_UNROLL
for (ushort tile_row = 0; tile_row < TILES_M; ++tile_row) {
METAL_PRAGMA_UNROLL
- for (ushort tile_col = 0; tile_col < TILES_N;
- ++tile_col) {
- thread auto& frag =
- accumulator_tile.fragment_at(tile_row, tile_col);
- const short col_base = short(
- tile_col *
- uzu::matmul::MxuFragmentOps::FRAGMENT_COLS +
- thread_position.x
- );
+ for (ushort tile_col = 0; tile_col < TILES_N; ++tile_col) {
+ thread auto& frag = accumulator_tile.fragment_at(tile_row, tile_col);
+ const short col_base =
+ short(tile_col * uzu::matmul::MxuFragmentOps::FRAGMENT_COLS + thread_position.x);
METAL_PRAGMA_UNROLL
- for (ushort i = 0;
- i <
- uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_ROWS;
- ++i) {
+ for (ushort i = 0; i < uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_ROWS; ++i) {
METAL_PRAGMA_UNROLL
- for (ushort j = 0;
- j <
- uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS;
- ++j) {
- const ushort element_index =
- i * uzu::matmul::MxuFragmentOps::
- THREAD_ELEMENT_COLS +
- j;
+ for (ushort j = 0; j < uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS; ++j) {
+ const ushort element_index = i * uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS + j;
const short col_local = short(col_base + j);
if constexpr (aligned_n.value) {
- frag[element_index] +=
- AccumulatorType(bias_simdgroup[col_local]);
+ frag[element_index] += AccumulatorType(bias_simdgroup[col_local]);
} else {
if (col_local < simdgroup_limit_n) {
- frag[element_index] +=
- AccumulatorType(bias_simdgroup[col_local]);
+ frag[element_index] += AccumulatorType(bias_simdgroup[col_local]);
}
}
}
@@ -476,10 +378,7 @@ struct MxuMmaCore {
}
if constexpr (aligned_m.value && aligned_n.value) {
- accumulator_tile.store(
- d_simdgroup,
- int(params->leading_dimension_d)
- );
+ accumulator_tile.store(d_simdgroup, int(params->leading_dimension_d));
} else {
accumulator_tile.store_safe(
d_simdgroup,
@@ -495,14 +394,9 @@ struct MxuMmaCore {
if (output_transform.contains(GemmDTransform::RHT)) {
threadgroup_barrier(mem_flags::mem_device);
- device DT* d_block =
- d + block_row * params->leading_dimension_d + block_col;
- const ushort tile_block_rows = ushort(
- min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row))
- );
- const ushort tile_block_cols = ushort(
- min(int(THREADGROUP_BLOCK_N), int(params->N) - int(block_col))
- );
+ device DT* d_block = d + block_row * params->leading_dimension_d + block_col;
+ const ushort tile_block_rows = ushort(min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row)));
+ const ushort tile_block_cols = ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(block_col)));
apply_output_random_hadamard_transform(
d_block,
rht_factors + block_col,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
index 6e34d9caf..bcdcc86a8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
@@ -22,28 +22,18 @@ template <
short GROUP_SIZE,
short BITS>
struct QuantizedBlockLoaderScaleBias {
- static_assert(
- THREADGROUP_TILE_COLS <= GROUP_SIZE,
- "Group size should be larger than columns"
- );
- static_assert(
- GROUP_SIZE % THREADGROUP_TILE_COLS == 0,
- "Group size should be divisible by columns"
- );
+ static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");
+ static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");
static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();
METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();
- METAL_CONST short THREADGROUP_TILE_COLS_PACKED =
- THREADGROUP_TILE_COLS / pack_factor;
- METAL_CONST short READS_PER_THREAD =
- (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)
- ? 1
- : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) /
- THREADGROUP_SIZE;
+ METAL_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;
+ METAL_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)
+ ? 1
+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;
METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;
- METAL_CONST bool TILE_HAS_IDLE_THREADS =
- THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
+ METAL_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
const int src_leading_dim;
const int tile_stride;
@@ -71,23 +61,14 @@ struct QuantizedBlockLoaderScaleBias {
: src_leading_dim(src_leading_dim_),
tile_stride(
REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS_PACKED * bytes_per_pack
- : THREADGROUP_TILE_ROWS * src_leading_dim_ *
- bytes_per_pack / pack_factor
+ : THREADGROUP_TILE_ROWS * src_leading_dim_ * bytes_per_pack / pack_factor
),
- group_step_counter(0),
- group_stride(THREADGROUP_TILE_ROWS * src_leading_dim_ / GROUP_SIZE),
+ group_step_counter(0), group_stride(THREADGROUP_TILE_ROWS * src_leading_dim_ / GROUP_SIZE),
thread_index(simd_group_id * 32 + simd_lane_id),
- tile_row_index(
- READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED
- ),
- tile_col_index(
- (READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED
- ),
- dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION +
- tile_col_index * pack_factor),
- src(src_ +
- tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor +
- tile_col_index * bytes_per_pack),
+ tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),
+ tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),
+ dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index * pack_factor),
+ src(src_ + tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor + tile_col_index * bytes_per_pack),
scales(scales_ + tile_row_index * src_leading_dim_ / GROUP_SIZE),
biases(biases_ + tile_row_index * src_leading_dim_ / GROUP_SIZE) {}
@@ -101,12 +82,7 @@ struct QuantizedBlockLoaderScaleBias {
T scale = *scales;
T bias = *biases;
for (int i = 0; i < READS_PER_THREAD; i++) {
- dequantize<T, pack_factor, BITS>(
- src + i * bytes_per_pack,
- scale,
- bias,
- dst + i * pack_factor
- );
+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
}
}
@@ -136,12 +112,7 @@ struct QuantizedBlockLoaderScaleBias {
T scale = *scales;
T bias = *biases;
for (int i = 0; i < READS_PER_THREAD; i++) {
- dequantize<T, pack_factor, BITS>(
- src + i * bytes_per_pack,
- scale,
- bias,
- dst + i * pack_factor
- );
+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
index 5f21844d6..8a36d2549 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
@@ -23,28 +23,18 @@ template <
short BITS,
bool SCALE_SYMMETRIC = false>
struct QuantizedBlockLoaderScaleZeroPoint {
- static_assert(
- THREADGROUP_TILE_COLS <= GROUP_SIZE,
- "Group size should be larger than columns"
- );
- static_assert(
- GROUP_SIZE % THREADGROUP_TILE_COLS == 0,
- "Group size should be divisible by columns"
- );
+ static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");
+ static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");
static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();
METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();
- METAL_CONST short THREADGROUP_TILE_COLS_PACKED =
- THREADGROUP_TILE_COLS / pack_factor;
- METAL_CONST short READS_PER_THREAD =
- (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)
- ? 1
- : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) /
- THREADGROUP_SIZE;
+ METAL_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;
+ METAL_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)
+ ? 1
+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;
METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;
- METAL_CONST bool TILE_HAS_IDLE_THREADS =
- THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
+ METAL_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
const int src_leading_dim;
const int groups_per_row;
@@ -76,48 +66,25 @@ struct QuantizedBlockLoaderScaleZeroPoint {
: src_leading_dim(src_leading_dim_), groups_per_row(groups_per_row_),
tile_stride(
REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS_PACKED * bytes_per_pack
- : THREADGROUP_TILE_ROWS * src_leading_dim_ *
- bytes_per_pack / pack_factor
+ : THREADGROUP_TILE_ROWS * src_leading_dim_ * bytes_per_pack / pack_factor
),
- group_step_counter(0), k_base(0),
- group_stride(THREADGROUP_TILE_ROWS * groups_per_row_),
+ group_step_counter(0), k_base(0), group_stride(THREADGROUP_TILE_ROWS * groups_per_row_),
thread_index(simd_group_id * 32 + simd_lane_id),
- tile_row_index(
- READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED
- ),
- tile_col_index(
- (READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED
- ),
- dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION +
- tile_col_index * pack_factor),
- src(src_ +
- tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor +
- tile_col_index * bytes_per_pack),
- scales(
- REDUCTION_DIMENSION == 1
- ? (scales_ + tile_row_index * groups_per_row_)
- : scales_
- ),
- scales_row_start(
- REDUCTION_DIMENSION == 1
- ? (scales_ + tile_row_index * groups_per_row_)
- : scales_
- ),
+ tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),
+ tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),
+ dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index * pack_factor),
+ src(src_ + tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor + tile_col_index * bytes_per_pack),
+ scales(REDUCTION_DIMENSION == 1 ? (scales_ + tile_row_index * groups_per_row_) : scales_),
+ scales_row_start(REDUCTION_DIMENSION == 1 ? (scales_ + tile_row_index * groups_per_row_) : scales_),
zero_points_row_start(
- SCALE_SYMMETRIC
- ? nullptr
- : (REDUCTION_DIMENSION == 1
- ? (zero_points_row_start_ +
- tile_row_index * (BITS == 4
- ? ((groups_per_row_ + 1) / 2)
- : groups_per_row_))
- : zero_points_row_start_)
+ SCALE_SYMMETRIC ? nullptr
+ : (REDUCTION_DIMENSION == 1
+ ? (zero_points_row_start_ +
+ tile_row_index * (BITS == 4 ? ((groups_per_row_ + 1) / 2) : groups_per_row_))
+ : zero_points_row_start_)
) {}
- inline void current_scale_bias(
- thread T& out_scale,
- thread T& out_bias
- ) const {
+ inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {
uint zero_point_value;
T scale_value;
int group_index;
@@ -131,11 +98,9 @@ struct QuantizedBlockLoaderScaleZeroPoint {
if constexpr (SCALE_SYMMETRIC) {
zero_point_value = 1u << (BITS - 1);
} else if constexpr (BITS == 4) {
- const device uint8_t* zero_point_ptr =
- zero_points_row_start + (group_index >> 1);
+ const device uint8_t* zero_point_ptr = zero_points_row_start + (group_index >> 1);
uint8_t zero_point_byte = *zero_point_ptr;
- zero_point_value =
- (uint(zero_point_byte) >> (uint(group_index & 1) * 4u)) & 0x0Fu;
+ zero_point_value = (uint(zero_point_byte) >> (uint(group_index & 1) * 4u)) & 0x0Fu;
} else {
zero_point_value = zero_points_row_start[group_index];
}
@@ -154,12 +119,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {
T bias;
current_scale_bias(scale, bias);
for (int i = 0; i < READS_PER_THREAD; i++) {
- dequantize<T, pack_factor, BITS>(
- src + i * bytes_per_pack,
- scale,
- bias,
- dst + i * pack_factor
- );
+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
}
}
@@ -187,12 +147,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {
for (int i = 0; i < READS_PER_THREAD; i++) {
int pack_index = tile_col_index + i;
if (pack_index < valid_packs) {
- dequantize<T, pack_factor, BITS>(
- src + i * bytes_per_pack,
- scale,
- bias,
- dst + i * pack_factor
- );
+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
if (pack_index == valid_packs - 1) {
int remaining = valid_cols - pack_index * pack_factor;
@@ -221,12 +176,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {
T bias;
current_scale_bias(scale, bias);
for (int i = 0; i < READS_PER_THREAD; i++) {
- dequantize<T, pack_factor, BITS>(
- src + i * bytes_per_pack,
- scale,
- bias,
- dst + i * pack_factor
- );
+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h
index e892d0a2f..654c72165 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h
@@ -51,12 +51,7 @@ METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {
}
template <typename U, int N, int bits>
-inline void dequantize(
- const device uint8_t* w,
- U scale,
- U bias,
- threadgroup U* w_local
-) {
+inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
if (bits == 4) {
@@ -74,18 +69,10 @@ inline void dequantize(
}
template <>
-inline void dequantize<bfloat, 8, 4>(
- const device uint8_t* w,
- bfloat scale,
- bfloat bias,
- threadgroup bfloat* w_local
-) {
+inline void dequantize<bfloat, 8, 4>(const device uint8_t* w, bfloat scale, bfloat bias, threadgroup bfloat* w_local) {
const uint32_t packed = *reinterpret_cast<const device uint32_t*>(w);
- const bfloat4 lo =
- bfloat4(as_type<uchar4>(packed & 0x0f0f0f0fu)) * scale + bias;
- const bfloat4 hi = bfloat4(as_type<uchar4>(packed & 0xf0f0f0f0u)) *
- (scale * bfloat(0.0625f)) +
- bias;
+ const bfloat4 lo = bfloat4(as_type<uchar4>(packed & 0x0f0f0f0fu)) * scale + bias;
+ const bfloat4 hi = bfloat4(as_type<uchar4>(packed & 0xf0f0f0f0u)) * (scale * bfloat(0.0625f)) + bias;
w_local[0] = lo.x;
w_local[1] = hi.x;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index 1ce69ed3b..30be122da 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -33,25 +33,16 @@ struct SimdgroupMmaCore {
METAL_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
METAL_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
METAL_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);
- METAL_CONST int SIMDGROUPS_PER_ROW =
- gemm_tiling_simdgroups_per_row(GEMM_TILING);
- METAL_CONST int SIMDGROUPS_PER_COLUMN =
- gemm_tiling_simdgroups_per_column(GEMM_TILING);
+ METAL_CONST int SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);
+ METAL_CONST int SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);
METAL_CONST ushort PADDING_A = 16 / sizeof(AT);
METAL_CONST ushort PADDING_B = 16 / sizeof(BT);
METAL_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;
- METAL_CONST ushort SHARED_STRIDE_B =
- (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;
- METAL_CONST ushort THREADGROUP_THREADS =
- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
+ METAL_CONST ushort SHARED_STRIDE_B = (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;
+ METAL_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
- using ALoader = uzu::matmul::ThreadgroupLoader<
- AT,
- THREADGROUP_BLOCK_M,
- THREADGROUP_BLOCK_K,
- SHARED_STRIDE_A,
- true,
- THREADGROUP_THREADS>;
+ using ALoader = uzu::matmul::
+ ThreadgroupLoader<AT, THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_K, SHARED_STRIDE_A, true, THREADGROUP_THREADS>;
using BLoaderFp = uzu::matmul::ThreadgroupLoader<
BT,
TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,
@@ -118,8 +109,7 @@ struct SimdgroupMmaCore {
constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};
short2 tile_dimensions_a = short2(THREADGROUP_BLOCK_K, tile_block_rows);
short2 tile_dimensions_b =
- TRANSPOSE_B ? short2(THREADGROUP_BLOCK_K, tile_block_cols)
- : short2(tile_block_cols, THREADGROUP_BLOCK_K);
+ TRANSPOSE_B ? short2(THREADGROUP_BLOCK_K, tile_block_cols) : short2(tile_block_cols, THREADGROUP_BLOCK_K);
for (int k = 0; k < aligned_k_iterations; k++) {
threadgroup_barrier(mem_flags::mem_threadgroup);
@@ -144,11 +134,9 @@ struct SimdgroupMmaCore {
if constexpr (!gemm_alignment.contains(GemmAlignment::K)) {
threadgroup_barrier(mem_flags::mem_threadgroup);
- short2 last_tile_dimensions_a =
- short2(leftover_block_depth, tile_block_rows);
+ short2 last_tile_dimensions_a = short2(leftover_block_depth, tile_block_rows);
short2 last_tile_dimensions_b =
- TRANSPOSE_B ? short2(leftover_block_depth, tile_block_cols)
- : short2(tile_block_cols, leftover_block_depth);
+ TRANSPOSE_B ? short2(leftover_block_depth, tile_block_cols) : short2(tile_block_cols, leftover_block_depth);
loader_a.load_safe(last_tile_dimensions_a);
loader_b.load_safe(last_tile_dimensions_b);
@@ -166,8 +154,7 @@ struct SimdgroupMmaCore {
const thread ushort& tile_block_rows,
const thread ushort& tile_block_cols,
const bool needs_epilogue,
- const thread uzu::matmul::TransformScaleAccumulate<float, float>&
- epilogue,
+ const thread uzu::matmul::TransformScaleAccumulate<float, float>& epilogue,
const device BT* bias_block,
const bool needs_bias,
const device int32_t* rht_factors_block,
@@ -175,10 +162,7 @@ struct SimdgroupMmaCore {
const thread ThreadContext& thread_context
) {
constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};
- if constexpr (
- gemm_alignment.contains(GemmAlignment::M) &&
- gemm_alignment.contains(GemmAlignment::N)
- ) {
+ if constexpr (gemm_alignment.contains(GemmAlignment::M) && gemm_alignment.contains(GemmAlignment::N)) {
if (needs_epilogue) {
accumulator.apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
}
@@ -188,25 +172,13 @@ struct SimdgroupMmaCore {
accumulator.store_result(d, params->leading_dimension_d);
} else {
if (needs_epilogue) {
- accumulator.apply_epilogue_safe(
- d,
- params->leading_dimension_d,
- 1,
- short2(tile_block_cols, tile_block_rows),
- epilogue
- );
+ accumulator
+ .apply_epilogue_safe(d, params->leading_dimension_d, 1, short2(tile_block_cols, tile_block_rows), epilogue);
}
if (needs_bias) {
- accumulator.apply_bias_safe(
- bias_block,
- short2(tile_block_cols, tile_block_rows)
- );
+ accumulator.apply_bias_safe(bias_block, short2(tile_block_cols, tile_block_rows));
}
- accumulator.store_result_safe(
- d,
- params->leading_dimension_d,
- short2(tile_block_cols, tile_block_rows)
- );
+ accumulator.store_result_safe(d, params->leading_dimension_d, short2(tile_block_cols, tile_block_rows));
}
if (needs_rht) {
@@ -241,17 +213,13 @@ struct SimdgroupMmaCore {
) {
const uint partition = thread_context.threadgroup_position.z;
const uint tile_row = thread_context.threadgroup_position.y;
- const uint2 tile =
- tile_id(uint2(thread_context.threadgroup_position.x, tile_row), params);
- const auto geometry =
- ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::
- compute(tile, params);
+ const uint2 tile = tile_id(uint2(thread_context.threadgroup_position.x, tile_row), params);
+ const auto geometry = ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::compute(tile, params);
if (geometry.out_of_bounds) {
return;
}
- const uint k_offset =
- partition * params->aligned_inner_iterations * THREADGROUP_BLOCK_K;
+ const uint k_offset = partition * params->aligned_inner_iterations * THREADGROUP_BLOCK_K;
threadgroup_barrier(mem_flags::mem_none);
@@ -259,27 +227,20 @@ struct SimdgroupMmaCore {
const size_t block_col = size_t(geometry.block_col_start);
a += block_row * params->leading_dimension_a + k_offset;
- d += size_t(partition) * size_t(params->M) * size_t(params->N) +
- block_row * params->leading_dimension_d + block_col;
+ d +=
+ size_t(partition) * size_t(params->M) * size_t(params->N) + block_row * params->leading_dimension_d + block_col;
- thread ALoader
- loader_a(a, params->leading_dimension_a, a_shared, thread_context);
+ thread ALoader loader_a(a, params->leading_dimension_a, a_shared, thread_context);
thread TileAccumulator accumulator(thread_context);
const ushort tile_block_rows =
- min(THREADGROUP_BLOCK_M,
- static_cast<int>(params->M) -
- static_cast<int>(geometry.block_row_start));
+ min(THREADGROUP_BLOCK_M, static_cast<int>(params->M) - static_cast<int>(geometry.block_row_start));
const ushort tile_block_cols =
- min(THREADGROUP_BLOCK_N,
- static_cast<int>(params->N) -
- static_cast<int>(geometry.block_col_start));
- const ushort leftover_block_depth =
- params->K - params->aligned_inner_iterations * THREADGROUP_BLOCK_K;
+ min(THREADGROUP_BLOCK_N, static_cast<int>(params->N) - static_cast<int>(geometry.block_col_start));
+ const ushort leftover_block_depth = params->K - params->aligned_inner_iterations * THREADGROUP_BLOCK_K;
const bool needs_scale = output_transform.contains(GemmDTransform::SCALE);
- const bool needs_accumulate =
- output_transform.contains(GemmDTransform::ACCUMULATE);
+ const bool needs_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);
const bool needs_bias = output_transform.contains(GemmDTransform::BIAS);
const bool needs_rht = output_transform.contains(GemmDTransform::RHT);
const bool needs_epilogue = needs_scale || needs_accumulate;
@@ -291,35 +252,23 @@ struct SimdgroupMmaCore {
auto loader_b = [&]() {
if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
- const device BT* b_block_fp =
- b +
- (TRANSPOSE_B ? block_col * params->leading_dimension_b
- : block_col) +
- (TRANSPOSE_B ? k_offset : k_offset * params->leading_dimension_b);
- return BLoaderFp(
- b_block_fp,
- params->leading_dimension_b,
- b_shared,
- thread_context
- );
+ const device BT* b_block_fp = b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) +
+ (TRANSPOSE_B ? k_offset : k_offset * params->leading_dimension_b);
+ return BLoaderFp(b_block_fp, params->leading_dimension_b, b_shared, thread_context);
} else {
constexpr int pack_factor = get_pack_factor<BITS, 8>();
constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
const int k_elements = int(params->K);
- const int weights_row_stride_bytes =
- k_elements * bytes_per_pack / pack_factor;
+ const int weights_row_stride_bytes = k_elements * bytes_per_pack / pack_factor;
const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
const int k_offset_groups = int(k_offset) / GROUP_SIZE;
- const device uint8_t* weights_block =
- reinterpret_cast<const device uint8_t*>(b) +
- block_col * weights_row_stride_bytes +
- int(k_offset) * bytes_per_pack / pack_factor;
- const device BT* scales_offset =
- scales + block_col * groups_per_row + k_offset_groups;
+ const device uint8_t* weights_block = reinterpret_cast<const device uint8_t*>(b) +
+ block_col * weights_row_stride_bytes +
+ int(k_offset) * bytes_per_pack / pack_factor;
+ const device BT* scales_offset = scales + block_col * groups_per_row + k_offset_groups;
if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
- const device BT* biases_offset =
- biases + block_col * groups_per_row + k_offset_groups;
+ const device BT* biases_offset = biases + block_col * groups_per_row + k_offset_groups;
return BLoaderScaleBias(
weights_block,
scales_offset,
@@ -329,14 +278,10 @@ struct SimdgroupMmaCore {
thread_context.simdgroup_index,
thread_context.simd_lane_id
);
- } else if constexpr (
- B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant
- ) {
- const int zero_point_stride_per_row =
- (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;
- const device uint8_t* zero_points_row_start =
- zero_points + block_col * zero_point_stride_per_row +
- ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);
+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {
+ const int zero_point_stride_per_row = (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;
+ const device uint8_t* zero_points_row_start = zero_points + block_col * zero_point_stride_per_row +
+ ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);
return BLoaderScaleZeroPoint(
weights_block,
scales_offset,
@@ -362,22 +307,14 @@ struct SimdgroupMmaCore {
}
}();
- const bool all_aligned = ((alignment.contains(GemmAlignment::M)) ||
- (tile_block_rows == THREADGROUP_BLOCK_M)) &&
- ((alignment.contains(GemmAlignment::N)) ||
- (tile_block_cols == THREADGROUP_BLOCK_N)) &&
+ const bool all_aligned = ((alignment.contains(GemmAlignment::M)) || (tile_block_rows == THREADGROUP_BLOCK_M)) &&
+ ((alignment.contains(GemmAlignment::N)) || (tile_block_cols == THREADGROUP_BLOCK_N)) &&
alignment.contains(GemmAlignment::K);
- constexpr uint MASK_ALL = static_cast<uint>(GemmAlignment::M) |
- static_cast<uint>(GemmAlignment::N) |
- static_cast<uint>(GemmAlignment::K);
+ constexpr uint MASK_ALL =
+ static_cast<uint>(GemmAlignment::M) | static_cast<uint>(GemmAlignment::N) | static_cast<uint>(GemmAlignment::K);
const uint dynamic_alignment_mask =
- alignment.raw_value |
- ((tile_block_rows == THREADGROUP_BLOCK_M)
- ? static_cast<uint>(GemmAlignment::M)
- : 0u) |
- ((tile_block_cols == THREADGROUP_BLOCK_N)
- ? static_cast<uint>(GemmAlignment::N)
- : 0u);
+ alignment.raw_value | ((tile_block_rows == THREADGROUP_BLOCK_M) ? static_cast<uint>(GemmAlignment::M) : 0u) |
+ ((tile_block_cols == THREADGROUP_BLOCK_N) ? static_cast<uint>(GemmAlignment::N) : 0u);
auto kernel_invoke = [&](auto gemm_alignment_mask) {
constexpr uint gemm_alignment = gemm_alignment_mask.value;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 62046f8b8..66fe494e6 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -10,18 +10,12 @@
using namespace metal;
using namespace uzu::gemm;
-#define GEMM_MXU_QUANT \
- (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision)
-#define GEMM_TGA_ELEMENTS \
- ((USE_MXU) ? 1 \
- : (gemm_tiling_block_m(GEMM_TILING) * \
- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))
-#define GEMM_TGB_ELEMENTS \
- ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * \
- (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) \
- : 1) \
- : (gemm_tiling_block_n(GEMM_TILING) * \
- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))
+#define GEMM_MXU_QUANT (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision)
+#define GEMM_TGA_ELEMENTS \
+ ((USE_MXU) ? 1 : (gemm_tiling_block_m(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))
+#define GEMM_TGB_ELEMENTS \
+ ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) : 1) \
+ : (gemm_tiling_block_n(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))
template <
typename AT,
@@ -117,51 +111,37 @@ KERNEL(Gemm)(
(void)thread_z;
if constexpr (USE_MXU) {
- MxuMmaCore<
- AT,
- BT,
- DT,
- GEMM_TILING,
- TRANSPOSE_B,
- B_PROLOGUE,
- BITS,
- GROUP_SIZE>::
- run(a,
- b,
- d,
- params,
- alignment,
- output_transform,
- scales,
- biases,
- zero_points,
- output_bias,
- rht_factors,
- b_shared,
- thread_context);
+ MxuMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B, B_PROLOGUE, BITS, GROUP_SIZE>::run(
+ a,
+ b,
+ d,
+ params,
+ alignment,
+ output_transform,
+ scales,
+ biases,
+ zero_points,
+ output_bias,
+ rht_factors,
+ b_shared,
+ thread_context
+ );
} else {
- SimdgroupMmaCore<
- AT,
- BT,
- DT,
- GEMM_TILING,
- TRANSPOSE_B,
- B_PROLOGUE,
- BITS,
- GROUP_SIZE>::
- run(a,
- b,
- d,
- params,
- alignment,
- output_transform,
- scales,
- biases,
- zero_points,
- output_bias,
- rht_factors,
- a_shared,
- b_shared,
- thread_context);
+ SimdgroupMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B, B_PROLOGUE, BITS, GROUP_SIZE>::run(
+ a,
+ b,
+ d,
+ params,
+ alignment,
+ output_transform,
+ scales,
+ biases,
+ zero_points,
+ output_bias,
+ rht_factors,
+ a_shared,
+ b_shared,
+ thread_context
+ );
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
index cc579546d..ef20f04a3 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
@@ -26,28 +26,21 @@ KERNEL(GemmSplitKReduce)(
(void)threadgroup_index;
(void)thread_index_in_threadgroup;
- const uint threads_per_threadgroup =
- thread_context.simdgroups_per_threadgroup * thread_context.simdgroup_size;
+ const uint threads_per_threadgroup = thread_context.simdgroups_per_threadgroup * thread_context.simdgroup_size;
const uint local_thread_index =
- thread_context.simdgroup_index * thread_context.simdgroup_size +
- thread_context.simd_lane_id;
+ thread_context.simdgroup_index * thread_context.simdgroup_size + thread_context.simd_lane_id;
const uint vector_count = element_count / 4u;
- const uint vector_index =
- thread_context.threadgroup_position.x * threads_per_threadgroup +
- local_thread_index;
+ const uint vector_index = thread_context.threadgroup_position.x * threads_per_threadgroup + local_thread_index;
if (vector_index >= vector_count) {
return;
}
- device vec<T, 4>* output_vectors =
- reinterpret_cast<device vec<T, 4>*>(output);
- const device vec<T, 4>* partial_sum_vectors =
- reinterpret_cast<const device vec<T, 4>*>(partial_sums);
+ device vec<T, 4>* output_vectors = reinterpret_cast<device vec<T, 4>*>(output);
+ const device vec<T, 4>* partial_sum_vectors = reinterpret_cast<const device vec<T, 4>*>(partial_sums);
float4 accumulator = float4(0.0f);
for (uint partition = 0u; partition < partition_count; ++partition) {
- accumulator +=
- float4(partial_sum_vectors[partition * vector_count + vector_index]);
+ accumulator += float4(partial_sum_vectors[partition * vector_count + vector_index]);
}
if (output_transform.contains(GemmDTransform::SCALE)) {
@@ -58,9 +51,7 @@ KERNEL(GemmSplitKReduce)(
}
if (output_transform.contains(GemmDTransform::BIAS)) {
const uint column = (vector_index * 4u) % column_count;
- accumulator += float4(
- *reinterpret_cast<const device vec<T, 4>*>(output_bias + column)
- );
+ accumulator += float4(*reinterpret_cast<const device vec<T, 4>*>(output_bias + column));
}
output_vectors[vector_index] = vec<T, 4>(accumulator);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h
index a9fa82495..070ccaf06 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h
@@ -40,42 +40,24 @@ struct BSource {
const uint k_start = k_slice * block_size;
const device BT* w = reinterpret_cast<const device BT*>(b);
w += out_row * in_vec_size + simd_lane * values_per_thread + k_start;
- const device AT* in =
- a + batch_idx * in_vec_size + simd_lane * values_per_thread + k_start;
+ const device AT* in = a + batch_idx * in_vec_size + simd_lane * values_per_thread + k_start;
uint k = k_start;
for (; k + block_size <= in_vec_size; k += k_stride) {
- float4 xv =
- static_cast<float4>(*reinterpret_cast<const device I4*>(in));
- result[0] +=
- dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w)),
- xv);
- result[1] +=
- dot(static_cast<float4>(
- *reinterpret_cast<const device W4*>(w + in_vec_size)
- ),
- xv);
- result[2] +=
- dot(static_cast<float4>(
- *reinterpret_cast<const device W4*>(w + 2 * in_vec_size)
- ),
- xv);
- result[3] +=
- dot(static_cast<float4>(
- *reinterpret_cast<const device W4*>(w + 3 * in_vec_size)
- ),
- xv);
+ float4 xv = static_cast<float4>(*reinterpret_cast<const device I4*>(in));
+ result[0] += dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w)), xv);
+ result[1] += dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w + in_vec_size)), xv);
+ result[2] += dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w + 2 * in_vec_size)), xv);
+ result[3] += dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w + 3 * in_vec_size)), xv);
w += k_stride;
in += k_stride;
}
if constexpr (K_SPLIT == 1 && !INPUT_ALIGNED) {
const uint thread_offset = simd_lane * values_per_thread;
- const int remaining =
- (k + thread_offset < in_vec_size)
- ? min(static_cast<int>(in_vec_size - k - thread_offset),
- static_cast<int>(values_per_thread))
- : 0;
+ const int remaining = (k + thread_offset < in_vec_size) ? min(static_cast<int>(in_vec_size - k - thread_offset),
+ static_cast<int>(values_per_thread))
+ : 0;
if (remaining > 0) {
const device BT* w0 = w;
const device BT* w1 = w + in_vec_size;
@@ -102,8 +84,7 @@ struct BSource {
const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
const uint in_vec_size_g = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;
- ws += out_row * in_vec_size_w +
- simd_lane * packs_per_thread * bytes_per_pack;
+ ws += out_row * in_vec_size_w + simd_lane * packs_per_thread * bytes_per_pack;
QuantRowOffsets<BT, U, B_PROLOGUE, BITS> prep;
prep.group_stride = in_vec_size_g;
@@ -111,9 +92,7 @@ struct BSource {
prep.scales = scales + out_row * in_vec_size_g + g_offset;
if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
prep.biases = biases + out_row * in_vec_size_g + g_offset;
- } else if constexpr (
- B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant
- ) {
+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {
if (BITS == 4) {
prep.zp_stride = (in_vec_size_g + 1) / 2;
prep.zps = zero_points + out_row * prep.zp_stride + g_offset / 2;
@@ -124,8 +103,7 @@ struct BSource {
}
}
- const device AT* in =
- a + batch_idx * in_vec_size + simd_lane * values_per_thread;
+ const device AT* in = a + batch_idx * in_vec_size + simd_lane * values_per_thread;
uint k = 0;
for (; k + block_size <= in_vec_size; k += block_size) {
@@ -139,34 +117,10 @@ struct BSource {
U scale[4];
U offset[4];
prep.load(scale, offset);
- result[0] += qdot<U, values_per_thread, BITS>(
- wl0,
- x_thread,
- scale[0],
- offset[0],
- sum
- );
- result[1] += qdot<U, values_per_thread, BITS>(
- wl1,
- x_thread,
- scale[1],
- offset[1],
- sum
- );
- result[2] += qdot<U, values_per_thread, BITS>(
- wl2,
- x_thread,
- scale[2],
- offset[2],
- sum
- );
- result[3] += qdot<U, values_per_thread, BITS>(
- wl3,
- x_thread,
- scale[3],
- offset[3],
- sum
- );
+ result[0] += qdot<U, values_per_thread, BITS>(wl0, x_thread, scale[0], offset[0], sum);
+ result[1] += qdot<U, values_per_thread, BITS>(wl1, x_thread, scale[1], offset[1], sum);
+ result[2] += qdot<U, values_per_thread, BITS>(wl2, x_thread, scale[2], offset[2], sum);
+ result[3] += qdot<U, values_per_thread, BITS>(wl3, x_thread, scale[3], offset[3], sum);
ws += block_size * bytes_per_pack / pack_factor;
prep.advance(block_size / GROUP_SIZE);
@@ -175,17 +129,11 @@ struct BSource {
if constexpr (!INPUT_ALIGNED) {
const uint thread_offset = simd_lane * values_per_thread;
- const int remaining =
- (k + thread_offset < in_vec_size)
- ? min(static_cast<int>(in_vec_size - k - thread_offset),
- static_cast<int>(values_per_thread))
- : 0;
+ const int remaining = (k + thread_offset < in_vec_size) ? min(static_cast<int>(in_vec_size - k - thread_offset),
+ static_cast<int>(values_per_thread))
+ : 0;
if (remaining > 0) {
- U sum = load_vector_safe<AT, U, values_per_thread>(
- in,
- x_thread,
- remaining
- );
+ U sum = load_vector_safe<AT, U, values_per_thread>(in, x_thread, remaining);
const device uint8_t* wl0 = ws;
const device uint8_t* wl1 = ws + in_vec_size_w;
@@ -195,38 +143,10 @@ struct BSource {
U scale[4];
U offset[4];
prep.load(scale, offset);
- result[0] += qdot_safe<U, values_per_thread, BITS>(
- wl0,
- x_thread,
- scale[0],
- offset[0],
- sum,
- remaining
- );
- result[1] += qdot_safe<U, values_per_thread, BITS>(
- wl1,
- x_thread,
- scale[1],
- offset[1],
- sum,
- remaining
- );
- result[2] += qdot_safe<U, values_per_thread, BITS>(
- wl2,
- x_thread,
- scale[2],
- offset[2],
- sum,
- remaining
- );
- result[3] += qdot_safe<U, values_per_thread, BITS>(
- wl3,
- x_thread,
- scale[3],
- offset[3],
- sum,
- remaining
- );
+ result[0] += qdot_safe<U, values_per_thread, BITS>(wl0, x_thread, scale[0], offset[0], sum, remaining);
+ result[1] += qdot_safe<U, values_per_thread, BITS>(wl1, x_thread, scale[1], offset[1], sum, remaining);
+ result[2] += qdot_safe<U, values_per_thread, BITS>(wl2, x_thread, scale[2], offset[2], sum, remaining);
+ result[3] += qdot_safe<U, values_per_thread, BITS>(wl3, x_thread, scale[3], offset[3], sum, remaining);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
index 1babf6c5a..72707af54 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
@@ -24,8 +24,7 @@ struct Epilogue {
bool writer
) {
const bool is_scale = output_transform.contains(GemmDTransform::SCALE);
- const bool is_accumulate =
- output_transform.contains(GemmDTransform::ACCUMULATE);
+ const bool is_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);
const bool is_bias = output_transform.contains(GemmDTransform::BIAS);
const bool use_hadamard = output_transform.contains(GemmDTransform::RHT);
@@ -51,8 +50,7 @@ struct Epilogue {
if (simd_lane == 0) {
METAL_PRAGMA_UNROLL
for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] =
- result[row];
+ shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] = result[row];
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h
index 6a28cbbb5..6fba89d29 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h
@@ -12,17 +12,12 @@ struct OutputTile {
uint k_slice;
bool writer;
- static METAL_FUNC OutputTile
- make(uint out_block_idx, uint simd_group, uint out_vec_size) {
- constexpr uint rows_per_threadgroup =
- (NUM_SIMDGROUPS / K_SPLIT) * RESULTS_PER_SIMDGROUP;
+ static METAL_FUNC OutputTile make(uint out_block_idx, uint simd_group, uint out_vec_size) {
+ constexpr uint rows_per_threadgroup = (NUM_SIMDGROUPS / K_SPLIT) * RESULTS_PER_SIMDGROUP;
const uint row_group = simd_group / K_SPLIT;
const uint k_slice = simd_group % K_SPLIT;
- uint out_row = out_block_idx * rows_per_threadgroup +
- row_group * RESULTS_PER_SIMDGROUP;
- out_row = out_vec_size > RESULTS_PER_SIMDGROUP
- ? min(out_row, out_vec_size - RESULTS_PER_SIMDGROUP)
- : 0u;
+ uint out_row = out_block_idx * rows_per_threadgroup + row_group * RESULTS_PER_SIMDGROUP;
+ out_row = out_vec_size > RESULTS_PER_SIMDGROUP ? min(out_row, out_vec_size - RESULTS_PER_SIMDGROUP) : 0u;
OutputTile tile;
tile.out_row = out_row;
tile.row_group = row_group;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h
index 0965132c0..d68cd99ca 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h
@@ -8,10 +8,7 @@ namespace gemm {
template <typename BT, typename U, GemmBPrologueKind B_PROLOGUE, uint BITS>
struct QuantRowOffsets {
- static_assert(
- BITS == 4 || BITS == 8,
- "QuantRowOffsets supports 4- and 8-bit only"
- );
+ static_assert(BITS == 4 || BITS == 8, "QuantRowOffsets supports 4- and 8-bit only");
const device BT* scales = nullptr;
const device BT* biases = nullptr;
const device uint8_t* zps = nullptr;
@@ -30,15 +27,8 @@ struct QuantRowOffsets {
offset[1] = static_cast<U>(biases[group_stride]);
offset[2] = static_cast<U>(biases[2 * group_stride]);
offset[3] = static_cast<U>(biases[3 * group_stride]);
- } else if constexpr (
- B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant
- ) {
- uchar4 zp_bytes = uchar4(
- zps[0],
- zps[zp_stride],
- zps[2 * zp_stride],
- zps[3 * zp_stride]
- );
+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {
+ uchar4 zp_bytes = uchar4(zps[0], zps[zp_stride], zps[2 * zp_stride], zps[3 * zp_stride]);
uchar4 zp_nibbles;
if constexpr (BITS == 4) {
const uint8_t shift = high_nibble ? 4u : 0u;
@@ -63,9 +53,7 @@ struct QuantRowOffsets {
scales += groups;
if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
biases += groups;
- } else if constexpr (
- B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant
- ) {
+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {
constexpr uint zps_per_byte = get_pack_factor<BITS, 8>();
zps += groups / zps_per_byte;
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h
index d08ad70bb..ac3947b3a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h
@@ -24,8 +24,7 @@ struct Reduce {
if (simd_lane == 0) {
METAL_PRAGMA_UNROLL
for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] =
- result[row];
+ shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] = result[row];
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
@@ -35,8 +34,7 @@ struct Reduce {
U acc = 0;
METAL_PRAGMA_UNROLL
for (uint s = 0; s < K_SPLIT; s++) {
- acc += shared_results
- [(row_group * K_SPLIT + s) * RESULTS_PER_SIMDGROUP + row];
+ acc += shared_results[(row_group * K_SPLIT + s) * RESULTS_PER_SIMDGROUP + row];
}
result[row] = acc;
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 0aa6c8cf5..e69aa5cc2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -68,37 +68,25 @@ KERNEL(Gemv)(
thread U result[RESULTS_PER_SIMDGROUP] = {0};
OutputTile<K_SPLIT, NUM_SIMDGROUPS> tile =
- OutputTile<K_SPLIT, NUM_SIMDGROUPS>::make(
- out_block_idx,
- simd_group,
- out_vec_size
- );
+ OutputTile<K_SPLIT, NUM_SIMDGROUPS>::make(out_block_idx, simd_group, out_vec_size);
d += batch_idx * out_vec_size + tile.out_row;
- BSource<BT, AT, U, B_PROLOGUE, GROUP_SIZE, BITS, K_SPLIT, INPUT_ALIGNED>::
- accumulate(
- result,
- b,
- scales,
- zero_points,
- biases,
- a,
- in_vec_size,
- tile.out_row,
- batch_idx,
- simd_lane,
- tile.k_slice
- );
-
- Reduce<U, K_SPLIT, NUM_SIMDGROUPS>::run(
+ BSource<BT, AT, U, B_PROLOGUE, GROUP_SIZE, BITS, K_SPLIT, INPUT_ALIGNED>::accumulate(
result,
- shared_results,
- simd_group,
+ b,
+ scales,
+ zero_points,
+ biases,
+ a,
+ in_vec_size,
+ tile.out_row,
+ batch_idx,
simd_lane,
- tile.row_group,
tile.k_slice
);
+ Reduce<U, K_SPLIT, NUM_SIMDGROUPS>::run(result, shared_results, simd_group, simd_lane, tile.row_group, tile.k_slice);
+
Epilogue<BT, DT, U>::store(
result,
d,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
index 8b10d91f3..b82f05de8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
@@ -6,11 +6,7 @@
#include "../common/threadgroup_reduce.h"
template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_raking_prefix_exclusive_sum(
- T value,
- threadgroup T* shared,
- const ushort lid
-) {
+static T threadgroup_raking_prefix_exclusive_sum(T value, threadgroup T* shared, const ushort lid) {
shared[lid] = value;
threadgroup_barrier(mem_flags::mem_threadgroup);
@@ -48,16 +44,14 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(
device const int* topk_ids,
device uint* offsets, // output: exclusive scan [E+1]
device uint* sum_k_out, // output: total count [1]
- device uint*
- partials, // output: partials [num_tiles * TILE_E] (for block_bases)
+ device uint* partials, // output: partials [num_tiles * TILE_E] (for block_bases)
constant uint& t_input,
constant uint& e_input,
constant uint& k_input,
threadgroup _atomic<uint> tg_hist[TILE_E],
threadgroup uint scan_shared[BLOCK_SIZE],
threadgroup uint reduce_shared[BLOCK_SIZE],
- threadgroup uint
- counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory
+ threadgroup uint counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory
threadgroup uint& carry,
const ThreadContext thread_context,
const uint lid THREADS(128)
@@ -102,8 +96,7 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(
// Write final counts and partials for this tile
for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {
- const uint count_val =
- atomic_load_explicit(&tg_hist[e], memory_order_relaxed);
+ const uint count_val = atomic_load_explicit(&tg_hist[e], memory_order_relaxed);
counts_shared[e0 + e] = count_val;
partials[e0 + e] = count_val;
}
@@ -124,22 +117,13 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(
uint v = (lid < chunk_n) ? counts_shared[lid] : 0u;
- uint prefix_local = threadgroup_raking_prefix_exclusive_sum<BLOCK_SIZE>(
- v,
- scan_shared,
- (ushort)lid
- );
+ uint prefix_local = threadgroup_raking_prefix_exclusive_sum<BLOCK_SIZE>(v, scan_shared, (ushort)lid);
uint prefix_global = prefix_local + carry;
if (lid < chunk_n) {
offsets[base + lid] = prefix_global;
}
- uint block_sum =
- threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(
- v,
- reduce_shared,
- thread_context
- );
+ uint block_sum = threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(v, reduce_shared, thread_context);
if (lid == 0) {
carry += block_sum;
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal
index 75e029e9b..9f4022cee 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal
@@ -46,8 +46,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
const ulong bias_base = (ulong)expert_u * (ulong)(2 * d_ff);
device const T* w_up_row = w13_all + w13_base + (ulong)h_idx * (ulong)d_model;
- device const T* w_gate_row =
- w13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
+ device const T* w_gate_row = w13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
float acc_up = 0.0f;
float acc_gate = 0.0f;
@@ -58,16 +57,14 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
uint base_idx = i * 128 + thread_context.simd_lane_id * 4;
device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);
- device const T* w_up_vec2 =
- reinterpret_cast<device const T*>(w_up_row + base_idx);
+ device const T* w_up_vec2 = reinterpret_cast<device const T*>(w_up_row + base_idx);
acc_up += float(x_vec2[0]) * float(w_up_vec2[0]);
acc_up += float(x_vec2[1]) * float(w_up_vec2[1]);
acc_up += float(x_vec2[2]) * float(w_up_vec2[2]);
acc_up += float(x_vec2[3]) * float(w_up_vec2[3]);
if (gating_sel > 1) {
- device const T* w_gate_vec2 =
- reinterpret_cast<device const T*>(w_gate_row + base_idx);
+ device const T* w_gate_vec2 = reinterpret_cast<device const T*>(w_gate_row + base_idx);
acc_gate += float(x_vec2[0]) * float(w_gate_vec2[0]);
acc_gate += float(x_vec2[1]) * float(w_gate_vec2[1]);
acc_gate += float(x_vec2[2]) * float(w_gate_vec2[2]);
@@ -90,25 +87,14 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
}
if (thread_context.simd_lane_id == 0) {
- float up_val = clamp(
- acc_up + float(biases[bias_base + h_idx]),
- up_clip_min,
- up_clip_max
- );
+ float up_val = clamp(acc_up + float(biases[bias_base + h_idx]), up_clip_min, up_clip_max);
float activated;
if (gating_sel <= 1) {
- activated = (gating_sel == 0) ? activate_gelu(up_val)
- : activate_silu_alpha(up_val, silu_alpha);
+ activated = (gating_sel == 0) ? activate_gelu(up_val) : activate_silu_alpha(up_val, silu_alpha);
} else {
- float gate_val = clamp(
- acc_gate + float(biases[bias_base + d_ff + h_idx]),
- gate_clip_min,
- gate_clip_max
- );
- float gate_act = (gating_sel == 2)
- ? activate_silu_alpha(gate_val, silu_alpha)
- : activate_gelu(gate_val);
+ float gate_val = clamp(acc_gate + float(biases[bias_base + d_ff + h_idx]), gate_clip_min, gate_clip_max);
+ float gate_act = (gating_sel == 2) ? activate_silu_alpha(gate_val, silu_alpha) : activate_gelu(gate_val);
activated = gate_act * up_val;
}
@@ -153,8 +139,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
const float prob = float(topk_probs[k]);
device const float* hidden_ptr = hidden + (ulong)k * (ulong)d_ff;
- device const T* w2_ptr = w2_all + (ulong)expert_u * w2_expert_stride +
- (ulong)my_col * (ulong)d_ff;
+ device const T* w2_ptr = w2_all + (ulong)expert_u * w2_expert_stride + (ulong)my_col * (ulong)d_ff;
float acc = 0.0f;
@@ -162,10 +147,8 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
for (uint i = 0; i < vec_iters; ++i) {
uint base_idx = i * 128 + thread_context.simd_lane_id * 4;
- float4 h_vec =
- *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
- device const T* w_vec2 =
- reinterpret_cast<device const T*>(w2_ptr + base_idx);
+ float4 h_vec = *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
+ device const T* w_vec2 = reinterpret_cast<device const T*>(w2_ptr + base_idx);
acc += h_vec.x * float(w_vec2[0]);
acc += h_vec.y * float(w_vec2[1]);
acc += h_vec.z * float(w_vec2[2]);
@@ -173,8 +156,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
}
// Remainder
- for (uint idx = vec_iters * 128 + thread_context.simd_lane_id; idx < d_ff;
- idx += 32) {
+ for (uint idx = vec_iters * 128 + thread_context.simd_lane_id; idx < d_ff; idx += 32) {
acc += hidden_ptr[idx] * float(w2_ptr[idx]);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index 4a46fe2a0..dabc1a252 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -56,8 +56,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
device const T* x_ptr = x_perm + x_row_base;
device const T* w_up_row = w13_all + w13_base + (ulong)h_idx * (ulong)d_model;
- device const T* w_gate_row =
- w13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
+ device const T* w_gate_row = w13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
float acc_up = 0.0f;
float acc_gate = 0.0f;
@@ -69,16 +68,14 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
uint base_idx = i * 128 + thread_context.simd_lane_id * 4;
device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);
- device const T* w_up_vec =
- reinterpret_cast<device const T*>(w_up_row + base_idx);
+ device const T* w_up_vec = reinterpret_cast<device const T*>(w_up_row + base_idx);
acc_up += float(x_vec[0]) * float(w_up_vec[0]);
acc_up += float(x_vec[1]) * float(w_up_vec[1]);
acc_up += float(x_vec[2]) * float(w_up_vec[2]);
acc_up += float(x_vec[3]) * float(w_up_vec[3]);
if (gating_sel > 1) {
- device const T* w_gate_vec =
- reinterpret_cast<device const T*>(w_gate_row + base_idx);
+ device const T* w_gate_vec = reinterpret_cast<device const T*>(w_gate_row + base_idx);
acc_gate += float(x_vec[0]) * float(w_gate_vec[0]);
acc_gate += float(x_vec[1]) * float(w_gate_vec[1]);
acc_gate += float(x_vec[2]) * float(w_gate_vec[2]);
@@ -109,14 +106,11 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
float activated;
if (gating_sel <= 1) {
- activated = (gating_sel == 0) ? activate_gelu(up_val)
- : activate_silu_alpha(up_val, silu_alpha);
+ activated = (gating_sel == 0) ? activate_gelu(up_val) : activate_silu_alpha(up_val, silu_alpha);
} else {
float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);
gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);
- float gate_act = (gating_sel == 2)
- ? activate_silu_alpha(gate_val, silu_alpha)
- : activate_gelu(gate_val);
+ float gate_act = (gating_sel == 2) ? activate_silu_alpha(gate_val, silu_alpha) : activate_gelu(gate_val);
activated = gate_act * up_val;
}
@@ -151,8 +145,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
const uint row_idx = tgpig_y;
// Each simdgroup computes one output column
- const uint my_col =
- tgpig_x * SIMDGROUPS_PER_TG + thread_context.simdgroup_index;
+ const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + thread_context.simdgroup_index;
if (my_col >= d_model)
return;
@@ -160,8 +153,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
// Base addresses for this output column
const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;
- const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff +
- (ulong)my_col * (ulong)d_ff;
+ const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff + (ulong)my_col * (ulong)d_ff;
// Dual accumulators for ILP: breaks FMA dependency chains
AccumT acc0 = AccumT(0.0);
@@ -173,8 +165,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
const uint k_vec_iters = k_iters / 8;
for (uint iter = 0; iter < k_vec_iters; ++iter) {
- const uint k_base =
- iter * (8 * THREADS_PER_SIMD) + thread_context.simd_lane_id;
+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + thread_context.simd_lane_id;
// hidden: stride-32 per lane, already f32 from Pass A
const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
@@ -187,22 +178,14 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];
// W2: lane-coalesced
- const AccumT w0 =
- AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
- const AccumT w1 =
- AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
- const AccumT w2 =
- AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
- const AccumT w3 =
- AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
- const AccumT w4 =
- AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
- const AccumT w5 =
- AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
- const AccumT w6 =
- AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
- const AccumT w7 =
- AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
+ const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
+ const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
+ const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
+ const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
+ const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
+ const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
+ const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
+ const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
// dual trees for ILP
acc0 = fma(h0, w0, acc0);
@@ -221,20 +204,14 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
// Handle remaining full iterations
for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
const uint k = iter * THREADS_PER_SIMD + thread_context.simd_lane_id;
- acc =
- fma(AccumT(hidden[hidden_base + k]),
- AccumT(w2_all[w2_col_base + k]),
- acc);
+ acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);
}
// Handle leftover elements (d_ff % 32)
const uint leftover_start = k_iters * THREADS_PER_SIMD;
if (leftover_start + thread_context.simd_lane_id < d_ff) {
const uint k = leftover_start + thread_context.simd_lane_id;
- acc =
- fma(AccumT(hidden[hidden_base + k]),
- AccumT(w2_all[w2_col_base + k]),
- acc);
+ acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);
}
// Simdgroup reduction
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index bccc6aab4..0d7ab3bbc 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -7,9 +7,7 @@
// ------------------------ helpers ------------------------
static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }
-static inline uint linear_tid(uint3 tid, uint3 tpg) {
- return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;
-}
+static inline uint linear_tid(uint3 tid, uint3 tpg) { return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x; }
template <typename T>
inline void store_vec4(device T* dst, ulong base, float4 vals) {
@@ -98,10 +96,8 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
const uint m_rows = min(Bm, seg_len - row_tg_off);
const uint n_cols = min(Bn, d_ff - col_tg_off);
- const ulong x_base =
- ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;
- const ulong w13_expert_base =
- (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;
+ const ulong x_base = ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;
+ const ulong w13_expert_base = (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;
const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
// simdgroup tile mapping
@@ -128,8 +124,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
}
}
- const uint threads_total =
- threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
+ const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
const uint lin = linear_tid(local_tid, threads_per_tg);
// K-loop (tile along D)
@@ -157,13 +152,10 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
if (r < m_rows && col_base < valid_k) {
// Safe to read - at least first element is valid
- const ulong base =
- x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);
+ const ulong base = x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);
METAL_PRAGMA_UNROLL
for (uint j = 0; j < vec_size; j++) {
- dst[j] = (col_base + j < valid_k)
- ? static_cast<T>(float(x_perm[base + j]))
- : static_cast<T>(0.0f);
+ dst[j] = (col_base + j < valid_k) ? static_cast<T>(float(x_perm[base + j])) : static_cast<T>(0.0f);
}
} else {
// Row out of bounds or entire vec8 chunk is out of K bounds
@@ -198,21 +190,14 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
if (n < n_cols && col_base < valid_k) {
// Safe to read - at least first element is valid
const uint gcol = col_tg_off + n;
- const ulong up_base = w13_expert_base +
- (ulong)gcol * (ulong)d_model +
- (ulong)(k_off + col_base);
- const ulong gate_base = w13_expert_base +
- (ulong)(d_ff + gcol) * (ulong)d_model +
- (ulong)(k_off + col_base);
+ const ulong up_base = w13_expert_base + (ulong)gcol * (ulong)d_model + (ulong)(k_off + col_base);
+ const ulong gate_base = w13_expert_base + (ulong)(d_ff + gcol) * (ulong)d_model + (ulong)(k_off + col_base);
METAL_PRAGMA_UNROLL
for (uint j = 0; j < vec_size; j++) {
- up_dst[j] = (col_base + j < valid_k)
- ? static_cast<T>(float(w13_all[up_base + j]))
- : static_cast<T>(0.0f);
+ up_dst[j] = (col_base + j < valid_k) ? static_cast<T>(float(w13_all[up_base + j])) : static_cast<T>(0.0f);
if (gating_sel > 1u) {
- gt_dst[j] = (col_base + j < valid_k)
- ? static_cast<T>(float(w13_all[gate_base + j]))
- : static_cast<T>(0.0f);
+ gt_dst[j] =
+ (col_base + j < valid_k) ? static_cast<T>(float(w13_all[gate_base + j])) : static_cast<T>(0.0f);
}
}
} else {
@@ -244,35 +229,13 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
const uint c_idx = n_sub / SG_TILE;
const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
metal::simdgroup_matrix<T, 8, 8> rhs_up;
- simdgroup_load(
- rhs_up,
- Wk_up,
- W_LD,
- ulong2(kk, col_sg_off + n_sub),
- true
- );
- simdgroup_multiply_accumulate(
- OutUp[tile],
- lhs_frag,
- rhs_up,
- OutUp[tile]
- );
+ simdgroup_load(rhs_up, Wk_up, W_LD, ulong2(kk, col_sg_off + n_sub), true);
+ simdgroup_multiply_accumulate(OutUp[tile], lhs_frag, rhs_up, OutUp[tile]);
if (gating_sel > 1u) {
metal::simdgroup_matrix<T, 8, 8> rhs_gate;
- simdgroup_load(
- rhs_gate,
- Wk_gate,
- W_LD,
- ulong2(kk, col_sg_off + n_sub),
- true
- );
- simdgroup_multiply_accumulate(
- OutGate[tile],
- lhs_frag,
- rhs_gate,
- OutGate[tile]
- );
+ simdgroup_load(rhs_gate, Wk_gate, W_LD, ulong2(kk, col_sg_off + n_sub), true);
+ simdgroup_multiply_accumulate(OutGate[tile], lhs_frag, rhs_gate, OutGate[tile]);
}
}
}
@@ -283,12 +246,9 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
// Note: bias layout is [up | gate] in contiguous FF chunks.
for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
const uint c_global = col_tg_off + c_local;
- bias_up[c_local] =
- (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;
+ bias_up[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;
if (gating_sel > 1u) {
- bias_gate[c_local] = (c_global < d_ff)
- ? float(up_biases[bias_base + d_ff + c_global])
- : 0.0f;
+ bias_gate[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + d_ff + c_global]) : 0.0f;
}
}
@@ -335,21 +295,13 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
const ulong out_row = seg_start + row_tg_off + local_row;
if (col0 < n_cols) {
- float up_v =
- clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);
+ float up_v = clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);
float out_val;
if (gating_sel <= 1u) {
- out_val = (gating_sel == 0u) ? activate_gelu(up_v)
- : activate_silu_alpha(up_v, silu_alpha);
+ out_val = (gating_sel == 0u) ? activate_gelu(up_v) : activate_silu_alpha(up_v, silu_alpha);
} else {
- float gate_v = clamp(
- gate_frag_0 + bias_gate[col0],
- gate_clip_min,
- gate_clip_max
- );
- const float gate_act = (gating_sel == 2u)
- ? activate_silu_alpha(gate_v, silu_alpha)
- : activate_gelu(gate_v);
+ float gate_v = clamp(gate_frag_0 + bias_gate[col0], gate_clip_min, gate_clip_max);
+ const float gate_act = (gating_sel == 2u) ? activate_silu_alpha(gate_v, silu_alpha) : activate_gelu(gate_v);
out_val = gate_act * up_v;
}
const ulong out_col = col_tg_off + col0;
@@ -357,21 +309,13 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
}
if (col1 < n_cols) {
- float up_v =
- clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);
+ float up_v = clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);
float out_val;
if (gating_sel <= 1u) {
- out_val = (gating_sel == 0u) ? activate_gelu(up_v)
- : activate_silu_alpha(up_v, silu_alpha);
+ out_val = (gating_sel == 0u) ? activate_gelu(up_v) : activate_silu_alpha(up_v, silu_alpha);
} else {
- float gate_v = clamp(
- gate_frag_1 + bias_gate[col1],
- gate_clip_min,
- gate_clip_max
- );
- const float gate_act = (gating_sel == 2u)
- ? activate_silu_alpha(gate_v, silu_alpha)
- : activate_gelu(gate_v);
+ float gate_v = clamp(gate_frag_1 + bias_gate[col1], gate_clip_min, gate_clip_max);
+ const float gate_act = (gating_sel == 2u) ? activate_silu_alpha(gate_v, silu_alpha) : activate_gelu(gate_v);
out_val = gate_act * up_v;
}
const ulong out_col = col_tg_off + col1;
@@ -489,8 +433,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
threadgroup float* my_dst = Hs + h_bi * H_LD + h_bj;
if (h_bi < m_rows && h_bj < valid_k) {
// Safe to read - at least first element is valid
- device const float* my_src =
- hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);
+ device const float* my_src = hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);
METAL_PRAGMA_UNROLL
for (uint j = 0; j < H_VEC; j++) {
my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;
@@ -512,13 +455,10 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
if (row < n_cols && w_bj < valid_k) {
// Safe to read - at least first element is valid
const uint gcol = col_tg_off + row;
- device const T* my_src = w2_all + w2_expert_base +
- (ulong)gcol * (ulong)d_ff +
- (ulong)(k_off + w_bj);
+ device const T* my_src = w2_all + w2_expert_base + (ulong)gcol * (ulong)d_ff + (ulong)(k_off + w_bj);
METAL_PRAGMA_UNROLL
for (uint j = 0; j < W_VEC; j++) {
- my_dst[j] = (w_bj + j < valid_k) ? static_cast<T>(float(my_src[j]))
- : static_cast<T>(0.0f);
+ my_dst[j] = (w_bj + j < valid_k) ? static_cast<T>(float(my_src[j])) : static_cast<T>(0.0f);
}
} else {
METAL_PRAGMA_UNROLL
@@ -556,18 +496,15 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
// ---- Bias tile (cooperative load) ----
for (uint c_local = lin; c_local < Bn; c_local += TGP_SIZE) {
const uint c_global = col_tg_off + c_local;
- bias_tile[c_local] =
- (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;
+ bias_tile[c_local] = (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- Writeout with fragment extraction ----
const uint lane_qid = thread_context.simd_lane_id >> 2;
- const uint lane_row =
- (lane_qid & 4u) + ((thread_context.simd_lane_id >> 1) & 3u);
- const uint lane_col_base =
- ((lane_qid & 2u) * 2u) + ((thread_context.simd_lane_id & 1u) * 2u);
+ const uint lane_row = (lane_qid & 4u) + ((thread_context.simd_lane_id >> 1) & 3u);
+ const uint lane_col_base = ((lane_qid & 2u) * 2u) + ((thread_context.simd_lane_id & 1u) * 2u);
const uint local_row = row_sg_off + lane_row;
if (local_row >= m_rows)
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal
index 78eca3add..7215c157f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal
@@ -43,27 +43,16 @@ PUBLIC KERNEL(MoeRouterTopK)(
const device ScalarT* x_vec = input + (ulong)token_idx * (ulong)vecs * 4;
for (uint c = lid; c < vecs; c += THREADS_PER_TG) {
- x_cache[c] = float4(
- x_vec[c * 4 + 0],
- x_vec[c * 4 + 1],
- x_vec[c * 4 + 2],
- x_vec[c * 4 + 3]
- );
+ x_cache[c] = float4(x_vec[c * 4 + 0], x_vec[c * 4 + 1], x_vec[c * 4 + 2], x_vec[c * 4 + 3]);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint row = thread_context.simdgroup_index; row < e;
- row += thread_context.simdgroups_per_threadgroup) {
+ for (uint row = thread_context.simdgroup_index; row < e; row += thread_context.simdgroups_per_threadgroup) {
const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;
float4 accum4 = float4(0.0f);
for (uint c = thread_context.simd_lane_id; c < vecs; c += 32u) {
- const float4 wv = float4(
- w_vec[c * 4 + 0],
- w_vec[c * 4 + 1],
- w_vec[c * 4 + 2],
- w_vec[c * 4 + 3]
- );
+ const float4 wv = float4(w_vec[c * 4 + 0], w_vec[c * 4 + 1], w_vec[c * 4 + 2], w_vec[c * 4 + 3]);
const float4 xv = x_cache[c];
accum4 = fma(wv, xv, accum4);
}
@@ -89,27 +78,18 @@ PUBLIC KERNEL(MoeRouterTopK)(
uint local_idx = 0xFFFFFFFFu;
for (uint row = lid; row < e; row += THREADS_PER_TG) {
float candidate = logits_shared[row];
- if (candidate > local_best ||
- (candidate == local_best && row < local_idx)) {
+ if (candidate > local_best || (candidate == local_best && row < local_idx)) {
local_best = candidate;
local_idx = row;
}
}
float max_val =
- threadgroup_cooperative_reduce<SimdReduceMax<float>, THREADS_PER_TG>(
- local_best,
- reduce_tmp,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceMax<float>, THREADS_PER_TG>(local_best, reduce_tmp, thread_context);
uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;
uint best_idx =
- threadgroup_cooperative_reduce<SimdReduceMin<uint>, THREADS_PER_TG>(
- candidate_id,
- reduce_tmp_u,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceMin<uint>, THREADS_PER_TG>(candidate_id, reduce_tmp_u, thread_context);
if (lid == 0) {
shared_best_idx[0] = best_idx;
@@ -143,9 +123,7 @@ PUBLIC KERNEL(MoeRouterTopK)(
float default_prob = 1.0f / float(effective_k);
float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;
for (uint i = 0; i < effective_k; ++i) {
- float prob = (sum_exp > 0.0f)
- ? exp(float(out_probs[i]) - max_logit) * inv_sum
- : default_prob;
+ float prob = (sum_exp > 0.0f) ? exp(float(out_probs[i]) - max_logit) * inv_sum : default_prob;
out_probs[i] = static_cast<ScalarT>(prob);
}
for (uint i = effective_k; i < k; ++i) {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
index 860fb1332..1b7a3269a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
@@ -45,11 +45,7 @@ inline void moe_scatter_buckets_impl(
// Zero counts and bases for active tile
for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {
for (uint sg = 0; sg < NUM_SG; ++sg) {
- atomic_store_explicit(
- &sg_counts[sg * TILE_E + te],
- 0u,
- memory_order_relaxed
- );
+ atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);
sg_base[sg * TILE_E + te] = 0u;
}
}
@@ -78,14 +74,12 @@ inline void moe_scatter_buckets_impl(
// Phase 2: compute per-sg bases via prefix across sgs with capacity clamp
for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {
const uint tile_id = e0 / TILE_E;
- const uint base_idx_block =
- (block_id * num_tiles + tile_id) * TILE_E + te;
+ const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;
const uint alloc_total = block_alloc[base_idx_block];
uint prefix = 0u;
for (uint sg = 0; sg < NUM_SG; ++sg) {
const uint idx = sg * TILE_E + te;
- const uint c =
- atomic_load_explicit(&sg_counts[idx], memory_order_relaxed);
+ const uint c = atomic_load_explicit(&sg_counts[idx], memory_order_relaxed);
const uint room = (prefix < alloc_total) ? (alloc_total - prefix) : 0u;
const uint take = min(c, room);
sg_base[idx] = prefix;
@@ -93,11 +87,7 @@ inline void moe_scatter_buckets_impl(
}
// Reuse sg_counts as write indices: reset to zero
for (uint sg = 0; sg < NUM_SG; ++sg) {
- atomic_store_explicit(
- &sg_counts[sg * TILE_E + te],
- 0u,
- memory_order_relaxed
- );
+ atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
@@ -115,19 +105,12 @@ inline void moe_scatter_buckets_impl(
if (ue >= e0 && ue < e0 + tile_e) {
const uint te = ue - e0;
const uint tile_id = e0 / TILE_E;
- const uint base_idx_block =
- (block_id * num_tiles + tile_id) * TILE_E + te;
- const uint idx_sg_te =
- thread_context.simdgroup_index * TILE_E + te;
- const uint local = atomic_fetch_add_explicit(
- &sg_counts[idx_sg_te],
- 1u,
- memory_order_relaxed
- );
+ const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;
+ const uint idx_sg_te = thread_context.simdgroup_index * TILE_E + te;
+ const uint local = atomic_fetch_add_explicit(&sg_counts[idx_sg_te], 1u, memory_order_relaxed);
const uint base_s = sg_base[idx_sg_te];
- const uint slot =
- offsets[ue] + block_bases[base_idx_block] + base_s + local;
+ const uint slot = offsets[ue] + block_bases[base_idx_block] + base_s + local;
out_ids[slot] = (int)t;
out_probs[slot] = topk_probs[base + k];
if (tok2row_opt != nullptr) {
@@ -167,10 +150,7 @@ PUBLIC KERNEL(MoeBlockBasesFromPartials)(
for (uint b = 0; b < num_blocks; ++b) {
const uint idx = (b * num_tiles + tile_id) * TILE_E + te;
const uint p = partials[idx];
- const uint alloc = clamp ? (running < capacity_per_expert
- ? min(p, capacity_per_expert - running)
- : 0u)
- : p;
+ const uint alloc = clamp ? (running < capacity_per_expert ? min(p, capacity_per_expert - running) : 0u) : p;
block_bases[idx] = running;
block_alloc[idx] = alloc;
running += alloc;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_map.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_map.metal
index c9ee593df..d49745802 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_map.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_map.metal
@@ -95,10 +95,9 @@ PUBLIC KERNEL(MoeBuildTileMap)(
// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:
// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1
PUBLIC KERNEL(MoeWriteDispatchArgs)(
- device const uint*
- total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows
- device uint* dispatch_args, // [3] u32
- constant uint& num_tiles_n, // x dimension
+ device const uint* total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows
+ device uint* dispatch_args, // [3] u32
+ constant uint& num_tiles_n, // x dimension
uint gid AXIS(1, 1)
) {
const uint total_rows = total_tiles_buf[0];
diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
index 899c6c168..1e566a1ea 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
@@ -108,8 +108,7 @@ PUBLIC KERNEL(MoePassABuildTileMap)(
const uint expert_idx = row_expert_map[row_idx];
const uint row_start = expert_offsets[expert_idx];
const uint row_in_expert = row_idx - row_start;
- const uint tile_base =
- tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;
+ const uint tile_base = tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;
tile_map[tile_base * 3 + 0] = h_block;
tile_map[tile_base * 3 + 1] = expert_idx;
@@ -119,9 +118,8 @@ PUBLIC KERNEL(MoePassABuildTileMap)(
// Write dispatch args for indirect dispatch (reusable from tiled version)
PUBLIC KERNEL(MoePassAWriteDispatchArgs)(
device const uint* total_tiles, // [1]
- device uint*
- dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments
- constant uint& num_tiles_y, // usually 1 for Pass A
+ device uint* dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments
+ constant uint& num_tiles_y, // usually 1 for Pass A
uint tid AXIS(1, 1)
) {
dispatch_args[0] = total_tiles[0]; // x dimension = total tiles
diff --git a/crates/backend-uzu/src/backends/metal/kernel/pooling/pooling.metal b/crates/backend-uzu/src/backends/metal/kernel/pooling/pooling.metal
index ba4c7a47f..f366c77e0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/pooling/pooling.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/pooling/pooling.metal
@@ -14,8 +14,7 @@ PUBLIC KERNEL(PoolingCls) (
uint dim_idx AXIS(hidden_dim, 16),
uint batch_idx AXIS(batch_size, 16)
) {
- output[batch_idx * hidden_dim + dim_idx] =
- input[batch_idx * seq_len * hidden_dim + dim_idx];
+ output[batch_idx * hidden_dim + dim_idx] = input[batch_idx * seq_len * hidden_dim + dim_idx];
}
// Mean pooling: Average across sequence [batch, seq_len, hidden_dim] → [batch,
@@ -33,9 +32,7 @@ PUBLIC KERNEL(PoolingMean) (
) {
float sum = 0.0f;
for (uint seq_idx = 0; seq_idx < seq_len; seq_idx++) {
- sum += float(
- input[batch_idx * seq_len * hidden_dim + seq_idx * hidden_dim + dim_idx]
- );
+ sum += float(input[batch_idx * seq_len * hidden_dim + seq_idx * hidden_dim + dim_idx]);
}
output[batch_idx * hidden_dim + dim_idx] = T(sum / float(seq_len));
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/qkv_norm.metal b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/qkv_norm.metal
index d0d78d8aa..2cd29ea4c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/qkv_norm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/qkv_norm.metal
@@ -47,8 +47,7 @@ PUBLIC KERNEL(QKVNorm)(
return;
const ulong slice_offset =
- (ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim +
- (ulong)logical_head_idx * (ulong)head_dim;
+ (ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim + (ulong)logical_head_idx * (ulong)head_dim;
const device InputT* input_data = qkv_input + slice_offset;
const device ScaleT* scales_data = scales;
@@ -58,8 +57,7 @@ PUBLIC KERNEL(QKVNorm)(
AccumT partial_sum = static_cast<AccumT>(0.0f);
// Sum of squares: each lane processes GRAIN_SIZE elements per iteration.
- for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;
- base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {
+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count; base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {
AccumT vals[GRAIN_SIZE];
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
@@ -74,13 +72,11 @@ PUBLIC KERNEL(QKVNorm)(
AccumT total_sum = simd_sum(partial_sum);
// RMS factor.
- AccumT mean_square =
- static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);
+ AccumT mean_square = static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);
AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));
// Normalize + scale.
- for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;
- base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {
+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count; base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {
AccumT vals[GRAIN_SIZE];
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
@@ -97,16 +93,12 @@ PUBLIC KERNEL(QKVNorm)(
if (scale_free) {
output_data[i] = static_cast<OutputT>(normalized_high);
} else if (full_layer) {
- AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +
- static_cast<AccumT>(scale_offset);
- output_data[i] =
- static_cast<OutputT>(normalized_high * scale_value_high);
+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset);
+ output_data[i] = static_cast<OutputT>(normalized_high * scale_value_high);
} else {
OutputT normalized_low = static_cast<OutputT>(normalized_high);
- OutputT scale_value_low = static_cast<OutputT>(
- static_cast<AccumT>(scales_data[i]) +
- static_cast<AccumT>(scale_offset)
- );
+ OutputT scale_value_low =
+ static_cast<OutputT>(static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset));
OutputT product_low = normalized_low * scale_value_low;
output_data[i] = static_cast<OutputT>(product_low);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 9915a5c20..af3911905 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -61,8 +61,7 @@ PUBLIC KERNEL(RMSNorm)(
if (residual_add) {
val += shortcut[i];
if (scale_residual_sum) {
- val =
- static_cast<InputT>(static_cast<float>(val) * post_layer_scalar);
+ val = static_cast<InputT>(static_cast<float>(val) * post_layer_scalar);
}
}
shortcut[i] = val;
@@ -72,18 +71,14 @@ PUBLIC KERNEL(RMSNorm)(
}
// Step 2 - threads reduce their partial sums of squares
- AccumT total_sum_of_squares =
- threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(
- thread_sum_of_squares,
- shared_sum,
- thread_context
- );
+ AccumT total_sum_of_squares = threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(
+ thread_sum_of_squares,
+ shared_sum,
+ thread_context
+ );
// And pre-calculate rms_inv
- AccumT rms_inv = rsqrt(
- total_sum_of_squares / static_cast<AccumT>(element_count) +
- static_cast<AccumT>(epsilon)
- );
+ AccumT rms_inv = rsqrt(total_sum_of_squares / static_cast<AccumT>(element_count) + static_cast<AccumT>(epsilon));
// Step 3 - elementwise normalization
for (uint i = thread_in_row; i < element_count; i += BLOCK_SIZE) {
@@ -96,8 +91,7 @@ PUBLIC KERNEL(RMSNorm)(
x = static_cast<AccumT>(input[i]);
}
- AccumT scale =
- static_cast<AccumT>(scales[i]) + static_cast<AccumT>(scale_offset);
+ AccumT scale = static_cast<AccumT>(scales[i]) + static_cast<AccumT>(scale_offset);
// If full_layer, normalize and scale in AccumT, cast to OutputT at the end
// If not, cast to OutputT after normalize, scale in OutputT
diff --git a/crates/backend-uzu/src/backends/metal/kernel/rng.h b/crates/backend-uzu/src/backends/metal/kernel/rng.h
index 8715ebc7d..bc887762d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/rng.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/rng.h
@@ -43,11 +43,7 @@ inline void philox4x32bumpkey(thread uint* key) {
key[1] += PHILOX_W32_1;
}
-inline void curand_philox4x32_10(
- thread uint* ctr,
- thread uint* key,
- thread uint* out
-) {
+inline void curand_philox4x32_10(thread uint* ctr, thread uint* key, thread uint* out) {
uint local_ctr[4] = {ctr[0], ctr[1], ctr[2], ctr[3]};
uint local_key[2] = {key[0], key[1]};
@@ -90,11 +86,7 @@ inline void philox_state_incr(thread PhiloxState* s) {
++s->ctr[3];
}
-inline void philox_init(
- thread PhiloxState* state,
- uint64_t seed,
- uint64_t offset
-) {
+inline void philox_init(thread PhiloxState* state, uint64_t seed, uint64_t offset) {
// Initialize key from seed
state->key[0] = uint(seed);
state->key[1] = uint(seed >> 32);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal
index 670208e68..933f90b0b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal
@@ -13,10 +13,8 @@ inline ElementT applyRopeTransform(
float sin_val
) {
float inputVal = float(qkv_tensor_view(token_idx, head_idx, dim_idx));
- float pairedVal =
- (dim_idx < half_dim)
- ? -float(qkv_tensor_view(token_idx, head_idx, dim_idx + half_dim))
- : float(qkv_tensor_view(token_idx, head_idx, dim_idx - half_dim));
+ float pairedVal = (dim_idx < half_dim) ? -float(qkv_tensor_view(token_idx, head_idx, dim_idx + half_dim))
+ : float(qkv_tensor_view(token_idx, head_idx, dim_idx - half_dim));
return static_cast<ElementT>(inputVal * cos_val + pairedVal * sin_val);
}
@@ -39,8 +37,7 @@ PUBLIC KERNEL(Rope)(
const uint token_index AXIS(suffix_length, 1),
const uint dimension_index AXIS(head_dim, 128)
) {
- if (head_index >= num_heads || token_index >= suffix_length ||
- dimension_index >= head_dim)
+ if (head_index >= num_heads || token_index >= suffix_length || dimension_index >= head_dim)
return;
if (head_dim & 1)
return;
@@ -55,31 +52,17 @@ PUBLIC KERNEL(Rope)(
const uint half_rope_dim = rope_dim / 2;
TensorView3D<const ElementT> qkv_tensor_view =
- TensorView3D<const ElementT>(qkv)
- .shaped(suffix_length, total_heads, head_dim);
- TensorView2D<const RopeT> cosines_tensor_view =
- TensorView2D<const RopeT>(cosines).shaped(suffix_length, rope_dim);
- TensorView2D<const RopeT> sines_tensor_view =
- TensorView2D<const RopeT>(sines).shaped(suffix_length, rope_dim);
+ TensorView3D<const ElementT>(qkv).shaped(suffix_length, total_heads, head_dim);
+ TensorView2D<const RopeT> cosines_tensor_view = TensorView2D<const RopeT>(cosines).shaped(suffix_length, rope_dim);
+ TensorView2D<const RopeT> sines_tensor_view = TensorView2D<const RopeT>(sines).shaped(suffix_length, rope_dim);
TensorView3D<ElementT> rotated_queries_tensor_view =
- TensorView3D<ElementT>(rotated_queries)
- .shaped(num_heads, suffix_length, head_dim);
+ TensorView3D<ElementT>(rotated_queries).shaped(num_heads, suffix_length, head_dim);
if (dimension_index < rope_dim) {
- const float cos_val =
- float(cosines_tensor_view(token_index, dimension_index));
- const float sin_val =
- float(sines_tensor_view(token_index, dimension_index));
+ const float cos_val = float(cosines_tensor_view(token_index, dimension_index));
+ const float sin_val = float(sines_tensor_view(token_index, dimension_index));
rotated_queries_tensor_view(head_index, token_index, dimension_index) =
- applyRopeTransform(
- qkv_tensor_view,
- token_index,
- head_index,
- dimension_index,
- half_rope_dim,
- cos_val,
- sin_val
- );
+ applyRopeTransform(qkv_tensor_view, token_index, head_index, dimension_index, half_rope_dim, cos_val, sin_val);
} else {
rotated_queries_tensor_view(head_index, token_index, dimension_index) =
qkv_tensor_view(token_index, head_index, dimension_index);
@@ -94,24 +77,20 @@ PUBLIC KERNEL(Rope)(
return;
TensorView3D<ElementT> rotated_keys_tensor_view =
- TensorView3D<ElementT>(rotated_keys)
- .shaped(num_groups, suffix_length, head_dim);
+ TensorView3D<ElementT>(rotated_keys).shaped(num_groups, suffix_length, head_dim);
const uint key_head_index = num_heads + group_index;
if (dimension_index < rope_dim) {
- const float cos_val =
- float(cosines_tensor_view(token_index, dimension_index));
- const float sin_val =
- float(sines_tensor_view(token_index, dimension_index));
- rotated_keys_tensor_view(group_index, token_index, dimension_index) =
- applyRopeTransform(
- qkv_tensor_view,
- token_index,
- key_head_index,
- dimension_index,
- half_rope_dim,
- cos_val,
- sin_val
- );
+ const float cos_val = float(cosines_tensor_view(token_index, dimension_index));
+ const float sin_val = float(sines_tensor_view(token_index, dimension_index));
+ rotated_keys_tensor_view(group_index, token_index, dimension_index) = applyRopeTransform(
+ qkv_tensor_view,
+ token_index,
+ key_head_index,
+ dimension_index,
+ half_rope_dim,
+ cos_val,
+ sin_val
+ );
} else {
rotated_keys_tensor_view(group_index, token_index, dimension_index) =
qkv_tensor_view(token_index, key_head_index, dimension_index);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/argmax.metal
index 852a43700..38a96c209 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/argmax.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/argmax.metal
@@ -17,19 +17,14 @@ bool argmax_is_better(ArgmaxPair a, ArgmaxPair b) {
return a.value > b.value || (a.value == b.value && a.index < b.index);
}
-ArgmaxPair argmax_combine(ArgmaxPair a, ArgmaxPair b) {
- return argmax_is_better(a, b) ? a : b;
-}
+ArgmaxPair argmax_combine(ArgmaxPair a, ArgmaxPair b) { return argmax_is_better(a, b) ? a : b; }
struct SimdReduceArgmax {
using value_type = ArgmaxPair;
static constant constexpr ArgmaxPair identity = ARGMAX_INIT;
static ArgmaxPair simd_reduce(ArgmaxPair x) {
for (ushort offset = 16; offset > 0; offset /= 2) {
- ArgmaxPair other = {
- simd_shuffle_down(x.value, offset),
- simd_shuffle_down(x.index, offset)
- };
+ ArgmaxPair other = {simd_shuffle_down(x.value, offset), simd_shuffle_down(x.index, offset)};
x = argmax_combine(x, other);
}
return x;
@@ -37,9 +32,7 @@ struct SimdReduceArgmax {
};
template <ushort GRAIN_SIZE_PARAM>
-static inline ArgmaxPair thread_argmax_reduce(
- thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM]
-) {
+static inline ArgmaxPair thread_argmax_reduce(thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM]) {
ArgmaxPair result = values[0];
for (ushort i = 1; i < GRAIN_SIZE_PARAM; i++) {
result = argmax_combine(result, values[i]);
@@ -70,11 +63,7 @@ static void load_blocked_argmax_batched(
}
template <ushort BLOCK_SIZE_PARAM>
-static ArgmaxPair threadgroup_raking_argmax(
- ArgmaxPair value,
- threadgroup ArgmaxPair* shared,
- const ushort lid
-) {
+static ArgmaxPair threadgroup_raking_argmax(ArgmaxPair value, threadgroup ArgmaxPair* shared, const ushort lid) {
shared[lid] = value;
threadgroup_barrier(mem_flags::mem_threadgroup);
@@ -90,17 +79,11 @@ static ArgmaxPair threadgroup_raking_argmax(
ArgmaxPair simd_result = thread_result;
for (ushort offset = 16; offset > 0; offset /= 2) {
- ArgmaxPair other = {
- simd_shuffle_down(simd_result.value, offset),
- simd_shuffle_down(simd_result.index, offset)
- };
+ ArgmaxPair other = {simd_shuffle_down(simd_result.value, offset), simd_shuffle_down(simd_result.index, offset)};
simd_result = argmax_combine(simd_result, other);
}
- ArgmaxPair final_result = {
- simd_broadcast(simd_result.value, 0),
- simd_broadcast(simd_result.index, 0)
- };
+ ArgmaxPair final_result = {simd_broadcast(simd_result.value, 0), simd_broadcast(simd_result.index, 0)};
for (ushort i = first_index; i < first_index + values_per_thread; i++) {
shared[i] = final_result;
@@ -132,23 +115,14 @@ PUBLIC KERNEL(ArgmaxSingle)(
const uint elements_per_tile = BLOCK_SIZE * GRAIN_SIZE;
ArgmaxPair best = ARGMAX_INIT;
- for (uint vocab_offset = 0; vocab_offset < vocab_size;
- vocab_offset += elements_per_tile) {
+ for (uint vocab_offset = 0; vocab_offset < vocab_size; vocab_offset += elements_per_tile) {
ArgmaxPair buf[GRAIN_SIZE];
- load_blocked_argmax_batched<GRAIN_SIZE>(
- buf,
- logits_data,
- local_id,
- batch_idx,
- vocab_offset,
- vocab_size
- );
+ load_blocked_argmax_batched<GRAIN_SIZE>(buf, logits_data, local_id, batch_idx, vocab_offset, vocab_size);
ArgmaxPair tile_best = thread_argmax_reduce<GRAIN_SIZE>(buf);
best = argmax_combine(best, tile_best);
}
- ArgmaxPair group_best =
- threadgroup_raking_argmax<BLOCK_SIZE>(best, shared, local_id);
+ ArgmaxPair group_best = threadgroup_raking_argmax<BLOCK_SIZE>(best, shared, local_id);
if (local_id == 0) {
final_tokens[batch_idx] = group_best.index;
@@ -178,26 +152,14 @@ PUBLIC KERNEL(ArgmaxMain)(
const uint vocab_offset = vocab_group_idx * elements_per_group;
ArgmaxPair values[GRAIN_SIZE];
- load_blocked_argmax_batched<GRAIN_SIZE>(
- values,
- logits_data,
- local_id,
- batch_idx,
- vocab_offset,
- vocab_size
- );
+ load_blocked_argmax_batched<GRAIN_SIZE>(values, logits_data, local_id, batch_idx, vocab_offset, vocab_size);
ArgmaxPair thread_result = thread_argmax_reduce<GRAIN_SIZE>(values);
ArgmaxPair group_result =
- threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(
- thread_result,
- shared,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(thread_result, shared, thread_context);
if (local_id == 0) {
- uint vocab_groups_per_batch =
- (vocab_size + elements_per_group - 1) / elements_per_group;
+ uint vocab_groups_per_batch = (vocab_size + elements_per_group - 1) / elements_per_group;
uint partial_idx = batch_idx * vocab_groups_per_batch + vocab_group_idx;
partial_results[partial_idx] = group_result;
}
@@ -218,23 +180,17 @@ PUBLIC KERNEL(ArgmaxFinal)(
return;
const uint elements_per_group = BLOCK_SIZE * GRAIN_SIZE;
- const uint vocab_groups_per_batch =
- (vocab_size + elements_per_group - 1) / elements_per_group;
+ const uint vocab_groups_per_batch = (vocab_size + elements_per_group - 1) / elements_per_group;
const uint partial_offset = batch_idx * vocab_groups_per_batch;
ArgmaxPair thread_result = ARGMAX_INIT;
for (uint i = local_id; i < vocab_groups_per_batch; i += BLOCK_SIZE) {
- thread_result =
- argmax_combine(thread_result, partial_results[partial_offset + i]);
+ thread_result = argmax_combine(thread_result, partial_results[partial_offset + i]);
}
ArgmaxPair result =
- threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(
- thread_result,
- shared,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(thread_result, shared, thread_context);
if (local_id == 0) {
final_tokens[batch_idx] = result.index;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/bitmask.metal
index 3dd8b2011..e5f776520 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/bitmask.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/bitmask.metal
@@ -23,8 +23,7 @@ PUBLIC KERNEL(Bitmask) (
}
uint bitmask_size = (vocab_size + (BITS_IN_U32 - 1)) / BITS_IN_U32;
- uint base_idx =
- batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
+ uint base_idx = batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
uint batch_end = batch_idx * vocab_size + vocab_size;
#pragma unroll(4)
@@ -34,8 +33,7 @@ PUBLIC KERNEL(Bitmask) (
uint token_idx = global_idx - batch_idx * vocab_size;
uint bitmask_idx = batch_idx * bitmask_size + (token_idx / BITS_IN_U32);
bool mask = (bitmask[bitmask_idx] >> (token_idx % BITS_IN_U32)) & 0b1;
- processed_logits[global_idx] =
- select(T(-INFINITY), logits[global_idx], mask);
+ processed_logits[global_idx] = select(T(-INFINITY), logits[global_idx], mask);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/gumbel.metal
index 2625bd27e..5fb7b19aa 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/gumbel.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/gumbel.metal
@@ -32,8 +32,7 @@ PUBLIC KERNEL(Gumbel) (
uint grain_offset = batch_start + grain_offset_in_batch;
uint64_t rng_seed = batch_seeds[batch_seeds_offset + batch_idx];
- uint64_t rng_offset = (group_idx * BLOCK_SIZE + thread_idx) *
- (GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET;
+ uint64_t rng_offset = (group_idx * BLOCK_SIZE + thread_idx) * (GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET;
PhiloxState rng;
philox_init(&rng, rng_seed, rng_offset);
@@ -41,8 +40,7 @@ PUBLIC KERNEL(Gumbel) (
for (uint i = 0; i < GRAIN_SIZE; i++) {
uint global_idx = grain_offset + i * BLOCK_SIZE;
if (global_idx < batch_end) {
- processed_logits[global_idx] =
- logits[global_idx] + T(-fast::log(-fast::log(uniform_float(&rng))));
+ processed_logits[global_idx] = logits[global_idx] + T(-fast::log(-fast::log(uniform_float(&rng))));
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/min_p.metal
index 36b988cec..79355c6fe 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/min_p.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/min_p.metal
@@ -34,11 +34,7 @@ PUBLIC KERNEL(MinP) (
local_max = fmax(local_max, logit_value);
}
float max_logit =
- threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(
- local_max,
- shared_reduce_buffer,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(local_max, shared_reduce_buffer, thread_context);
// Then the threshold is just max_logit + log(min_p), mask everything strictly
// below it
@@ -46,7 +42,6 @@ PUBLIC KERNEL(MinP) (
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
T logit_value = logits[batch_start + i];
- processed_logits[batch_start + i] =
- select(T(-INFINITY), logit_value, logit_value >= t_threshold);
+ processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/temperature.metal
index 73c7bd3e4..57b63dbb1 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/temperature.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/temperature.metal
@@ -21,8 +21,7 @@ PUBLIC KERNEL(Temperature) (
logits = processed_logits;
}
- uint base_idx =
- batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
+ uint base_idx = batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
uint batch_end = batch_idx * vocab_size + vocab_size;
#pragma unroll(4)
diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/top_k.metal
index 7a27876aa..b88dc4092 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -34,24 +34,12 @@ PUBLIC KERNEL(TopK)(
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
float logit_value = float(logits[batch_start + i]);
local_max = fmax(local_max, logit_value);
- local_min = select(
- local_min,
- fmin(local_min, logit_value),
- logit_value > -INFINITY
- );
+ local_min = select(local_min, fmin(local_min, logit_value), logit_value > -INFINITY);
}
float max_logit =
- threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(
- local_max,
- shared_reduce_buffer,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(local_max, shared_reduce_buffer, thread_context);
float min_logit =
- threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(
- local_min,
- shared_reduce_buffer,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(local_min, shared_reduce_buffer, thread_context);
// Do the binary search on the threshold
float low = min_logit;
float high = max_logit;
@@ -65,12 +53,11 @@ PUBLIC KERNEL(TopK)(
float logit_value = float(logits[batch_start + i]);
local_num_above_threshold += select(0, 1, logit_value >= threshold);
}
- uint num_above_threshold =
- threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(
- local_num_above_threshold,
- (threadgroup uint*)shared_reduce_buffer,
- thread_context
- );
+ uint num_above_threshold = threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(
+ local_num_above_threshold,
+ (threadgroup uint*)shared_reduce_buffer,
+ thread_context
+ );
// Update binary search
if (num_above_threshold == top_k) {
@@ -89,7 +76,6 @@ PUBLIC KERNEL(TopK)(
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
T logit_value = logits[batch_start + i];
- processed_logits[batch_start + i] =
- select(T(-INFINITY), logit_value, logit_value >= t_threshold);
+ processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/top_p.metal
index a4b9ea0d8..7a6303feb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/top_p.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/top_p.metal
@@ -35,42 +35,22 @@ PUBLIC KERNEL(TopP) (
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
float logit_value = float(logits[batch_start + i]);
local_max = fmax(local_max, logit_value);
- local_min = select(
- local_min,
- fmin(local_min, logit_value),
- logit_value > -INFINITY
- );
+ local_min = select(local_min, fmin(local_min, logit_value), logit_value > -INFINITY);
}
float max_logit =
- threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(
- local_max,
- shared_reduce_buffer,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(local_max, shared_reduce_buffer, thread_context);
float min_logit =
- threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(
- local_min,
- shared_reduce_buffer,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(local_min, shared_reduce_buffer, thread_context);
// Find denominator for softmax
float local_sum = 0.0f;
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
float logit_value = float(logits[batch_start + i]);
- local_sum += select(
- 0.0,
- fast::exp(logit_value - max_logit),
- logit_value > -INFINITY
- );
+ local_sum += select(0.0, fast::exp(logit_value - max_logit), logit_value > -INFINITY);
}
float total_sum =
- threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(
- local_sum,
- shared_reduce_buffer,
- thread_context
- );
+ threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(local_sum, shared_reduce_buffer, thread_context);
// Do the binary search on the threshold
float target_mass = top_p * total_sum;
@@ -87,29 +67,23 @@ PUBLIC KERNEL(TopP) (
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
float logit_value = float(logits[batch_start + i]);
float logit_mass = fast::exp(logit_value - max_logit);
- local_sum_above_threshold +=
- select(0.0, logit_mass, logit_value >= threshold);
- local_min_above_threshold = fmin(
- local_min_above_threshold,
- select(INFINITY, logit_mass, logit_value >= threshold)
- );
+ local_sum_above_threshold += select(0.0, logit_mass, logit_value >= threshold);
+ local_min_above_threshold =
+ fmin(local_min_above_threshold, select(INFINITY, logit_mass, logit_value >= threshold));
}
- float sum_above_threshold =
- threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(
- local_sum_above_threshold,
- shared_reduce_buffer,
- thread_context
- );
- float min_above_threshold =
- threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(
- local_min_above_threshold,
- shared_reduce_buffer,
- thread_context
- );
+ float sum_above_threshold = threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(
+ local_sum_above_threshold,
+ shared_reduce_buffer,
+ thread_context
+ );
+ float min_above_threshold = threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(
+ local_min_above_threshold,
+ shared_reduce_buffer,
+ thread_context
+ );
// Early exit
- if (sum_above_threshold >= target_mass &&
- sum_above_threshold - min_above_threshold < target_mass) {
+ if (sum_above_threshold >= target_mass && sum_above_threshold - min_above_threshold < target_mass) {
break;
}
// Update binary search
@@ -127,7 +101,6 @@ PUBLIC KERNEL(TopP) (
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
T logit_value = logits[batch_start + i];
- processed_logits[batch_start + i] =
- select(T(-INFINITY), logit_value, logit_value >= t_threshold);
+ processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal
index b8b61860a..4b81d6450 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -187,10 +187,8 @@ PUBLIC KERNEL(ShortConvTrie)(
// Select parent state (root uses base_state)
const int parent = parents[node];
- const device T* parent_state =
- (parent < 0) ? (base_state + base_state_offset)
- : (suffix_state +
- (parent * model_dim + channel_idx) * state_stride);
+ const device T* parent_state = (parent < 0) ? (base_state + base_state_offset)
+ : (suffix_state + (parent * model_dim + channel_idx) * state_stride);
float acc = 0.0f;
if (has_bias) {
@@ -212,8 +210,7 @@ PUBLIC KERNEL(ShortConvTrie)(
// Write post-state for this node
if (tap_count > 0) {
- device T* dst_state =
- suffix_state + (node * model_dim + channel_idx) * state_stride;
+ device T* dst_state = suffix_state + (node * model_dim + channel_idx) * state_stride;
for (uint tap = 0; tap < tap_count - 1; ++tap) {
dst_state[tap] = parent_state[tap + 1];
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal
index a716e7c92..3cb83c001 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal
@@ -42,8 +42,7 @@ PUBLIC KERNEL(Softmax)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// Pass 1: online (max, normalizer) accumulation over the row.
- for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length;
- base += stride) {
+ for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length; base += stride) {
float vals[SOFTMAX_ELEMENTS_PER_THREAD];
for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {
vals[i] = (base + i < row_length) ? float(row[base + i]) : -FLT_MAX;
@@ -86,12 +85,10 @@ PUBLIC KERNEL(Softmax)(
const float inv_norm = 1.0f / shared_norm[0];
// Pass 2: normalize.
- for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length;
- base += stride) {
+ for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length; base += stride) {
for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {
if (base + i < row_length) {
- row[base + i] =
- T(fast::exp(float(row[base + i]) - final_max) * inv_norm);
+ row[base + i] = T(fast::exp(float(row[base + i]) - final_max) * inv_norm);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal
index e84becbdd..9d997c001 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal
@@ -87,8 +87,7 @@ PUBLIC KERNEL(Conv1dDecode)(
const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);
b_out[dst] = activated;
} else if (channel_idx < inner_dim + 2 * proj_dim) {
- const uint dst =
- token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
+ const uint dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
c_out[dst] = activated;
}
@@ -153,8 +152,7 @@ PUBLIC KERNEL(Conv1dScan)(
const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);
b_out[dst] = activated;
} else if (channel_idx < inner_dim + 2 * proj_dim) {
- const uint dst =
- token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
+ const uint dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);
c_out[dst] = activated;
}
} else if (tap_count > 0) {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index a49389ef8..d351752fd 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -71,21 +71,18 @@ PUBLIC KERNEL(SSDPrefill64)(
const uint dt_idx = token * dt_token_stride + dt_base;
const float x_val = float(x[x_idx]);
- const float decay_val =
- fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));
+ const float decay_val = fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));
const float gate = float(activate_silu(z[x_idx]));
const float skip = d_scalar * x_val;
const float dt_scaled_input = x_val;
float contrib = 0.0f;
- const float new_state0 =
- decay_val * state0 + dt_scaled_input * float(b[cb_idx0]);
+ const float new_state0 = decay_val * state0 + dt_scaled_input * float(b[cb_idx0]);
state0 = new_state0;
contrib += new_state0 * float(c[cb_idx0]);
cb_idx0 += cb_token_stride;
- const float new_state1 =
- decay_val * state1 + dt_scaled_input * float(b[cb_idx1]);
+ const float new_state1 = decay_val * state1 + dt_scaled_input * float(b[cb_idx1]);
state1 = new_state1;
contrib += new_state1 * float(c[cb_idx1]);
cb_idx1 += cb_token_stride;
@@ -149,8 +146,7 @@ PUBLIC KERNEL(SSDPrefill)(
const float d_scalar = float(d[h_idx]);
const int max_chunks = SSM_PREFILL_MAX_STATE / thread_context.simdgroup_size;
- const int chunk_count = (state_dim + int(thread_context.simdgroup_size) - 1) /
- int(thread_context.simdgroup_size);
+ const int chunk_count = (state_dim + int(thread_context.simdgroup_size) - 1) / int(thread_context.simdgroup_size);
if (state_dim <= 0 || chunk_count > max_chunks) {
return;
}
@@ -173,8 +169,7 @@ PUBLIC KERNEL(SSDPrefill)(
const uint dt_idx = token * dt_token_stride + dt_base;
const float x_val = float(x[x_idx]);
- const float decay_val =
- fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));
+ const float decay_val = fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));
const float gate = float(activate_silu(z[x_idx]));
const float skip = d_scalar * x_val;
const float dt_scaled_input = x_val;
@@ -182,13 +177,10 @@ PUBLIC KERNEL(SSDPrefill)(
float contrib_sum = 0.0f;
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx =
- chunk * int(thread_context.simdgroup_size) + int(lane_idx);
+ const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);
if (idx < state_dim) {
- const uint cb_idx = cb_group_base + uint(idx) * cb_state_stride +
- token * cb_token_stride;
- const float new_state =
- decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);
+ const uint cb_idx = cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;
+ const float new_state = decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);
lane_states[chunk] = new_state;
contrib_sum += new_state * float(c[cb_idx]);
}
@@ -235,12 +227,10 @@ PUBLIC KERNEL(SSDPrefillSequential)(
) {
const uint safe_group = max(group_size, 1u);
const uint group_idx = h_idx / safe_group;
- device T* state_row = state + size_t(h_idx) * state_strides[0] +
- size_t(dh_idx) * state_strides[1];
+ device T* state_row = state + size_t(h_idx) * state_strides[0] + size_t(dh_idx) * state_strides[1];
for (uint token = 0; token < suffix_len; ++token) {
- const uint x_idx =
- token * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];
+ const uint x_idx = token * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];
const uint dt_idx = token * dt_strides[0] + h_idx * dt_strides[1];
const uint cb_base = token * cb_strides[0] + group_idx * cb_strides[1];
@@ -258,8 +248,7 @@ PUBLIC KERNEL(SSDPrefillSequential)(
for (; s < vec_bound; s += 4) {
const uint state_idx = s * state_strides[2];
const uint cb_idx = cb_base + s * cb_strides[2];
- auto prev_state =
- *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx);
+ auto prev_state = *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx);
auto b_vec = *reinterpret_cast<device const vec<T, 4>*>(b + cb_idx);
auto c_vec = *reinterpret_cast<device const vec<T, 4>*>(c + cb_idx);
vec<T, 4> new_state = prev_state * this_decay + b_vec * dt_scaled_input;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_update.metal
index c4564d601..3e6156b6a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_update.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_update.metal
@@ -39,14 +39,10 @@ PUBLIC KERNEL(SSDUpdate)(
state = next_state;
}
- const uint cb_start_idx =
- b_idx * cb_strides[0] + (h_idx / group_size) * cb_strides[1];
- const uint x_idx =
- b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];
+ const uint cb_start_idx = b_idx * cb_strides[0] + (h_idx / group_size) * cb_strides[1];
+ const uint x_idx = b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];
const uint dt_idx = b_idx * dt_strides[0] + h_idx * dt_strides[1];
- const uint state_start_idx = b_idx * state_strides[0] +
- h_idx * state_strides[1] +
- dh_idx * state_strides[2];
+ const uint state_start_idx = b_idx * state_strides[0] + h_idx * state_strides[1] + dh_idx * state_strides[2];
// load data
T this_x = x[x_idx];
@@ -63,8 +59,7 @@ PUBLIC KERNEL(SSDUpdate)(
for (uint i = 0; i < state_size; ++i) {
uint cb_idx = cb_start_idx + i;
uint state_idx = state_start_idx + i;
- T this_new_state =
- state[state_idx] * this_decay + b[cb_idx] * dt_scaled_input;
+ T this_new_state = state[state_idx] * this_decay + b[cb_idx] * dt_scaled_input;
next_state[state_idx] = this_new_state;
temp = temp + this_new_state * c[cb_idx];
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
index 3809cb350..8c2541c5d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
@@ -18,7 +18,5 @@ PUBLIC KERNEL(TensorAddBias)(
}
uint col = position % num_cols;
- output[position] = static_cast<T>(
- static_cast<float>(input[position]) + static_cast<float>(bias[col])
- );
+ output[position] = static_cast<T>(static_cast<float>(input[position]) + static_cast<float>(bias[col]));
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal
index e5cc942f6..72d0e3c0e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal
@@ -3,18 +3,10 @@
/// Copies a single sampled token (u32) to the token_ids buffer (u64).
/// Used in async pipeline to pass sampled token to next forward pass.
-PUBLIC KERNEL(TokenCopySampled)(
- device const uint32_t* src,
- device uint64_t* dst
-) {
+PUBLIC KERNEL(TokenCopySampled)(device const uint32_t* src, device uint64_t* dst) {
dst[0] = static_cast<uint64_t>(src[0]);
}
/// Copies a single sampled token (u32) to an indexed results buffer (u32).
/// Used in async pipeline to store result for callback access.
-PUBLIC KERNEL(TokenCopyToResults)(
- device const uint32_t* src,
- device uint32_t* dst
-) {
- dst[0] = src[0];
-}
+PUBLIC KERNEL(TokenCopyToResults)(device const uint32_t* src, device uint32_t* dst) { dst[0] = src[0]; }