A high-performance inference engine for AI models
speedup prefill
eugenece6298cb7be8
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rsindex ebdf056d2..5800a8bee 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs@@ -39,6 +39,7 @@ pub enum GemmTiling {Tile64x64x32_Simdgroups2x2,Tile32x32x32_Simdgroups2x2,Tile16x32x256_Simdgroups1x1,+ Tile16x32x256_Simdgroups1x1x4,Tile16x128x256_Simdgroups1x4,Tile32x64x256_Simdgroups2x2,Tile64x32x256_Simdgroups4x1,@@ -57,6 +58,7 @@ impl GemmTiling {Self::Tile64x64x32_Simdgroups2x2 => 64,Self::Tile32x32x32_Simdgroups2x2 => 32,Self::Tile16x32x256_Simdgroups1x1 => 16,+ Self::Tile16x32x256_Simdgroups1x1x4 => 16,Self::Tile16x128x256_Simdgroups1x4 => 16,Self::Tile32x64x256_Simdgroups2x2 => 32,Self::Tile64x32x256_Simdgroups4x1 => 64,@@ -73,6 +75,7 @@ impl GemmTiling {Self::Tile64x64x32_Simdgroups2x2 => 64,Self::Tile32x32x32_Simdgroups2x2 => 32,Self::Tile16x32x256_Simdgroups1x1 => 32,+ Self::Tile16x32x256_Simdgroups1x1x4 => 32,Self::Tile16x128x256_Simdgroups1x4 => 128,Self::Tile32x64x256_Simdgroups2x2 => 64,Self::Tile64x32x256_Simdgroups4x1 => 32,@@ -89,6 +92,7 @@ impl GemmTiling {Self::Tile64x64x32_Simdgroups2x2 => 32,Self::Tile32x32x32_Simdgroups2x2 => 32,Self::Tile16x32x256_Simdgroups1x1 => 256,+ Self::Tile16x32x256_Simdgroups1x1x4 => 256,Self::Tile16x128x256_Simdgroups1x4 => 256,Self::Tile32x64x256_Simdgroups2x2 => 256,Self::Tile64x32x256_Simdgroups4x1 => 256,@@ -105,6 +109,7 @@ impl GemmTiling {Self::Tile64x64x32_Simdgroups2x2 => 2,Self::Tile32x32x32_Simdgroups2x2 => 2,Self::Tile16x32x256_Simdgroups1x1 => 1,+ Self::Tile16x32x256_Simdgroups1x1x4 => 1,Self::Tile16x128x256_Simdgroups1x4 => 1,Self::Tile32x64x256_Simdgroups2x2 => 2,Self::Tile64x32x256_Simdgroups4x1 => 4,@@ -121,6 +126,7 @@ impl GemmTiling {Self::Tile64x64x32_Simdgroups2x2 => 2,Self::Tile32x32x32_Simdgroups2x2 => 2,Self::Tile16x32x256_Simdgroups1x1 => 1,+ Self::Tile16x32x256_Simdgroups1x1x4 => 1,Self::Tile16x128x256_Simdgroups1x4 => 4,Self::Tile32x64x256_Simdgroups2x2 => 2,Self::Tile64x32x256_Simdgroups4x1 => 1,@@ -129,10 +135,18 @@ impl GemmTiling {}}+ pub const fn simdgroups_k(self) -> u32 {+ match self {+ Self::Tile16x32x256_Simdgroups1x1x4 => 4,+ _ => 1,+ }+ }+pub const fn is_mxu_variant(self) -> bool {matches!(self,Self::Tile16x32x256_Simdgroups1x1+ | Self::Tile16x32x256_Simdgroups1x1x4| Self::Tile16x128x256_Simdgroups1x4| Self::Tile32x64x256_Simdgroups2x2| Self::Tile64x32x256_Simdgroups4x1@@ -148,6 +162,9 @@ pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {pub const fn gemm_tiling_simdgroups_per_column(t: GemmTiling) -> u32 {t.simdgroups_n()}+pub const fn gemm_tiling_simdgroups_per_k(t: GemmTiling) -> u32 {+ t.simdgroups_k()+}bitflags! {#[repr(transparent)]diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metalindex 435995901..dee18b6aa 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal@@ -109,6 +109,10 @@ KERNEL(DeltaNetChunkedOutputAndState)(const uint token_base = chunk_idx * OUTPUT_STATE_CHUNK;const uint valid_tokens = token_base < suffix_len ? min(uint(OUTPUT_STATE_CHUNK), suffix_len - token_base) : 0u;const uint chunk_head_base = (chunk_idx * num_v_heads + hv_idx);+ const bool full_chunk = valid_tokens == OUTPUT_STATE_CHUNK;+ auto chunk_source = [&](auto source, const short rows, const short cols) {+ return full_chunk ? source : source.bounded(rows, cols);+ };TokenValueFragment output_acc;// -- R phase: R = beta (.) (V - e^g (.) (K . S^T)) into residual_new_value_tile ----------@@ -126,10 +130,17 @@ KERNEL(DeltaNetChunkedOutputAndState)(TokenKeyFragment key_frag;TokenKeyFragment query_frag;KeyValueFragment state_frag;- key_frag.load_from(lane, fragment_source(k_tile + key_block_start, int(key_dim)).bounded(valid_token_rows, FR));+ key_frag.load_from(+ lane,+ chunk_source(fragment_source(k_tile + key_block_start, int(key_dim)), valid_token_rows, FR)+ );query_frag.load_from(lane,- fragment_source(q_norm + query_tile_base + key_block_start, int(key_dim)).bounded(valid_token_rows, FR)+ chunk_source(+ fragment_source(q_norm + query_tile_base + key_block_start, int(key_dim)),+ valid_token_rows,+ FR+ ));state_frag.load_from(lane, fragment_source(transposed_state_tile + key_block_start * VT, int(VT), 1));fragment_mma(residual_acc, key_frag, state_frag);@@ -170,8 +181,11 @@ KERNEL(DeltaNetChunkedOutputAndState)(ScratchKeyValueFragment residual_frag;causal_inv_frag.load_from(lane,- fragment_source(causal_inv_rows + source_token_block_start, int(OUTPUT_STATE_CHUNK))- .bounded(valid_token_rows, FR)+ chunk_source(+ fragment_source(causal_inv_rows + source_token_block_start, int(OUTPUT_STATE_CHUNK)),+ valid_token_rows,+ FR+ ));residual_frag.load_from(lane,@@ -215,8 +229,11 @@ KERNEL(DeltaNetChunkedOutputAndState)(ScratchKeyValueFragment new_value_frag;qk_frag.load_from(lane,- fragment_source(qk_scaled + qk_tile_base + source_token_block_start, int(OUTPUT_STATE_CHUNK))- .bounded(valid_token_rows, valid_source_tokens)+ chunk_source(+ fragment_source(qk_scaled + qk_tile_base + source_token_block_start, int(OUTPUT_STATE_CHUNK)),+ valid_token_rows,+ valid_source_tokens+ ));new_value_frag.load_from(lane,@@ -226,7 +243,13 @@ KERNEL(DeltaNetChunkedOutputAndState)(}const uint output_tile_base = (token_base + token_tile_base) * value_dim + hv_idx * head_v_dim + value_base;- output_acc.store_safe(lane, out + output_tile_base, int(value_dim), short2(short(VT), short(valid_token_rows)));+ output_acc.store_maybe_bounded(+ lane,+ out + output_tile_base,+ int(value_dim),+ full_chunk,+ short2(short(VT), short(valid_token_rows))+ );}threadgroup_barrier(mem_flags::mem_threadgroup);@@ -247,14 +270,21 @@ KERNEL(DeltaNetChunkedOutputAndState)(TokenKeyTileFragment key_frag;new_value_frag.load_from(lane,- fragment_source(residual_new_value_tile + source_token_block_start * VT, 1, int(VT))- .bounded(short(VT), short(valid_source_tokens))+ chunk_source(+ fragment_source(residual_new_value_tile + source_token_block_start * VT, 1, int(VT)),+ short(VT),+ short(valid_source_tokens)+ ));const device float* k_tile =k_norm + (token_base + source_token_block_start) * key_dim + hk_idx * HEAD_K_DIM + key_tile_base;key_frag.load_from(lane,- fragment_source(k_tile, int(key_dim)).bounded(short(valid_source_tokens), short(OUTPUT_STATE_KEY_TILE))+ chunk_source(+ fragment_source(k_tile, int(key_dim)),+ short(valid_source_tokens),+ short(OUTPUT_STATE_KEY_TILE)+ ));key_frag.map_coords(lane, [&](short row, short, float value) {if (uint(row) >= valid_source_tokens) {diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.hindex c47ed59de..1a7b47409 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h@@ -39,11 +39,12 @@ enum class GemmTiling : uint32_t {Tile64x64x32_Simdgroups2x2 = 3,Tile32x32x32_Simdgroups2x2 = 4,Tile16x32x256_Simdgroups1x1 = 5,- Tile16x128x256_Simdgroups1x4 = 6,- Tile32x64x256_Simdgroups2x2 = 7,- Tile64x32x256_Simdgroups4x1 = 8,- Tile64x64x256_Simdgroups2x2 = 9,- Tile128x128x256_Simdgroups4x4 = 10,+ Tile16x32x256_Simdgroups1x1x4 = 6,+ Tile16x128x256_Simdgroups1x4 = 7,+ Tile32x64x256_Simdgroups2x2 = 8,+ Tile64x32x256_Simdgroups4x1 = 9,+ Tile64x64x256_Simdgroups2x2 = 10,+ Tile128x128x256_Simdgroups4x4 = 11,};static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.hindex 326dc2c7d..2647fa395 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h@@ -273,7 +273,12 @@ struct Fragment {);}- template <class Ptr, class FragmentRowStride = Int<1>, class FragmentColStride = Int<1>>+ template <+ bool ALIGNED_M = false,+ bool ALIGNED_N = false,+ class Ptr,+ class FragmentRowStride = Int<1>,+ class FragmentColStride = Int<1>>METAL_FUNC void store_safe(const ushort simd_lane_id,Ptr destination,@@ -282,7 +287,7 @@ struct Fragment {FragmentRowStride fragment_row_stride = {},FragmentColStride fragment_col_stride = {}) thread {- transfer<STORE, SAFE>(+ transfer<STORE, SAFE, ALIGNED_M, ALIGNED_N>(simd_lane_id,destination,leading_dimension,@@ -346,6 +351,8 @@ private:template <bool IS_LOAD,bool IS_SAFE,+ bool ALIGNED_M = false,+ bool ALIGNED_N = false,class Ptr,class RowStride,class ColStride,@@ -384,14 +391,14 @@ private:if constexpr (IS_LOAD) {if constexpr (IS_SAFE) {- const bool in_bounds = (row < local_row_limit) && (col < local_col_limit);+ const bool in_bounds = (ALIGNED_M || row < local_row_limit) && (ALIGNED_N || col < local_col_limit);frag[element_index] = in_bounds ? static_cast<T>(ptr[offset]) : T(0);} else {frag[element_index] = static_cast<T>(ptr[offset]);}} else {if constexpr (IS_SAFE) {- if ((row < local_row_limit) && (col < local_col_limit)) {+ if ((ALIGNED_M || row < local_row_limit) && (ALIGNED_N || col < local_col_limit)) {ptr[offset] = static_cast<U>(frag[element_index]);}} else {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.hindex b9fd94728..7c2d44dc8 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h@@ -127,16 +127,21 @@ METAL_FUNC static void fragment_matmul(constexpr ushort output_col_count = pair_output_rows ? 1 : cols;constexpr ushort output_col_step = pair_output_rows ? 1 : 2;+ constexpr bool interleave_rows = rows == 2 && cols == 2 && depth == 2 &&+ metal::is_same_v<typename OutputFragment::ElementType, int> &&+ metal::is_same_v<typename LeftFragment::ElementType, int8_t> &&+ metal::is_same_v<typename RightFragment::ElementType, int8_t>;+METAL_PRAGMA_UNROLL- for (ushort row = 0; row < rows; row += output_row_step) {+ for (ushort outer_index = 0; outer_index < (interleave_rows ? depth : rows); outer_index += output_row_step) {METAL_PRAGMA_UNROLLfor (ushort col = 0; col < output_col_count; col += output_col_step) {METAL_PRAGMA_UNROLL- for (ushort depth_index = 0; depth_index < depth; ++depth_index) {+ for (ushort inner_index = 0; inner_index < (interleave_rows ? rows : depth); ++inner_index) {matmul_paired_outputs(- row,+ interleave_rows ? inner_index : outer_index,col,- depth_index,+ interleave_rows ? outer_index : inner_index,uzu::integral_constant<MatmulMode, MatmulMode::multiply_accumulate>{});}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.hindex 71532ffd6..4524c4d93 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h@@ -29,6 +29,7 @@ METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint grUZU_CONST ushort W4_BITS = 4;UZU_CONST uint W4_SIGN_MASK = symmetric_zero_point<W4_BITS>() * 0x11111111u;+UZU_CONST uint W8_SIGN_MASK = symmetric_zero_point<8>() * 0x01010101u;UZU_CONST uint W4_NIBBLE_MASK = 0x0F0F0F0Fu;template <typename U, int N, int bits>diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.hindex 30e3519fc..23efd27b3 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h@@ -5,7 +5,7 @@using namespace metal;-template <typename T, typename BiasT>+template <typename T, typename BiasT, bool SINGLE_SIMDGROUP = false>static METAL_FUNC void apply_output_random_hadamard_transform(device T* output_block,const device int32_t* rht_factors_block,@@ -20,7 +20,8 @@ static METAL_FUNC void apply_output_random_hadamard_transform(const ushort stripes_per_row = tile_block_cols / HADAMARD_TRANSFORM_BLOCK_SIZE;const ushort simd_lane = thread_context.simd_lane_id;const uint total_work = uint(tile_block_rows) * uint(stripes_per_row);- for (uint cell = thread_context.simdgroup_index; cell < total_work; cell += simdgroup_count) {+ for (uint cell = SINGLE_SIMDGROUP ? 0 : thread_context.simdgroup_index; cell < total_work;+ cell += SINGLE_SIMDGROUP ? 1 : simdgroup_count) {const ushort row_local = ushort(cell / stripes_per_row);const ushort stripe = ushort(cell % stripes_per_row);const ushort col_local = stripe * ushort(HADAMARD_TRANSFORM_BLOCK_SIZE) + simd_lane;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.hindex 5fc88119f..77161f5ca 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h@@ -12,6 +12,7 @@ constexpr uint gemm_tiling_block_m(GemmTiling t) {: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 16+ : t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 16: t == GemmTiling::Tile16x128x256_Simdgroups1x4 ? 16: t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 32: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 64@@ -27,6 +28,7 @@ constexpr uint gemm_tiling_block_n(GemmTiling t) {: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 32+ : t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 32: t == GemmTiling::Tile16x128x256_Simdgroups1x4 ? 128: t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 64: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 32@@ -42,6 +44,7 @@ constexpr uint gemm_tiling_block_k(GemmTiling t) {: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 32: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 256+ : t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 256: t == GemmTiling::Tile16x128x256_Simdgroups1x4 ? 256: t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 256: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 256@@ -57,6 +60,7 @@ constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 1+ : t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 1: t == GemmTiling::Tile16x128x256_Simdgroups1x4 ? 1: t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 2: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 4@@ -72,6 +76,7 @@ constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 1+ : t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 1: t == GemmTiling::Tile16x128x256_Simdgroups1x4 ? 4: t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 2: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 1@@ -80,5 +85,9 @@ constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {: 0;}+constexpr uint gemm_tiling_simdgroups_per_k(GemmTiling t) {+ return t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 4 : 1;+}+} // namespace gemm} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex beee31ab8..d89dba965 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -46,6 +46,8 @@ struct MxuMmaCore {UZU_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);UZU_CONST ushort SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);UZU_CONST ushort SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);+ UZU_CONST ushort SIMDGROUPS_PER_K = gemm_tiling_simdgroups_per_k(GEMM_TILING);+ static_assert(SIMDGROUPS_PER_K == 1 || SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN == 1, "K slices share one tile");UZU_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;UZU_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;UZU_CONST ushort SIMDGROUP_BLOCK_K = static_cast<ushort>(MXU_SIMDGROUP_BLOCK_K);@@ -91,21 +93,24 @@ struct MxuMmaCore {const size_t block_row = size_t(geometry.block_row_start);const size_t block_col = size_t(geometry.block_col_start);- const uint k_offset = partition * params->aligned_inner_iterations * uint(THREADGROUP_BLOCK_K);+ const ushort k_slice = ushort(thread_context.simdgroup_index) / (SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN);+ const ushort tile_index = ushort(thread_context.simdgroup_index) % (SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN);+ const uint k_offset =+ (partition * SIMDGROUPS_PER_K + k_slice) * params->aligned_inner_iterations * uint(THREADGROUP_BLOCK_K);- const ushort tile_row_offset = SIMDGROUP_BLOCK_M * (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);- const ushort tile_col_offset = SIMDGROUP_BLOCK_N * (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);+ const ushort tile_row_offset = SIMDGROUP_BLOCK_M * (tile_index / SIMDGROUPS_PER_COLUMN);+ const ushort tile_col_offset = SIMDGROUP_BLOCK_N * (tile_index % SIMDGROUPS_PER_COLUMN);device OutputElementType* d_simdgroup = d + size_t(partition) * size_t(params->M) * size_t(params->N) +block_row * params->leading_dimension_d + block_col +tile_row_offset * params->leading_dimension_d + tile_col_offset;const short simdgroup_limit_m =- alignment.contains(GemmAlignment::M)+ !Left::QUANTIZED && alignment.contains(GemmAlignment::M)? SIMDGROUP_BLOCK_M: short(min(int(SIMDGROUP_BLOCK_M), int(params->M) - int(geometry.block_row_start + tile_row_offset)));const short simdgroup_limit_n =- alignment.contains(GemmAlignment::N)+ !Left::QUANTIZED && alignment.contains(GemmAlignment::N)? SIMDGROUP_BLOCK_N: short(min(int(SIMDGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start + tile_col_offset)));@@ -127,10 +132,11 @@ struct MxuMmaCore {const device RightElementType* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);dispatch_bool(- alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),+ (alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M)) && simdgroup_limit_m > 0,[&](auto aligned_m) {dispatch_bool(- alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),+ (alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N)) &&+ simdgroup_limit_n > 0,[&](auto aligned_n) {AccumFragment accumulator_tile =Schedule::template launch<MxuMmaCore, aligned_m.value, aligned_n.value>(@@ -143,6 +149,29 @@ struct MxuMmaCore {thread_context);+ if constexpr (SIMDGROUPS_PER_K > 1) {+ threadgroup float* k_partials = reinterpret_cast<threadgroup float*>(b_shared);+ thread AccumulatorType* values = accumulator_tile.elements();+ const ushort lane = ushort(thread_context.simd_lane_id);+ if (k_slice != 0) {+ for (ushort element = 0; element < AccumFragment::ELEMENTS_PER_FRAGMENT; ++element) {+ k_partials+ [((k_slice - 1) * AccumFragment::ELEMENTS_PER_FRAGMENT + element) * METAL_SIMD_SIZE + lane] =+ values[element];+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (k_slice != 0) {+ return;+ }+ for (ushort slice = 1; slice < SIMDGROUPS_PER_K; ++slice) {+ for (ushort element = 0; element < AccumFragment::ELEMENTS_PER_FRAGMENT; ++element) {+ values[element] += k_partials+ [((slice - 1) * AccumFragment::ELEMENTS_PER_FRAGMENT + element) * METAL_SIMD_SIZE + lane];+ }+ }+ }+if (apply_scale) {const AccumulatorType scale = AccumulatorType(params->ab_scale);accumulator_tile.map([&](auto value) { return value * scale; });@@ -176,7 +205,7 @@ struct MxuMmaCore {if constexpr (aligned_m.value && aligned_n.value) {accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));} else {- accumulator_tile.store_safe(+ accumulator_tile.template store_safe<aligned_m.value, aligned_n.value>(thread_context.simd_lane_id,d_simdgroup,int(params->leading_dimension_d),@@ -188,7 +217,34 @@ struct MxuMmaCore {});+ if (k_slice != 0) {+ return;+ }+if (output_transform.contains(GemmDTransform::RHT)) {+ if constexpr (+ schedules::IntegerSchedule<Left, Right>::CENTER_RIGHT_ZERO_POINTS &&+ SIMDGROUP_BLOCK_N == HADAMARD_TRANSFORM_BLOCK_SIZE+ ) {+ if (alignment.contains(GemmAlignment::M) && alignment.contains(GemmAlignment::N)) {+ if (simdgroup_limit_m <= 0 || simdgroup_limit_n <= 0) {+ return;+ }+ simdgroup_barrier(mem_flags::mem_device);+ apply_output_random_hadamard_transform<OutputElementType, RightElementType, true>(+ d_simdgroup,+ rht_factors + block_col + tile_col_offset,+ bias_simdgroup,+ apply_bias,+ SIMDGROUP_BLOCK_M,+ SIMDGROUP_BLOCK_N,+ params->leading_dimension_d,+ 1,+ thread_context+ );+ return;+ }+ }// Metal rejects a threadgroup barrier when the threadgroup contains one simdgroup.if constexpr (SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN == 1) {simdgroup_barrier(mem_flags::mem_device);diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.hindex 0197e014d..e2ba61926 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h@@ -42,7 +42,7 @@ struct RightOperand {UZU_CONST ushort BITS = QUANTIZED ? BITS_ : 0;UZU_CONST ushort GROUP_SIZE = QUANTIZED ? GROUP_SIZE_ : 0;UZU_CONST GemmBPrologueKind SCHEME = PROLOGUE;- UZU_CONST bool NEEDS_CORRECTION = QUANTIZED && PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant;+ UZU_CONST bool NEEDS_CORRECTION = PROLOGUE == GemmBPrologueKind::ScaleBiasDequant;UZU_CONST ushort CODE_ORIGIN = (!QUANTIZED || (BITS_ == 4 && PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant))? 0diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.hindex 4fe454596..a9f0ff38d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h@@ -54,58 +54,62 @@ load_int8_tile(const device int8_t* src, const int row_stride, const short simdg}});} else {- auto source = uzu::matmul::fragment_source(src, row_stride);- if constexpr (!ALIGNED) {- source = source.bounded(simdgroup_limit, Fragment::COL_FRAGMENTS * Fragment::FragmentOpsType::FRAGMENT_ROWS);- }- tile.load_from(simd_lane_id, source);+ using Ops = typename Fragment::FragmentOpsType;+ static_assert(+ Ops::THREAD_ELEMENT_COLS == 4 && sizeof(packed_char4) == 4 && alignof(packed_char4) == 1,+ "INT8 fragments require four-byte, byte-aligned loads"+ );+ const short2 position = Ops::get_position(simd_lane_id);+ const device int8_t* base = src + int(position.y) * row_stride + int(position.x);+ const short row_limit = simdgroup_limit - position.y;+ for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {+ METAL_PRAGMA_UNROLL+ for (ushort fragment_column = 0; fragment_column < Fragment::COL_FRAGMENTS; ++fragment_column) {+ char4 codes(0);+ if (ALIGNED || row_offset < row_limit) {+ codes = *reinterpret_cast<const device packed_char4*>(+ base + int(row_offset) * row_stride + int(fragment_column) * Ops::FRAGMENT_COLS+ );+ }+ reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, fragment_column))[row_slot] =+ as_type<uint>(codes);+ }+ });}return tile;}-template <typename Fragment, bool ALIGNED, bool CODES_GROUPED_BY_NIBBLE, bool HOISTED>+template <typename Fragment, bool ALIGNED, bool CODES_GROUPED_BY_NIBBLE>struct Int8Cursor {using Ops = typename Fragment::FragmentOpsType;UZU_CONST short BLOCK_K = short(Fragment::COL_FRAGMENTS * Ops::FRAGMENT_ROWS);struct PackedChunk {};- const device int8_t* origin;const device int8_t* address;int row_stride;short simdgroup_limit;ushort simd_lane_id;- METAL_FUNC Fragment load(const uint chunk_index) const thread {- const device int8_t* source = address;- if constexpr (!HOISTED) {- source += chunk_index * uint(BLOCK_K);- }+ METAL_FUNC Fragment load(const uint) const thread {return load_int8_tile<Fragment, ALIGNED, CODES_GROUPED_BY_NIBBLE>(- source,+ address,row_stride,simdgroup_limit,simd_lane_id);}- METAL_FUNC void advance() thread {- if constexpr (HOISTED) {- address += BLOCK_K;- }- }+ METAL_FUNC void advance() thread { address += BLOCK_K; }- METAL_FUNC void begin_k_group(const uint k_offset) thread {- if constexpr (!HOISTED) {- address = origin + k_offset;- }- }+ METAL_FUNC void begin_k_group(const uint) thread {}};-template <typename Fragment, bool ALIGNED, ushort CODE_ORIGIN>+template <typename Fragment, typename Operand, bool ALIGNED>struct W4Cursor {using Ops = typename Fragment::FragmentOpsType;UZU_CONST short BLOCK_K = short(Fragment::COL_FRAGMENTS * Ops::FRAGMENT_ROWS);+ UZU_CONST bool HAS_ZERO_POINTS = Operand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant;UZU_CONST uint CHUNK_BYTES = uint(BLOCK_K) * uint(get_bytes_per_pack<W4_BITS>()) / uint(get_pack_factor<W4_BITS>());@@ -123,10 +127,12 @@ struct W4Cursor {short tile_row_limit;short2 position;bool signed_codes;+ const device uint8_t* zero_points;+ uint zero_point_group_stride_bytes;METAL_FUNC PackedChunk fetch(const uint chunk_index) const thread {// Padding decodes to zero after the optional sign flip and code-origin adjustment.- const uint padding_word = (CODE_ORIGIN == 0) == signed_codes ? W4_SIGN_MASK : 0u;+ const uint padding_word = (Operand::CODE_ORIGIN == 0) == signed_codes ? W4_SIGN_MASK : 0u;const device uint8_t* base =current + chunk_index * CHUNK_BYTES + int(position.y) * row_stride_bytes + int(position.x);const short row_limit = tile_row_limit - position.y;@@ -143,14 +149,25 @@ struct W4Cursor {METAL_FUNC Fragment decode(const thread PackedChunk& packed_chunk) const thread {Fragment tile;- for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short) {+ for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {const uint word =packed_chunk.words[PackedChunk::word_index(fragment_row, row_slot)] ^ (signed_codes ? W4_SIGN_MASK : 0u);uint low = word & W4_NIBBLE_MASK;uint high = (word >> W4_BITS) & W4_NIBBLE_MASK;- if constexpr (CODE_ORIGIN != 0) {- low = as_type<uint>(as_type<char4>(low) - char4(char(CODE_ORIGIN)));- high = as_type<uint>(as_type<char4>(high) - char4(char(CODE_ORIGIN)));+ if constexpr (HAS_ZERO_POINTS) {+ const uint column = uint(position.y + row_offset);+ uint zero_point = 0;+ if (ALIGNED || short(column) < tile_row_limit) {+ // Hoisting these loads introduces register spills in the M5 fused-RHT specialization.+ const device volatile uint8_t* group = zero_points;+ zero_point = decode_zero_point<W4_BITS>(group[column / get_pack_factor<W4_BITS>()], column);+ }+ const uint origin_offset = (symmetric_zero_point<8>() - zero_point) * 0x01010101u;+ low = (low + origin_offset) ^ W8_SIGN_MASK;+ high = (high + origin_offset) ^ W8_SIGN_MASK;+ } else if constexpr (Operand::CODE_ORIGIN != 0) {+ low = as_type<uint>(as_type<char4>(low) - char4(char(Operand::CODE_ORIGIN)));+ high = as_type<uint>(as_type<char4>(high) - char4(char(Operand::CODE_ORIGIN)));}reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, 0))[row_slot] = low;reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, 1))[row_slot] = high;@@ -162,10 +179,16 @@ struct W4Cursor {METAL_FUNC void advance() thread { current += CHUNK_BYTES; }- METAL_FUNC void begin_k_group(const uint) thread {}+ METAL_FUNC void begin_k_group(const uint k_offset) thread {+ if constexpr (HAS_ZERO_POINTS) {+ if (k_offset != 0) {+ zero_points += zero_point_group_stride_bytes;+ }+ }+ }};-template <bool HOIST_OPERAND_ADDRESSING, typename Core, typename LeftOperand, bool ALIGNED>+template <typename Core, typename LeftOperand, bool ALIGNED>static METAL_FUNC auto make_left_cursor(const typename Core::LeftStorage source,const constant uzu::matmul::GemmParams* params,@@ -174,8 +197,7 @@ static METAL_FUNC auto make_left_cursor() {using Fragment = uzu::matmul::Fragment<int8_t, Core::TILES_M, Core::TILES_K, typename Core::FragmentOps>;const device int8_t* origin = source.codes + size_t(tile.abs_row_base) * params->leading_dimension_a + tile.k_offset;- return Int8Cursor<Fragment, ALIGNED, LeftOperand::GROUPED_BY_NIBBLE, HOIST_OPERAND_ADDRESSING>{- origin,+ return Int8Cursor<Fragment, ALIGNED, LeftOperand::GROUPED_BY_NIBBLE>{origin,int(params->leading_dimension_a),tile.simdgroup_limit_m,@@ -183,7 +205,7 @@ static METAL_FUNC auto make_left_cursor(};}-template <bool HOIST_OPERAND_ADDRESSING, typename Core, typename Operand, bool ALIGNED>+template <typename Core, typename Operand, bool ALIGNED>static METAL_FUNC auto make_right_cursor(const typename Core::RightStorage source,const constant uzu::matmul::GemmParams* params,@@ -200,18 +222,26 @@ static METAL_FUNC auto make_right_cursor(if constexpr (Operand::BITS == 4) {static_assert(Core::TILES_K == get_pack_factor<Operand::BITS>(), "W4 requires two K fragments");- return W4Cursor<Fragment, ALIGNED, Operand::CODE_ORIGIN>{+ using Cursor = W4Cursor<Fragment, Operand, ALIGNED>;+ const device uint8_t* zero_points = nullptr;+ const uint zero_point_group_stride_bytes = params->zero_point_group_stride / get_pack_factor<W4_BITS>();+ if constexpr (Cursor::HAS_ZERO_POINTS) {+ zero_points = source.zp() + (tile.k_offset / Operand::GROUP_SIZE) * zero_point_group_stride_bytes ++ tile.absolute_column_base() / get_pack_factor<W4_BITS>();+ }+ return Cursor{current,row_stride_bytes,tile.simdgroup_limit_n,Ops::get_position(thread_context.simd_lane_id),- source.signed_codes+ source.signed_codes,+ zero_points,+ zero_point_group_stride_bytes};} else {static_assert(Operand::BITS == 8, "integer tile cursors support 4-bit and 8-bit codes");const device int8_t* origin_int8 = reinterpret_cast<const device int8_t*>(current);- return Int8Cursor<Fragment, ALIGNED, false, HOIST_OPERAND_ADDRESSING>{- origin_int8,+ return Int8Cursor<Fragment, ALIGNED, false>{origin_int8,row_stride_bytes,tile.simdgroup_limit_n,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.hindex de9f50c10..044de6e91 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h@@ -55,13 +55,14 @@ struct MetadataContext {uint left_group_count;uint right_group_count;uint right_scale_group_stride;- uint right_zero_point_group_stride;};template <typename LeftOperand, typename RightOperand>struct IntegerSchedule {UZU_CONST uint RIGHT_GROUP_SIZE = uint(RightOperand::GROUP_SIZE);- UZU_CONST bool HAS_ZERO_POINTS = RightOperand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant;+ UZU_CONST bool CENTER_RIGHT_ZERO_POINTS =+ LeftOperand::QUANTIZED && RightOperand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant;+ static_assert(!CENTER_RIGHT_ZERO_POINTS || RightOperand::BITS == 4, "zero points are 4-bit");UZU_CONST bool HAS_BIAS = RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant;UZU_CONST uchar RIGHT_CODE_OFFSET = uchar(RightOperand::CODE_ORIGIN);@@ -72,7 +73,7 @@ struct IntegerSchedule {UZU_CONST ushort LEFT_VALUE_COUNT = Core::TILES_M * THREAD_ROWS_PER_FRAGMENT;float scales[LEFT_VALUE_COUNT];- metal::conditional_t<HAS_ZERO_POINTS, int, float> group_corrections[LEFT_VALUE_COUNT];+ float group_corrections[LEFT_VALUE_COUNT];struct LeftSlot {uint row_index;@@ -105,7 +106,7 @@ struct IntegerSchedule {});}- if constexpr (HAS_ZERO_POINTS || HAS_BIAS) {+ if constexpr (HAS_BIAS) {const uint right_group_index = k_offset / RIGHT_GROUP_SIZE;for_each_static_index<int(LEFT_VALUE_COUNT)>([&](const ushort index) {const LeftSlot slot = left_slot(metadata_context, index);@@ -113,11 +114,7 @@ struct IntegerSchedule {slot.live? left.correction_sums()[slot.row_index * metadata_context.right_group_count + right_group_index]: 0;- if constexpr (HAS_ZERO_POINTS) {- group_corrections[index] = code_sum;- } else {- group_corrections[index] = slot.live ? (scales[index] * float(code_sum)) : 0.0f;- }+ group_corrections[index] = slot.live ? (scales[index] * float(code_sum)) : 0.0f;});}}@@ -128,7 +125,6 @@ struct IntegerSchedule {using Ops = typename Core::FragmentOps;using ScaleElement = typename RightOperand::ScaleElement;using ScaleVector = vec<ScaleElement, Ops::THREAD_ELEMENT_COLS>;- using ZeroPointVector = uchar4;static_assert(Ops::THREAD_ELEMENT_COLS == uzu::matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT,"group-major metadata must use the metadata load width"@@ -136,7 +132,6 @@ struct IntegerSchedule {ScaleVector scales[Core::TILES_N];ScaleVector bias_offsets[Core::TILES_N];- ZeroPointVector zero_points[Core::TILES_N];METAL_FUNC void load(const typename Core::RightStorage right,@@ -159,25 +154,6 @@ struct IntegerSchedule {);bias_offsets[tile_n] += scales[tile_n] * ScaleElement(RIGHT_CODE_OFFSET);}- if constexpr (HAS_ZERO_POINTS) {- constexpr uint ZERO_POINT_PACK_FACTOR = 8u / uint(RightOperand::BITS);- const device uint8_t* zero_point_row =- right.zp() +- right_group_index * (metadata_context.right_zero_point_group_stride / ZERO_POINT_PACK_FACTOR);- uint right_zero_point_column_start = metadata_context.right_column_base + uint(right_column_offset);- if constexpr (!ALIGNED_N) {- right_zero_point_column_start = clamp_group_output_column(- right_zero_point_column_start,- metadata_context.right_zero_point_group_stride- );- }- static_assert(RightOperand::BITS == 4, "zero points are 4-bit");- const ushort packed =- *reinterpret_cast<const device ushort*>(zero_point_row + (right_zero_point_column_start >> 1));- uint spread = (uint(packed) | (uint(packed) << 8)) & 0x00FF00FFu;- spread = (spread | (spread << 4)) & 0x0F0F0F0Fu;- zero_points[tile_n] = as_type<ZeroPointVector>(spread);- }if constexpr (!ALIGNED_N) {const bool4 live =(short4(right_column_offset) + short4(0, 1, 2, 3)) < short4(metadata_context.right_column_limit);@@ -185,9 +161,6 @@ struct IntegerSchedule {if constexpr (HAS_BIAS) {bias_offsets[tile_n] = select(ScaleVector(0), bias_offsets[tile_n], live);}- if constexpr (HAS_ZERO_POINTS) {- zero_points[tile_n] = select(ZeroPointVector(RIGHT_CODE_OFFSET), zero_points[tile_n], live);- }}});}@@ -203,7 +176,14 @@ struct IntegerSchedule {template <typename Core>static constexpr bool prefetches_int4_chunks() {- return Core::TILES_M == 1 && chunks_per_k_group<Core>() > 1 && RightOperand::BITS == 4;+ return (Core::TILES_M == 1 || (Core::TILES_M == 2 && Core::TILES_N == 2 && CENTER_RIGHT_ZERO_POINTS)) &&+ chunks_per_k_group<Core>() > 1 && RightOperand::BITS == 4;+ }++ template <typename Core, bool ALIGNED_M, bool ALIGNED_N>+ static constexpr bool multiplies_device_groups() {+ return RightOperand::BITS == 8 && RIGHT_GROUP_SIZE == 64 && Core::TILES_M == 2 && Core::TILES_N == 2 && ALIGNED_M &&+ ALIGNED_N;}template <typename RightCodes, int K_GROUP_COUNT, int CHUNKS_PER_K_GROUP>@@ -218,6 +198,50 @@ struct IntegerSchedule {});}+ template <typename Core>+ static METAL_FUNC GroupProducts<Core> multiply_device_group(+ const typename Core::LeftStorage left,+ const typename Core::RightStorage right,+ const constant uzu::matmul::GemmParams* params,+ const TileContext tile,+ const uint k_offset+ ) {+ using Ops = typename Core::FragmentOps;+ constexpr auto descriptor = mpp::tensor_ops::matmul2d_descriptor(+ Core::SIMDGROUP_BLOCK_M,+ Core::SIMDGROUP_BLOCK_N,+ RIGHT_GROUP_SIZE,+ false,+ true,+ true,+ mpp::tensor_ops::matmul2d_descriptor::mode::multiply+ );+ mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> op;+ const device int8_t* right_origin = reinterpret_cast<const device int8_t*>(right.codes) ++ size_t(tile.absolute_column_base()) * params->K + k_offset;+ auto right_tensor = tensor(+ const_cast<device int8_t*>(right_origin),+ extents<int, RIGHT_GROUP_SIZE, Core::SIMDGROUP_BLOCK_N>{},+ array<int, 2>{1, int(params->K)}+ );+ auto cooperative_right = op.template get_right_input_cooperative_tensor<int8_t, int8_t, int>();+ cooperative_right.load(right_tensor);+ const device int8_t* left_origin = left.codes + size_t(tile.abs_row_base) * params->leading_dimension_a + k_offset;+ auto left_tensor = tensor(+ const_cast<device int8_t*>(left_origin),+ extents<int, RIGHT_GROUP_SIZE, Core::SIMDGROUP_BLOCK_M>{},+ array<int, 2>{1, int(params->leading_dimension_a)}+ );+ auto destination =+ op.template get_destination_cooperative_tensor<decltype(left_tensor), decltype(cooperative_right), int>();+ op.run(left_tensor, cooperative_right, destination);+ GroupProducts<Core> product;+ for_each_static_index<int(Core::TILES_M) * int(Core::TILES_N) * int(Ops::ELEMENTS_PER_THREAD)>(+ [&](const ushort element) { product.elements()[element] = destination[element]; }+ );+ return product;+ }+template <typename Core, typename LeftCodes, typename RightCodes, int CHUNKS_PER_K_GROUP>static METAL_FUNC GroupProducts<Core> multiply_k_group(thread LeftCodes& left_codes,@@ -235,14 +259,16 @@ struct IntegerSchedule {right_codes.advance();});} else {- METAL_PRAGMA_NO_UNROLL- for (int chunk = 0; chunk < CHUNKS_PER_K_GROUP; ++chunk) {+ decltype(right_codes.load(0u)) right_tiles[CHUNKS_PER_K_GROUP];+ for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {+ right_tiles[chunk] = right_codes.load(uint(chunk));+ right_codes.advance();+ });+ for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {auto left_tile = left_codes.load(uint(chunk));- auto right_tile = right_codes.load(uint(chunk));- uzu::matmul::fragment_mma(group_product, left_tile, right_tile);+ uzu::matmul::fragment_mma(group_product, left_tile, right_tiles[chunk]);left_codes.advance();- right_codes.advance();- }+ });}return group_product;}@@ -265,13 +291,10 @@ struct IntegerSchedule {const ushort right_column = element % Ops::THREAD_ELEMENT_COLS;const ushort left_row = tile_m * Ops::THREAD_ELEMENT_ROWS + element / Ops::THREAD_ELEMENT_COLS;const float right_scale = float(right_metadata.scales[tile_n][right_column]);- int centered_product = group_product_value[element];- if constexpr (HAS_ZERO_POINTS) {- centered_product += (int(RIGHT_CODE_OFFSET) - int(right_metadata.zero_points[tile_n][right_column])) *- left_metadata.group_corrections[left_row];- }accumulated[element] =- fma(left_metadata.scales[left_row] * right_scale, float(centered_product), accumulated[element]);+ fma(left_metadata.scales[left_row] * right_scale,+ float(group_product_value[element]),+ accumulated[element]);if constexpr (HAS_BIAS) {accumulated[element] =fma(right_metadata.bias_offsets[tile_n][right_column],@@ -300,23 +323,18 @@ struct IntegerSchedule {LeftOperand::GROUP_SIZE % RightOperand::GROUP_SIZE == 0,"the left group must hold a whole number of right groups");+ if constexpr (!ALIGNED_M || !ALIGNED_N) {+ if (tile.simdgroup_limit_m <= 0 || tile.simdgroup_limit_n <= 0) {+ typename Core::AccumFragment empty;+ empty.clear();+ return empty;+ }+ }- constexpr bool HOIST_OPERAND_ADDRESSING =- !(RightOperand::SCHEME == GemmBPrologueKind::ScaleSymmetricDequant &&- Core::TILING == GemmTiling::Tile128x128x256_Simdgroups4x4);-- auto left_codes = quantized::make_left_cursor<HOIST_OPERAND_ADDRESSING, Core, LeftOperand, ALIGNED_M>(- left_storage,- params,- tile,- thread_context- );- auto right_codes = quantized::make_right_cursor<HOIST_OPERAND_ADDRESSING, Core, RightOperand, ALIGNED_N>(- right_storage,- params,- tile,- thread_context- );+ auto left_codes =+ quantized::make_left_cursor<Core, LeftOperand, ALIGNED_M>(left_storage, params, tile, thread_context);+ auto right_codes =+ quantized::make_right_cursor<Core, RightOperand, ALIGNED_N>(right_storage, params, tile, thread_context);const short2 position = Core::FragmentOps::get_position(thread_context.simd_lane_id);const MetadataContext metadata_context = {@@ -327,7 +345,6 @@ struct IntegerSchedule {uint(params->K) / uint(LeftOperand::GROUP_SIZE),uint(params->K) / RIGHT_GROUP_SIZE,params->scale_group_stride,- params->zero_point_group_stride,};const uint first_right_group = tile.k_offset / RIGHT_GROUP_SIZE;@@ -340,11 +357,36 @@ struct IntegerSchedule {constexpr int CHUNKS_PER_K_GROUP = chunks_per_k_group<Core>();constexpr bool PREFETCH_INT4_CHUNKS = prefetches_int4_chunks<Core>();constexpr bool IS_ALIGNED_M_INT4_TILE16X32 =- PREFETCH_INT4_CHUNKS && ALIGNED_M && Core::TILING == GemmTiling::Tile16x32x256_Simdgroups1x1;+ PREFETCH_INT4_CHUNKS && ALIGNED_M && Core::THREADGROUP_BLOCK_M == 16 && Core::THREADGROUP_BLOCK_N == 32;constexpr int K_GROUPS_PER_FETCH = IS_ALIGNED_M_INT4_TILE16X32 ? 4 : 1;constexpr int MIN_K_GROUPS_PER_THREADGROUP_FOR_BATCHED_FETCH = 16;using PackedRightChunk = typename decltype(right_codes)::PackedChunk;+ auto multiply_group = [&](const int right_group_index,+ const thread PackedRightChunk(&packed_right_chunks)[CHUNKS_PER_K_GROUP]) {+ const uint right_group_offset = uint(right_group_index) * RIGHT_GROUP_SIZE;+ left_codes.begin_k_group(right_group_offset);+ right_codes.begin_k_group(right_group_offset);+ auto load_metadata = [&]() {+ left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);+ right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));+ };+ if constexpr (!PREFETCH_INT4_CHUNKS) {+ load_metadata();+ }+ GroupProducts<Core> group_product;+ if constexpr (multiplies_device_groups<Core, ALIGNED_M, ALIGNED_N>()) {+ group_product =+ multiply_device_group<Core>(left_storage, right_storage, params, tile, tile.k_offset + right_group_offset);+ } else {+ group_product = multiply_k_group<Core>(left_codes, right_codes, packed_right_chunks);+ }+ if constexpr (PREFETCH_INT4_CHUNKS) {+ load_metadata();+ }+ accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);+ };+int right_group_index = 0;if constexpr (K_GROUPS_PER_FETCH > 1) {const int batched_fetch_right_group_limit =@@ -355,14 +397,7 @@ struct IntegerSchedule {PackedRightChunk packed_right_chunks[K_GROUPS_PER_FETCH][CHUNKS_PER_K_GROUP];fetch_k_groups(right_codes, packed_right_chunks);for_each_static_index<K_GROUPS_PER_FETCH>([&](const ushort k_group) {- const int fetched_right_group_index = right_group_index + int(k_group);- const uint right_group_offset = uint(fetched_right_group_index) * RIGHT_GROUP_SIZE;- left_codes.begin_k_group(right_group_offset);- right_codes.begin_k_group(right_group_offset);- auto group_product = multiply_k_group<Core>(left_codes, right_codes, packed_right_chunks[k_group]);- left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);- right_metadata.load(right_storage, metadata_context, first_right_group + uint(fetched_right_group_index));- accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);+ multiply_group(right_group_index + int(k_group), packed_right_chunks[k_group]);});}}@@ -372,13 +407,7 @@ struct IntegerSchedule {if constexpr (PREFETCH_INT4_CHUNKS) {fetch_k_groups(right_codes, packed_right_chunks);}- const uint right_group_offset = uint(right_group_index) * RIGHT_GROUP_SIZE;- left_codes.begin_k_group(right_group_offset);- right_codes.begin_k_group(right_group_offset);- auto group_product = multiply_k_group<Core>(left_codes, right_codes, packed_right_chunks[0]);- left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);- right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));- accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);+ multiply_group(right_group_index, packed_right_chunks[0]);}return accumulator;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metalindex db6f59292..d1d234faa 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -11,14 +11,20 @@ using namespace metal;using namespace uzu::gemm;#define A_IS_INT8 (A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)-#define NEEDS_ASYMMETRIC_WEIGHT_CORRECTION (A_IS_INT8 && B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant)+#define NEEDS_ASYMMETRIC_WEIGHT_CORRECTION (A_IS_INT8 && B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant)#define GEMM_MXU_QUANT (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision && !A_IS_INT8)#define GEMM_TGA_ELEMENTS \((USE_MXU) ? 1 : (gemm_tiling_block_m(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))+#define GEMM_K_PARTIAL_ELEMENTS \+ (gemm_tiling_simdgroups_per_k(GEMM_TILING) == 1 \+ ? 1 \+ : (gemm_tiling_simdgroups_per_k(GEMM_TILING) - 1) * gemm_tiling_block_m(GEMM_TILING) * \+ gemm_tiling_block_n(GEMM_TILING) * uint(sizeof(float) / sizeof(BT)))// The A8 MXU route reads its scales into registers, so it asks for no// threadgroup B memory.#define GEMM_TGB_ELEMENTS \- ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) : 1) \+ ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) \+ : GEMM_K_PARTIAL_ELEMENTS) \: (gemm_tiling_block_n(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))template <@@ -45,6 +51,7 @@ VARIANTS(GemmTiling::Tile64x64x32_Simdgroups2x2,GemmTiling::Tile32x32x32_Simdgroups2x2,GemmTiling::Tile16x32x256_Simdgroups1x1,+ GemmTiling::Tile16x32x256_Simdgroups1x1x4,GemmTiling::Tile16x128x256_Simdgroups1x4,GemmTiling::Tile32x64x256_Simdgroups2x2,GemmTiling::Tile64x32x256_Simdgroups4x1,@@ -68,6 +75,7 @@ VARIANTS(A_GROUP_SIZE, 0, 128)CONSTRAINT(USE_MXU ==(GEMM_TILING == GemmTiling::Tile16x32x256_Simdgroups1x1 ||+ GEMM_TILING == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ||GEMM_TILING == GemmTiling::Tile16x128x256_Simdgroups1x4 ||GEMM_TILING == GemmTiling::Tile32x64x256_Simdgroups2x2 ||GEMM_TILING == GemmTiling::Tile64x32x256_Simdgroups4x1 ||@@ -93,6 +101,9 @@ CONSTRAINT((TRANSPOSE_B &&(B_PROLOGUE == GemmBPrologueKind::FullPrecision ||A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)))+CONSTRAINT(+ GEMM_TILING != GemmTiling::Tile16x32x256_Simdgroups1x1x4 ||+ A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)CONSTRAINT(GEMM_TILING != GemmTiling::Tile16x128x256_Simdgroups1x4 ||(TRANSPOSE_B && B_PROLOGUE == GemmBPrologueKind::FullPrecision))@@ -100,7 +111,7 @@ CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || USE_MXU)CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision ||(TRANSPOSE_B && B_PROLOGUE != GemmBPrologueKind::FullPrecision))-CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || (AT == "bfloat" && DT == "bfloat"))+CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || AT == "bfloat")CONSTRAINT((A_PROLOGUE == GemmAPrologueKind::FullPrecision) == (A_GROUP_SIZE == 0))KERNEL(Gemm)(const device AT* a OPTIONAL(A_PROLOGUE == GemmAPrologueKind::FullPrecision),@@ -133,7 +144,7 @@ KERNEL(Gemm)(const uint group_z GROUPS(group_count_z),const uint thread_x THREADS(METAL_SIMD_SIZE),const uint thread_y THREADS(gemm_tiling_simdgroups_per_column(GEMM_TILING)),- const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING)),+ const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING) * gemm_tiling_simdgroups_per_k(GEMM_TILING)),const ThreadContext thread_context) {(void)group_x;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metalindex 7faab6ed8..553259e9d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal@@ -5,10 +5,12 @@using namespace metal;using namespace uzu::gemm;-template <typename T>+template <typename Input, typename T>+VARIANTS(Input, float, bfloat)VARIANTS(T, float, bfloat)+CONSTRAINT(Input == "float" || Input == T)KERNEL(GemmSplitKReduce)(- const device T* partial_sums,+ const device Input* partial_sums,device T* output,const device T* output_biasOPTIONAL(output_transform.contains(GemmDTransform::BIAS)),@@ -36,7 +38,7 @@ KERNEL(GemmSplitKReduce)(}device vec<T, 4>* output_vectors = reinterpret_cast<device vec<T, 4>*>(output);- const device vec<T, 4>* partial_sum_vectors = reinterpret_cast<const device vec<T, 4>*>(partial_sums);+ const device vec<Input, 4>* partial_sum_vectors = reinterpret_cast<const device vec<Input, 4>*>(partial_sums);float4 accumulator = float4(0.0f);for (uint partition = 0u; partition < partition_count; ++partition) {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metalindex 137171472..5e5fdaffd 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal@@ -108,7 +108,7 @@ KERNEL(GemmTrellis)(products.clear();uzu::dispatch_bool(alignment.contains(GemmAlignment::M) || token_end == Core::SIMDGROUP_BLOCK_M, [&](auto aligned_m) {uzu::dispatch_bool(alignment.contains(GemmAlignment::N) || row_end == Core::SIMDGROUP_BLOCK_N, [&](auto aligned_n) {- auto left = quantized::make_left_cursor<true, Core, LeftOperand, aligned_m.value>(+ auto left = quantized::make_left_cursor<Core, LeftOperand, aligned_m.value>(left_storage,params,tile_context,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex fe4ae2bdf..158082bbf 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -36,11 +36,15 @@ pub struct GemmKernel {input_data_type: DataType,output_data_type: DataType,trellis: TrellisGemm,- kernels: HashMap<GemmSpecialization, GemmMetalKernel>,- split_k_reduce: HashMap<GemmDTransform, GemmSplitKReduceMetalKernel>,+ kernels: HashMap<(GemmSpecialization, DataType), GemmMetalKernel>,+ split_k_reduce: HashMap<(GemmDTransform, DataType), GemmSplitKReduceMetalKernel>,}impl GemmKernel {+ pub fn requires_activation_group_sums(shape: &MatmulShape) -> bool {+ shape.b_prologue == GemmBPrologueKind::ScaleBiasDequant+ }+pub fn new(weights_data_type: DataType,input_data_type: DataType,@@ -60,15 +64,16 @@ impl GemmKernel {&mut self,context: &MetalContext,specialization: GemmSpecialization,+ output_data_type: DataType,) -> Result<&GemmMetalKernel, MetalError> {- match self.kernels.entry(specialization) {+ match self.kernels.entry((specialization, output_data_type)) {Entry::Occupied(entry) => Ok(entry.into_mut()),Entry::Vacant(entry) => {let kernel = GemmMetalKernel::new(context,self.input_data_type,self.weights_data_type,- self.output_data_type,+ output_data_type,specialization.tiling,specialization.transpose_b,specialization.use_mxu,@@ -90,11 +95,17 @@ impl GemmKernel {&mut self,context: &MetalContext,output_transform: GemmDTransform,+ partial_data_type: DataType,) -> Result<&GemmSplitKReduceMetalKernel, MetalError> {- match self.split_k_reduce.entry(output_transform) {+ match self.split_k_reduce.entry((output_transform, partial_data_type)) {Entry::Occupied(entry) => Ok(entry.into_mut()),Entry::Vacant(entry) => {- let kernel = GemmSplitKReduceMetalKernel::new(context, self.output_data_type, output_transform)?;+ let kernel = GemmSplitKReduceMetalKernel::new(+ context,+ partial_data_type,+ self.output_data_type,+ output_transform,+ )?;Ok(entry.insert(kernel))},}@@ -288,6 +299,8 @@ impl GemmKernel {},};+ let a_group_sums = a_group_sums.filter(|_| Self::requires_activation_group_sums(&shape));+if plan.split_k > 1 {self.encode_split_k(a,@@ -312,8 +325,12 @@ impl GemmKernel {}let tiling = plan.tiling;- let alignment =- GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);+ let (alignment_m, alignment_n) = if use_mxu && a_int8.is_some() {+ (tiling.block_m() / tiling.simdgroups_m(), tiling.block_n() / tiling.simdgroups_n())+ } else {+ (tiling.block_m(), tiling.block_n())+ };+ let alignment = GemmAlignment::new(m % alignment_m == 0, n % alignment_n == 0, k % tiling.block_k() == 0);let mut params = gemm_params(shape, plan, ab_scale, scale_strides, zero_point_strides);let threadgroups_per_row = n.div_ceil(tiling.block_n());let threadgroups_per_column = m.div_ceil(tiling.block_m());@@ -349,7 +366,7 @@ impl GemmKernel {a_prologue,a_group_size,)?;- let kernel = self.get_or_create(command_buffer.context(), specialization)?;+ let kernel = self.get_or_create(command_buffer.context(), specialization, self.output_data_type)?;kernel.encode(a_full_precision,weights,@@ -413,6 +430,7 @@ impl GemmKernel {.into());},};+ let a_group_sums = a_group_sums.filter(|_| Self::requires_activation_group_sums(&shape));let tiling = plan.tiling;let split_k = plan.split_k;let kp = k / split_k;@@ -432,11 +450,16 @@ impl GemmKernel {)?;let elem = (m as usize) * (n as usize);- let slice_bytes = elem * self.output_data_type.size_in_bytes();+ let partial_data_type = if a_int8.is_some() {+ DataType::F32+ } else {+ self.output_data_type+ };+ let slice_bytes = elem * partial_data_type.size_in_bytes();let mut temp = command_buffer.allocate_scratch(split_k as usize * slice_bytes)?;let mut params = gemm_params(shape, plan, 1.0, scale_strides, zero_point_strides);- params.aligned_inner_iterations = kp / k_step;- let part_kernel = self.get_or_create(command_buffer.context(), part_spec)?;+ params.aligned_inner_iterations = kp / k_step / tiling.simdgroups_k();+ let part_kernel = self.get_or_create(command_buffer.context(), part_spec, partial_data_type)?;part_kernel.encode(a_full_precision,weights,@@ -470,7 +493,8 @@ impl GemmKernel {} else {None};- let reduce = self.get_or_create_split_k_reduce(command_buffer.context(), reduce_transform)?;+ let reduce =+ self.get_or_create_split_k_reduce(command_buffer.context(), reduce_transform, partial_data_type)?;reduce.encode(&temp, d.reborrow(), bias_arg, elem as u32, split_k, group_count, n, scale_arg, command_buffer);Ok(())@@ -491,9 +515,7 @@ fn validate_int8_left_operand(}.into());}- let needs_group_sums =- matches!(shape.b_prologue, GemmBPrologueKind::ScaleBiasDequant | GemmBPrologueKind::ScaleZeroPointDequant);- if needs_group_sums && !has_group_sums {+ if GemmKernel::requires_activation_group_sums(&shape) && !has_group_sums {return Err(MatmulError::IncompatibleA {path: "Gemm",reason: "quantized correction requires left group sums",@@ -542,7 +564,8 @@ fn gemm_params(leading_dimension_d: n,threadgroups_per_row: n.div_ceil(tiling.block_n()),threadgroups_per_column: m.div_ceil(tiling.block_m()),- aligned_inner_iterations: outer_block_k(shape, plan.engine, plan.tiling).map_or(0, |step| k / step),+ aligned_inner_iterations: outer_block_k(shape, plan.engine, plan.tiling)+ .map_or(0, |step| k / step / tiling.simdgroups_k()),use_morton: false,ab_scale,scale_output_stride: scale_strides.output_stride,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rsindex b7beec182..db4e537a4 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs@@ -11,6 +11,7 @@ use crate::backends::common::gpu_types::gemm::GemmTiling;const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;const MXU_SKINNY_M_MAX: u32 = 16;+const MXU_K_SPLIT_FULL_BLOCK_N_MAX: u32 = 16384;const SKINNY_SQUARE_K_MAX: u32 = 2560;const READOUT_N_TO_K_RATIO: u32 = 32;const WIDE_N_DEEP_K_MIN: u32 = 4096;@@ -29,6 +30,7 @@ const SIMDGROUP_QUANT_WIDE_N_MIN: u32 = 6144;const SPLIT_K_TARGET_TILES_FP: u32 = 512;pub(super) const SPLIT_K_TARGET_TILES_A8: u32 = 256;pub(super) const SPLIT_K_TARGET_TILES_A8_TILE16X32: u32 = 4 * SPLIT_K_TARGET_TILES_A8;+const SPLIT_K_TARGET_TILES_A8_TILE16X32_K_SPLIT: u32 = 2 * SPLIT_K_TARGET_TILES_A8;pub(super) const SPLIT_K_TARGET_TILES_A8_TILE32_W4: u32 = 512;const SPLIT_K_TARGET_TILES_A8_TILE32_W8: u32 = 1024;@@ -55,6 +57,19 @@ pub(super) fn mxu_mn_tile(}}+pub(super) fn mxu_int8_gemm_tile(+ m: u32,+ n: u32,+) -> GemmTiling {+ match mxu_mn_tile(true, m, n) {+ GemmTiling::Tile128x128x256_Simdgroups4x4 => MXU_DEFAULT_TILE,+ GemmTiling::Tile16x32x256_Simdgroups1x1 if m < MXU_SKINNY_M_MAX || n <= MXU_K_SPLIT_FULL_BLOCK_N_MAX => {+ GemmTiling::Tile16x32x256_Simdgroups1x1x4+ },+ tiling => tiling,+ }+}+pub(super) fn mxu_fp_tile(m: u32,n: u32,@@ -135,6 +150,7 @@ pub(super) fn split_k_target_tiles((true, GemmTiling::Tile32x64x256_Simdgroups2x2, Some(4)) => SPLIT_K_TARGET_TILES_A8_TILE32_W4,(true, GemmTiling::Tile32x64x256_Simdgroups2x2, _) => SPLIT_K_TARGET_TILES_A8_TILE32_W8,(true, GemmTiling::Tile16x32x256_Simdgroups1x1, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32,+ (true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32_K_SPLIT,(true, _, _) => SPLIT_K_TARGET_TILES_A8,(false, _, _) => SPLIT_K_TARGET_TILES_FP,}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex 4f8921c8e..86fb18c70 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -117,7 +117,7 @@ impl GemmProblem {let Some(step) = outer_block_k(shape, engine, tiling) else {return 1;};- let align = step.max(ACTIVATION_SCALE_GROUP_SIZE);+ let align = step.max(ACTIVATION_SCALE_GROUP_SIZE) * tiling.simdgroups_k();let target_tiles = policy::split_k_target_tiles(!shape.a_full_precision, tiling, shape.b_bits);let mut split_k = (target_tiles / base_tiles).max(1).min((shape.k / align).max(1));if !shape.a_full_precision && engine == GemmEngine::Mxu && tiling.block_k() != 0 {@@ -203,13 +203,21 @@ fn select_tiling(match engine {GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m, shape.n, apple_gpu_family),GemmEngine::Simdgroup => policy::simdgroup_fp_tile(shape.m, shape.n, shape.k),- GemmEngine::Mxu if !shape.a_full_precision || shape.is_quant() => {- policy::mxu_mn_tile(!shape.a_full_precision, shape.m, shape.n)+ GemmEngine::Mxu if !shape.a_full_precision => match policy::mxu_int8_gemm_tile(shape.m, shape.n) {+ tiling+ if tiling.simdgroups_k() > 1+ && !shape.k.is_multiple_of(tiling.simdgroups_k() * ACTIVATION_SCALE_GROUP_SIZE) =>+ {+ GemmTiling::Tile16x32x256_Simdgroups1x1+ },+ tiling => tiling,},+ GemmEngine::Mxu if shape.is_quant() => policy::mxu_mn_tile(false, shape.m, shape.n),GemmEngine::Mxu if shape.b_transpose => policy::mxu_fp_tile(shape.m, shape.n, shape.k),GemmEngine::Mxu => policy::mxu_mn_tile(false, shape.m, shape.n),}}+#[cfg(test)]#[path = "../../../../../../unit/backends/metal/kernel/matmul/gemm/selection_test.rs"]mod tests;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.hindex abae2e1a5..e6bab1f36 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h@@ -69,7 +69,6 @@ private:bool signed_codes) {constexpr uint CODES_PER_WORD = WORD_BITS / BITS;- constexpr uint W8_SIGN_BITS = 0x80808080u;if constexpr (BITS == 4) {word0 ^= signed_codes ? W4_SIGN_MASK : 0u;METAL_PRAGMA_UNROLL@@ -77,7 +76,7 @@ private:values[i] = as_type<float>(extract_bits(word0, BITS * i, BITS) | MANTISSA_BASE_BITS) - origin;}} else {- const uint mask = signed_codes ? W8_SIGN_BITS : 0u;+ const uint mask = signed_codes ? W8_SIGN_MASK : 0u;word0 ^= mask;word1 ^= mask;METAL_PRAGMA_UNROLLdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_dispatch.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_dispatch.rsnew file mode 100644index 000000000..8e7fb7cf8--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_dispatch.rs@@ -0,0 +1,6 @@+use super::{gemm::GemmPlan, gemv::GemvSpecialization};++pub enum MatmulDispatch {+ Gemv(GemvSpecialization),+ Gemm(GemmPlan),+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rsnew file mode 100644index 000000000..e40b744f7--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs@@ -0,0 +1,274 @@+use metal::MTLGPUFamily;++use super::{+ GemmKernel,+ gemm::{self, GemmPlan, GemmProblem},+ gemv::{GemvKernel, GemvSpecialization},+ matmul_dispatch::MatmulDispatch,+ matmul_output_work::MatmulOutputWork,+ qmv::{self, QmvRoute},+};+use crate::{+ backends::{+ common::{+ BufferMut, BufferRef, CommandBufferEncoding,+ gpu_types::gemm::GemmTiling,+ kernel::{+ ActivationQuantization,+ activation_transform::ACTIVATION_SCALE_GROUP_SIZE,+ matmul::{+ ActivationFormat, Int8CodeLayout, MatmulArguments, MatmulError, MatmulKernel, MatmulShape,+ QuantParamsLayout,+ },+ },+ },+ metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},+ },+ data_type::DataType,+};++pub struct MatmulMetalKernel {+ gemv: GemvKernel,+ pub gemm: GemmKernel,+ output_work: MatmulOutputWork,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+}++pub fn supports_integer_right_operand(shape: &MatmulShape) -> bool {+ matches!((shape.b_bits, shape.signed_codes), (Some(4), _) | (Some(8), true))+}++impl MatmulMetalKernel {+ pub fn prefer_gemm_over_gemv(+ shape: MatmulShape,+ plan: GemmPlan,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+ ) -> bool {+ if shape.gathered || plan.engine != gemm::GemmEngine::Mxu {+ return false;+ }+ match (shape.m, shape.n == shape.k, (weights_data_type, input_data_type, output_data_type)) {+ (4, true, (DataType::F32, DataType::F32, DataType::F32))+ | (5, _, (DataType::BF16, DataType::BF16, DataType::BF16)) => return false,+ _ => {},+ }+ match shape.m {+ 0..=3 => return false,+ 4 => {+ let small_enough_for_mxu = shape.n <= 6144 && shape.k <= 9728;+ let k_dominates = shape.k > 3_u32.saturating_mul(shape.n);+ if !(small_enough_for_mxu || k_dominates) {+ return false;+ }+ },+ _ => {},+ }+ matches!(+ plan.tiling,+ GemmTiling::Tile16x32x256_Simdgroups1x1+ | GemmTiling::Tile16x32x256_Simdgroups1x1x4+ | GemmTiling::Tile16x128x256_Simdgroups1x4+ )+ }++ pub fn choose_dispatch(+ shape: &MatmulShape,+ device_name: &str,+ gpu_core_count: u32,+ apple_gpu_family: MTLGPUFamily,+ supports_mxu: bool,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+ ) -> MatmulDispatch {+ let all_bf16 = weights_data_type == DataType::BF16+ && input_data_type == DataType::BF16+ && output_data_type == DataType::BF16;+ let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);+ if shape.b_is_trellis {+ return MatmulDispatch::Gemm(problem.select_plan());+ }+ if let Some(route) = qmv::route(device_name, apple_gpu_family, supports_mxu, shape, all_bf16) {+ return match route {+ QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(+ GemvSpecialization::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile)+ .expect("typed QMV route must contain a legal GEMV tile"),+ ),+ QmvRoute::MainGemm(plan) => MatmulDispatch::Gemm(plan),+ };+ }+ let gemv = GemvSpecialization::select_shape(+ shape,+ weights_data_type,+ input_data_type,+ output_data_type,+ gpu_core_count,+ apple_gpu_family,+ );+ let plan = problem.select_plan();+ match gemv {+ None => MatmulDispatch::Gemm(plan),+ Some(_)+ if Self::prefer_gemm_over_gemv(*shape, plan, weights_data_type, input_data_type, output_data_type) =>+ {+ MatmulDispatch::Gemm(plan)+ },+ Some(gemv) => MatmulDispatch::Gemv(gemv),+ }+ }++ fn select_dispatch(+ &self,+ shape: &MatmulShape,+ context: &MetalContext,+ ) -> MatmulDispatch {+ Self::choose_dispatch(+ shape,+ &context.device_name,+ context.gpu_core_count,+ context.apple_gpu_family,+ context.supports_mxu,+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,+ )+ }+}++impl MatmulKernel for MatmulMetalKernel {+ type Backend = Metal;++ fn new(+ context: &MetalContext,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+ ) -> Result<Self, MetalError> {+ for data_type in [weights_data_type, input_data_type, output_data_type] {+ if !matches!(data_type, DataType::BF16 | DataType::F32) {+ return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());+ }+ }++ let output_work = MatmulOutputWork::new(context, weights_data_type, output_data_type)?;+ let gemm = GemmKernel::new(weights_data_type, input_data_type, output_data_type);+ let gemv = GemvKernel::new(weights_data_type, input_data_type, output_data_type);++ Ok(Self {+ gemv,+ gemm,+ output_work,+ weights_data_type,+ input_data_type,+ output_data_type,+ })+ }++ fn select_activation_quantization(+ &self,+ shape: &MatmulShape,+ context: &MetalContext,+ ) -> Option<ActivationQuantization> {+ let weight_group_size = shape.b_group_size?;+ let emit_group_sums = GemmKernel::requires_activation_group_sums(shape);+ let code_layout = shape.b_bits.and_then(Int8CodeLayout::for_right_bits)?;+ let quantization =+ ActivationQuantization::new(ACTIVATION_SCALE_GROUP_SIZE, weight_group_size, emit_group_sums, code_layout)?;+ if !context.supports_mxu+ || self.input_data_type != DataType::BF16+ || self.output_data_type != DataType::BF16+ || shape.a_full_precision+ || !shape.is_quant()+ || shape.params_layout != Some(QuantParamsLayout::GroupOutput)+ || !supports_integer_right_operand(shape)+ || !shape.b_transpose+ || shape.b_leading_dimension.is_some()+ || !shape.k.is_multiple_of(ACTIVATION_SCALE_GROUP_SIZE)+ || !shape.k.is_multiple_of(weight_group_size)+ {+ return None;+ }++ Some(quantization)+ }++ fn select_activation_format(+ &self,+ bf16_shape: &MatmulShape,+ context: &MetalContext,+ ) -> ActivationFormat {+ if bf16_shape.params_layout != Some(QuantParamsLayout::GroupOutput)+ || !supports_integer_right_operand(bf16_shape)+ || matches!(self.select_dispatch(bf16_shape, context), MatmulDispatch::Gemv(_))+ {+ return ActivationFormat::Bf16;+ }++ let a8_shape = MatmulShape {+ a_full_precision: false,+ ..*bf16_shape+ };+ match self.select_dispatch(&a8_shape, context) {+ MatmulDispatch::Gemm(plan) if plan.engine == gemm::GemmEngine::Mxu => ActivationFormat::Int8,+ _ => ActivationFormat::Bf16,+ }+ }++ fn encode(+ &mut self,+ arguments: MatmulArguments<+ '_,+ Metal,+ impl BufferRef<Backend = Metal>,+ impl BufferRef<Backend = Metal>,+ impl BufferMut<Backend = Metal>,+ impl BufferRef<Backend = Metal>,+ >,+ command_buffer: &mut MetalCommandBufferEncoding,+ ) -> Result<(), MetalError> {+ let shape = MatmulShape::from_arguments(&arguments);+ let plan = match self.select_dispatch(&shape, command_buffer.context()) {+ MatmulDispatch::Gemv(gemv) => {+ return self.gemv.encode(arguments, gemv, &self.output_work, command_buffer).map_err(MetalError::from);+ },+ MatmulDispatch::Gemm(plan) => plan,+ };++ // TODO: remove after GatherGEMM is supported+ if arguments.gather_indices.is_some() {+ return Err(MetalError::KernelDispatchFailed(+ format!(+ "gathered readout requires the GEMV path, but shape (m={}, n={}) routes to GEMM",+ arguments.m, arguments.n+ )+ .into(),+ ));+ }+ self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)+ }+}++#[cfg(test)]+impl MatmulMetalKernel {+ pub fn encode_with_gemm_engine(+ &mut self,+ arguments: MatmulArguments<+ '_,+ Metal,+ impl BufferRef<Backend = Metal>,+ impl BufferRef<Backend = Metal>,+ impl BufferMut<Backend = Metal>,+ impl BufferRef<Backend = Metal>,+ >,+ engine: gemm::GemmEngine,+ command_buffer: &mut MetalCommandBufferEncoding,+ ) -> Result<(), MetalError> {+ let shape = MatmulShape::from_arguments(&arguments);+ let plan = self.gemm.select_plan_for_engine(&shape, engine, command_buffer.context())?;+ self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_output_work.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_output_work.rsnew file mode 100644index 000000000..a46201fd9--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_output_work.rs@@ -0,0 +1,47 @@+use crate::{+ backends::{+ common::{Backend, BufferMut, BufferRef, kernel::ActivationTransform},+ metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},+ },+ data_type::DataType,+};++pub struct MatmulOutputWork {+ output_rht: ActivationTransform<Metal>,+ output_rht_with_bias: ActivationTransform<Metal>,+}++impl MatmulOutputWork {+ pub fn new(+ context: &MetalContext,+ weights_data_type: DataType,+ output_data_type: DataType,+ ) -> Result<Self, MetalError> {+ Ok(Self {+ output_rht: ActivationTransform::output_rht(context, output_data_type, None, true)?,+ output_rht_with_bias: ActivationTransform::output_rht(+ context,+ output_data_type,+ Some(weights_data_type),+ true,+ )?,+ })+ }++ pub fn apply(+ &self,+ output: impl BufferMut<Backend = Metal>,+ factors: impl BufferRef<Backend = Metal>,+ bias: Option<&<Metal as Backend>::GlobalBuffer>,+ m: u32,+ n: u32,+ command_buffer: &mut MetalCommandBufferEncoding,+ ) {+ let transform = if bias.is_some() {+ &self.output_rht_with_bias+ } else {+ &self.output_rht+ };+ transform.encode_fp_in_place(output, factors, bias, m, n, command_buffer);+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rsindex 4257c9dda..6121db708 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs@@ -1,308 +1,13 @@pub mod gemm;mod gemv;+mod matmul_dispatch;+mod matmul_metal_kernel;+mod matmul_output_work;mod qmv;-use metal::MTLGPUFamily;--pub use self::gemm::GemmKernel;-use self::{- gemm::{GemmPlan, GemmProblem},- gemv::{GemvKernel, GemvSpecialization},- qmv::QmvRoute,-};-use crate::{- backends::{- common::{- Backend, BufferMut, BufferRef, CommandBufferEncoding,- gpu_types::gemm::{GemmBPrologueKind, GemmTiling},- kernel::{- ActivationQuantization, ActivationTransform,- activation_transform::ACTIVATION_SCALE_GROUP_SIZE,- matmul::{- ActivationFormat, Int8CodeLayout, MatmulArguments, MatmulError, MatmulKernel, MatmulShape,- QuantParamsLayout,- },- },- },- metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},- },- data_type::DataType,-};--pub struct MatmulMetalKernel {- gemv: GemvKernel,- gemm: GemmKernel,- output_work: MatmulOutputWork,- weights_data_type: DataType,- input_data_type: DataType,- output_data_type: DataType,-}--enum MatmulDispatch {- Gemv(GemvSpecialization),- Gemm(GemmPlan),-}--pub struct MatmulOutputWork {- output_rht: ActivationTransform<Metal>,- output_rht_with_bias: ActivationTransform<Metal>,-}--impl MatmulOutputWork {- fn new(- context: &MetalContext,- weights_data_type: DataType,- output_data_type: DataType,- ) -> Result<Self, MetalError> {- Ok(Self {- output_rht: ActivationTransform::output_rht(context, output_data_type, None, true)?,- output_rht_with_bias: ActivationTransform::output_rht(- context,- output_data_type,- Some(weights_data_type),- true,- )?,- })- }-- fn apply(- &self,- output: impl BufferMut<Backend = Metal>,- factors: impl BufferRef<Backend = Metal>,- bias: Option<&<Metal as Backend>::GlobalBuffer>,- m: u32,- n: u32,- command_buffer: &mut MetalCommandBufferEncoding,- ) {- let transform = if bias.is_some() {- &self.output_rht_with_bias- } else {- &self.output_rht- };- transform.encode_fp_in_place(output, factors, bias, m, n, command_buffer);- }-}--fn supports_integer_right_operand(shape: &MatmulShape) -> bool {- matches!((shape.b_bits, shape.signed_codes), (Some(4), _) | (Some(8), true))-}--impl MatmulMetalKernel {- fn prefer_gemm_over_gemv(- shape: MatmulShape,- plan: GemmPlan,- weights_data_type: DataType,- input_data_type: DataType,- output_data_type: DataType,- ) -> bool {- if shape.gathered || plan.engine != gemm::GemmEngine::Mxu {- return false;- }- match (shape.m, shape.n == shape.k, (weights_data_type, input_data_type, output_data_type)) {- (4, true, (DataType::F32, DataType::F32, DataType::F32))- | (5, _, (DataType::BF16, DataType::BF16, DataType::BF16)) => return false,- _ => {},- }- match shape.m {- 0..=3 => return false,- 4 => {- let small_enough_for_mxu = shape.n <= 6144 && shape.k <= 9728;- let k_dominates = shape.k > 3_u32.saturating_mul(shape.n);- if !(small_enough_for_mxu || k_dominates) {- return false;- }- },- _ => {},- }- matches!(plan.tiling, GemmTiling::Tile16x32x256_Simdgroups1x1 | GemmTiling::Tile16x128x256_Simdgroups1x4)- }-- fn choose_dispatch(- shape: &MatmulShape,- device_name: &str,- gpu_core_count: u32,- apple_gpu_family: MTLGPUFamily,- supports_mxu: bool,- weights_data_type: DataType,- input_data_type: DataType,- output_data_type: DataType,- ) -> MatmulDispatch {- let all_bf16 = weights_data_type == DataType::BF16- && input_data_type == DataType::BF16- && output_data_type == DataType::BF16;- let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);- if shape.b_is_trellis {- return MatmulDispatch::Gemm(problem.select_plan());- }- if let Some(route) = qmv::route(device_name, apple_gpu_family, supports_mxu, shape, all_bf16) {- return match route {- QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(- GemvSpecialization::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile)- .expect("typed QMV route must contain a legal GEMV tile"),- ),- QmvRoute::MainGemm(plan) => MatmulDispatch::Gemm(plan),- };- }- let gemv = GemvSpecialization::select_shape(- shape,- weights_data_type,- input_data_type,- output_data_type,- gpu_core_count,- apple_gpu_family,- );- let plan = problem.select_plan();- match gemv {- None => MatmulDispatch::Gemm(plan),- Some(_)- if Self::prefer_gemm_over_gemv(*shape, plan, weights_data_type, input_data_type, output_data_type) =>- {- MatmulDispatch::Gemm(plan)- },- Some(gemv) => MatmulDispatch::Gemv(gemv),- }- }-- fn select_dispatch(- &self,- shape: &MatmulShape,- context: &MetalContext,- ) -> MatmulDispatch {- Self::choose_dispatch(- shape,- &context.device_name,- context.gpu_core_count,- context.apple_gpu_family,- context.supports_mxu,- self.weights_data_type,- self.input_data_type,- self.output_data_type,- )- }-}--impl MatmulKernel for MatmulMetalKernel {- type Backend = Metal;-- fn new(- context: &MetalContext,- weights_data_type: DataType,- input_data_type: DataType,- output_data_type: DataType,- ) -> Result<Self, MetalError> {- for data_type in [weights_data_type, input_data_type, output_data_type] {- if !matches!(data_type, DataType::BF16 | DataType::F32) {- return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());- }- }-- let output_work = MatmulOutputWork::new(context, weights_data_type, output_data_type)?;- let gemm = GemmKernel::new(weights_data_type, input_data_type, output_data_type);- let gemv = GemvKernel::new(weights_data_type, input_data_type, output_data_type);-- Ok(Self {- gemv,- gemm,- output_work,- weights_data_type,- input_data_type,- output_data_type,- })- }-- fn select_activation_quantization(- &self,- shape: &MatmulShape,- context: &MetalContext,- ) -> Option<ActivationQuantization> {- let weight_group_size = shape.b_group_size?;- let emit_group_sums = match shape.b_prologue {- GemmBPrologueKind::ScaleSymmetricDequant => false,- GemmBPrologueKind::ScaleBiasDequant | GemmBPrologueKind::ScaleZeroPointDequant => true,- GemmBPrologueKind::FullPrecision => return None,- };- let code_layout = shape.b_bits.and_then(Int8CodeLayout::for_right_bits)?;- let quantization =- ActivationQuantization::new(ACTIVATION_SCALE_GROUP_SIZE, weight_group_size, emit_group_sums, code_layout)?;- if !context.supports_mxu- || self.input_data_type != DataType::BF16- || self.output_data_type != DataType::BF16- || shape.a_full_precision- || !shape.is_quant()- || shape.params_layout != Some(QuantParamsLayout::GroupOutput)- || !supports_integer_right_operand(shape)- || !shape.b_transpose- || shape.b_leading_dimension.is_some()- || !shape.k.is_multiple_of(ACTIVATION_SCALE_GROUP_SIZE)- || !shape.k.is_multiple_of(weight_group_size)- {- return None;- }-- Some(quantization)- }-- fn select_activation_format(- &self,- bf16_shape: &MatmulShape,- context: &MetalContext,- ) -> ActivationFormat {- if bf16_shape.params_layout != Some(QuantParamsLayout::GroupOutput)- || !supports_integer_right_operand(bf16_shape)- || matches!(self.select_dispatch(bf16_shape, context), MatmulDispatch::Gemv(_))- {- return ActivationFormat::Bf16;- }-- let a8_shape = MatmulShape {- a_full_precision: false,- ..*bf16_shape- };- match self.select_dispatch(&a8_shape, context) {- MatmulDispatch::Gemm(plan) if plan.engine == gemm::GemmEngine::Mxu => ActivationFormat::Int8,- _ => ActivationFormat::Bf16,- }- }-- fn encode(- &mut self,- arguments: MatmulArguments<- '_,- Metal,- impl BufferRef<Backend = Metal>,- impl BufferRef<Backend = Metal>,- impl BufferMut<Backend = Metal>,- impl BufferRef<Backend = Metal>,- >,- command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(), MetalError> {- let shape = MatmulShape::from_arguments(&arguments);- match self.select_dispatch(&shape, command_buffer.context()) {- MatmulDispatch::Gemv(gemv) => {- self.gemv.encode(arguments, gemv, &self.output_work, command_buffer).map_err(MetalError::from)- },- MatmulDispatch::Gemm(plan) => self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer),- }- }-}-+pub use gemm::GemmKernel;#[cfg(test)]-impl MatmulMetalKernel {- pub fn encode_with_gemm_engine(- &mut self,- arguments: MatmulArguments<- '_,- Metal,- impl BufferRef<Backend = Metal>,- impl BufferRef<Backend = Metal>,- impl BufferMut<Backend = Metal>,- impl BufferRef<Backend = Metal>,- >,- engine: gemm::GemmEngine,- command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(), MetalError> {- let shape = MatmulShape::from_arguments(&arguments);- let plan = self.gemm.select_plan_for_engine(&shape, engine, command_buffer.context())?;- self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)- }-}+use matmul_dispatch::MatmulDispatch;+pub use matmul_metal_kernel::MatmulMetalKernel;+use matmul_metal_kernel::supports_integer_right_operand;+pub use matmul_output_work::MatmulOutputWork;diff --git a/crates/uzu-engine/src/backends/metal/kernel/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/mod.rsindex 9c0d00097..30c7939fb 100644--- a/crates/uzu-engine/src/backends/metal/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/mod.rs@@ -3,7 +3,7 @@ use crate::backends::{Kernels,gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE,- gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_row},+ gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_k, gemm_tiling_simdgroups_per_row},weaver::{FRONTIER_SELECT_THREADS, TOP_CHILDREN_THREADS},},},diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex 7b7ff7826..ac3233c8b 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -118,9 +118,11 @@ fn selection_fallbacks_and_split_k_are_preserved() {assert_eq!(problem(quant(shape(64, 4096, 4095)), DataType::BF16).select_plan().engine, Simdgroup);for (m, n, k, expected_tiling, expected_split_k) in [- (16, 4096, 4096, Tile16x32x256_Simdgroups1x1, 8),- (16, 1024, 4096, Tile16x32x256_Simdgroups1x1, 16),+ (16, 4096, 4096, Tile16x32x256_Simdgroups1x1x4, 4),+ (16, 1024, 4096, Tile16x32x256_Simdgroups1x1x4, 8),(16, 34816, 4096, Tile16x32x256_Simdgroups1x1, 1),+ (8, 34816, 4096, Tile16x32x256_Simdgroups1x1x4, 1),+ (8, 4096, 384, Tile16x32x256_Simdgroups1x1, 1),] {let mut a8 = quant(shape(m, n, k));a8.a_full_precision = false;