Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

speedup prefill

eugenece6298cb7be8
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs
index ebdf056d2..5800a8bee 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs
@@ -39,6 +39,7 @@ pub enum GemmTiling {
Tile64x64x32_Simdgroups2x2,
Tile32x32x32_Simdgroups2x2,
Tile16x32x256_Simdgroups1x1,
+ Tile16x32x256_Simdgroups1x1x4,
Tile16x128x256_Simdgroups1x4,
Tile32x64x256_Simdgroups2x2,
Tile64x32x256_Simdgroups4x1,
@@ -57,6 +58,7 @@ impl GemmTiling {
Self::Tile64x64x32_Simdgroups2x2 => 64,
Self::Tile32x32x32_Simdgroups2x2 => 32,
Self::Tile16x32x256_Simdgroups1x1 => 16,
+ Self::Tile16x32x256_Simdgroups1x1x4 => 16,
Self::Tile16x128x256_Simdgroups1x4 => 16,
Self::Tile32x64x256_Simdgroups2x2 => 32,
Self::Tile64x32x256_Simdgroups4x1 => 64,
@@ -73,6 +75,7 @@ impl GemmTiling {
Self::Tile64x64x32_Simdgroups2x2 => 64,
Self::Tile32x32x32_Simdgroups2x2 => 32,
Self::Tile16x32x256_Simdgroups1x1 => 32,
+ Self::Tile16x32x256_Simdgroups1x1x4 => 32,
Self::Tile16x128x256_Simdgroups1x4 => 128,
Self::Tile32x64x256_Simdgroups2x2 => 64,
Self::Tile64x32x256_Simdgroups4x1 => 32,
@@ -89,6 +92,7 @@ impl GemmTiling {
Self::Tile64x64x32_Simdgroups2x2 => 32,
Self::Tile32x32x32_Simdgroups2x2 => 32,
Self::Tile16x32x256_Simdgroups1x1 => 256,
+ Self::Tile16x32x256_Simdgroups1x1x4 => 256,
Self::Tile16x128x256_Simdgroups1x4 => 256,
Self::Tile32x64x256_Simdgroups2x2 => 256,
Self::Tile64x32x256_Simdgroups4x1 => 256,
@@ -105,6 +109,7 @@ impl GemmTiling {
Self::Tile64x64x32_Simdgroups2x2 => 2,
Self::Tile32x32x32_Simdgroups2x2 => 2,
Self::Tile16x32x256_Simdgroups1x1 => 1,
+ Self::Tile16x32x256_Simdgroups1x1x4 => 1,
Self::Tile16x128x256_Simdgroups1x4 => 1,
Self::Tile32x64x256_Simdgroups2x2 => 2,
Self::Tile64x32x256_Simdgroups4x1 => 4,
@@ -121,6 +126,7 @@ impl GemmTiling {
Self::Tile64x64x32_Simdgroups2x2 => 2,
Self::Tile32x32x32_Simdgroups2x2 => 2,
Self::Tile16x32x256_Simdgroups1x1 => 1,
+ Self::Tile16x32x256_Simdgroups1x1x4 => 1,
Self::Tile16x128x256_Simdgroups1x4 => 4,
Self::Tile32x64x256_Simdgroups2x2 => 2,
Self::Tile64x32x256_Simdgroups4x1 => 1,
@@ -129,10 +135,18 @@ impl GemmTiling {
}
}
+ pub const fn simdgroups_k(self) -> u32 {
+ match self {
+ Self::Tile16x32x256_Simdgroups1x1x4 => 4,
+ _ => 1,
+ }
+ }
+
pub const fn is_mxu_variant(self) -> bool {
matches!(
self,
Self::Tile16x32x256_Simdgroups1x1
+ | Self::Tile16x32x256_Simdgroups1x1x4
| Self::Tile16x128x256_Simdgroups1x4
| Self::Tile32x64x256_Simdgroups2x2
| Self::Tile64x32x256_Simdgroups4x1
@@ -148,6 +162,9 @@ pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {
pub const fn gemm_tiling_simdgroups_per_column(t: GemmTiling) -> u32 {
t.simdgroups_n()
}
+pub const fn gemm_tiling_simdgroups_per_k(t: GemmTiling) -> u32 {
+ t.simdgroups_k()
+}
bitflags! {
#[repr(transparent)]
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal
index 435995901..dee18b6aa 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal
@@ -109,6 +109,10 @@ KERNEL(DeltaNetChunkedOutputAndState)(
const uint token_base = chunk_idx * OUTPUT_STATE_CHUNK;
const uint valid_tokens = token_base < suffix_len ? min(uint(OUTPUT_STATE_CHUNK), suffix_len - token_base) : 0u;
const uint chunk_head_base = (chunk_idx * num_v_heads + hv_idx);
+ const bool full_chunk = valid_tokens == OUTPUT_STATE_CHUNK;
+ auto chunk_source = [&](auto source, const short rows, const short cols) {
+ return full_chunk ? source : source.bounded(rows, cols);
+ };
TokenValueFragment output_acc;
// -- R phase: R = beta (.) (V - e^g (.) (K . S^T)) into residual_new_value_tile ----------
@@ -126,10 +130,17 @@ KERNEL(DeltaNetChunkedOutputAndState)(
TokenKeyFragment key_frag;
TokenKeyFragment query_frag;
KeyValueFragment state_frag;
- key_frag.load_from(lane, fragment_source(k_tile + key_block_start, int(key_dim)).bounded(valid_token_rows, FR));
+ key_frag.load_from(
+ lane,
+ chunk_source(fragment_source(k_tile + key_block_start, int(key_dim)), valid_token_rows, FR)
+ );
query_frag.load_from(
lane,
- fragment_source(q_norm + query_tile_base + key_block_start, int(key_dim)).bounded(valid_token_rows, FR)
+ chunk_source(
+ fragment_source(q_norm + query_tile_base + key_block_start, int(key_dim)),
+ valid_token_rows,
+ FR
+ )
);
state_frag.load_from(lane, fragment_source(transposed_state_tile + key_block_start * VT, int(VT), 1));
fragment_mma(residual_acc, key_frag, state_frag);
@@ -170,8 +181,11 @@ KERNEL(DeltaNetChunkedOutputAndState)(
ScratchKeyValueFragment residual_frag;
causal_inv_frag.load_from(
lane,
- fragment_source(causal_inv_rows + source_token_block_start, int(OUTPUT_STATE_CHUNK))
- .bounded(valid_token_rows, FR)
+ chunk_source(
+ fragment_source(causal_inv_rows + source_token_block_start, int(OUTPUT_STATE_CHUNK)),
+ valid_token_rows,
+ FR
+ )
);
residual_frag.load_from(
lane,
@@ -215,8 +229,11 @@ KERNEL(DeltaNetChunkedOutputAndState)(
ScratchKeyValueFragment new_value_frag;
qk_frag.load_from(
lane,
- fragment_source(qk_scaled + qk_tile_base + source_token_block_start, int(OUTPUT_STATE_CHUNK))
- .bounded(valid_token_rows, valid_source_tokens)
+ chunk_source(
+ fragment_source(qk_scaled + qk_tile_base + source_token_block_start, int(OUTPUT_STATE_CHUNK)),
+ valid_token_rows,
+ valid_source_tokens
+ )
);
new_value_frag.load_from(
lane,
@@ -226,7 +243,13 @@ KERNEL(DeltaNetChunkedOutputAndState)(
}
const uint output_tile_base = (token_base + token_tile_base) * value_dim + hv_idx * head_v_dim + value_base;
- output_acc.store_safe(lane, out + output_tile_base, int(value_dim), short2(short(VT), short(valid_token_rows)));
+ output_acc.store_maybe_bounded(
+ lane,
+ out + output_tile_base,
+ int(value_dim),
+ full_chunk,
+ short2(short(VT), short(valid_token_rows))
+ );
}
threadgroup_barrier(mem_flags::mem_threadgroup);
@@ -247,14 +270,21 @@ KERNEL(DeltaNetChunkedOutputAndState)(
TokenKeyTileFragment key_frag;
new_value_frag.load_from(
lane,
- fragment_source(residual_new_value_tile + source_token_block_start * VT, 1, int(VT))
- .bounded(short(VT), short(valid_source_tokens))
+ chunk_source(
+ fragment_source(residual_new_value_tile + source_token_block_start * VT, 1, int(VT)),
+ short(VT),
+ short(valid_source_tokens)
+ )
);
const device float* k_tile =
k_norm + (token_base + source_token_block_start) * key_dim + hk_idx * HEAD_K_DIM + key_tile_base;
key_frag.load_from(
lane,
- fragment_source(k_tile, int(key_dim)).bounded(short(valid_source_tokens), short(OUTPUT_STATE_KEY_TILE))
+ chunk_source(
+ fragment_source(k_tile, int(key_dim)),
+ short(valid_source_tokens),
+ short(OUTPUT_STATE_KEY_TILE)
+ )
);
key_frag.map_coords(lane, [&](short row, short, float value) {
if (uint(row) >= valid_source_tokens) {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
index c47ed59de..1a7b47409 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
@@ -39,11 +39,12 @@ enum class GemmTiling : uint32_t {
Tile64x64x32_Simdgroups2x2 = 3,
Tile32x32x32_Simdgroups2x2 = 4,
Tile16x32x256_Simdgroups1x1 = 5,
- Tile16x128x256_Simdgroups1x4 = 6,
- Tile32x64x256_Simdgroups2x2 = 7,
- Tile64x32x256_Simdgroups4x1 = 8,
- Tile64x64x256_Simdgroups2x2 = 9,
- Tile128x128x256_Simdgroups4x4 = 10,
+ Tile16x32x256_Simdgroups1x1x4 = 6,
+ Tile16x128x256_Simdgroups1x4 = 7,
+ Tile32x64x256_Simdgroups2x2 = 8,
+ Tile64x32x256_Simdgroups4x1 = 9,
+ Tile64x64x256_Simdgroups2x2 = 10,
+ Tile128x128x256_Simdgroups4x4 = 11,
};
static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
index 326dc2c7d..2647fa395 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
@@ -273,7 +273,12 @@ struct Fragment {
);
}
- template <class Ptr, class FragmentRowStride = Int<1>, class FragmentColStride = Int<1>>
+ template <
+ bool ALIGNED_M = false,
+ bool ALIGNED_N = false,
+ class Ptr,
+ class FragmentRowStride = Int<1>,
+ class FragmentColStride = Int<1>>
METAL_FUNC void store_safe(
const ushort simd_lane_id,
Ptr destination,
@@ -282,7 +287,7 @@ struct Fragment {
FragmentRowStride fragment_row_stride = {},
FragmentColStride fragment_col_stride = {}
) thread {
- transfer<STORE, SAFE>(
+ transfer<STORE, SAFE, ALIGNED_M, ALIGNED_N>(
simd_lane_id,
destination,
leading_dimension,
@@ -346,6 +351,8 @@ private:
template <
bool IS_LOAD,
bool IS_SAFE,
+ bool ALIGNED_M = false,
+ bool ALIGNED_N = false,
class Ptr,
class RowStride,
class ColStride,
@@ -384,14 +391,14 @@ private:
if constexpr (IS_LOAD) {
if constexpr (IS_SAFE) {
- const bool in_bounds = (row < local_row_limit) && (col < local_col_limit);
+ const bool in_bounds = (ALIGNED_M || row < local_row_limit) && (ALIGNED_N || col < local_col_limit);
frag[element_index] = in_bounds ? static_cast<T>(ptr[offset]) : T(0);
} else {
frag[element_index] = static_cast<T>(ptr[offset]);
}
} else {
if constexpr (IS_SAFE) {
- if ((row < local_row_limit) && (col < local_col_limit)) {
+ if ((ALIGNED_M || row < local_row_limit) && (ALIGNED_N || col < local_col_limit)) {
ptr[offset] = static_cast<U>(frag[element_index]);
}
} else {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h
index b9fd94728..7c2d44dc8 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h
@@ -127,16 +127,21 @@ METAL_FUNC static void fragment_matmul(
constexpr ushort output_col_count = pair_output_rows ? 1 : cols;
constexpr ushort output_col_step = pair_output_rows ? 1 : 2;
+ constexpr bool interleave_rows = rows == 2 && cols == 2 && depth == 2 &&
+ metal::is_same_v<typename OutputFragment::ElementType, int> &&
+ metal::is_same_v<typename LeftFragment::ElementType, int8_t> &&
+ metal::is_same_v<typename RightFragment::ElementType, int8_t>;
+
METAL_PRAGMA_UNROLL
- for (ushort row = 0; row < rows; row += output_row_step) {
+ for (ushort outer_index = 0; outer_index < (interleave_rows ? depth : rows); outer_index += output_row_step) {
METAL_PRAGMA_UNROLL
for (ushort col = 0; col < output_col_count; col += output_col_step) {
METAL_PRAGMA_UNROLL
- for (ushort depth_index = 0; depth_index < depth; ++depth_index) {
+ for (ushort inner_index = 0; inner_index < (interleave_rows ? rows : depth); ++inner_index) {
matmul_paired_outputs(
- row,
+ interleave_rows ? inner_index : outer_index,
col,
- depth_index,
+ interleave_rows ? outer_index : inner_index,
uzu::integral_constant<MatmulMode, MatmulMode::multiply_accumulate>{}
);
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
index 71532ffd6..4524c4d93 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
@@ -29,6 +29,7 @@ METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint gr
UZU_CONST ushort W4_BITS = 4;
UZU_CONST uint W4_SIGN_MASK = symmetric_zero_point<W4_BITS>() * 0x11111111u;
+UZU_CONST uint W8_SIGN_MASK = symmetric_zero_point<8>() * 0x01010101u;
UZU_CONST uint W4_NIBBLE_MASK = 0x0F0F0F0Fu;
template <typename U, int N, int bits>
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
index 30e3519fc..23efd27b3 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
@@ -5,7 +5,7 @@
using namespace metal;
-template <typename T, typename BiasT>
+template <typename T, typename BiasT, bool SINGLE_SIMDGROUP = false>
static METAL_FUNC void apply_output_random_hadamard_transform(
device T* output_block,
const device int32_t* rht_factors_block,
@@ -20,7 +20,8 @@ static METAL_FUNC void apply_output_random_hadamard_transform(
const ushort stripes_per_row = tile_block_cols / HADAMARD_TRANSFORM_BLOCK_SIZE;
const ushort simd_lane = thread_context.simd_lane_id;
const uint total_work = uint(tile_block_rows) * uint(stripes_per_row);
- for (uint cell = thread_context.simdgroup_index; cell < total_work; cell += simdgroup_count) {
+ for (uint cell = SINGLE_SIMDGROUP ? 0 : thread_context.simdgroup_index; cell < total_work;
+ cell += SINGLE_SIMDGROUP ? 1 : simdgroup_count) {
const ushort row_local = ushort(cell / stripes_per_row);
const ushort stripe = ushort(cell % stripes_per_row);
const ushort col_local = stripe * ushort(HADAMARD_TRANSFORM_BLOCK_SIZE) + simd_lane;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
index 5fc88119f..77161f5ca 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
@@ -12,6 +12,7 @@ constexpr uint gemm_tiling_block_m(GemmTiling t) {
: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64
: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32
: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 16
+ : t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 16
: t == GemmTiling::Tile16x128x256_Simdgroups1x4 ? 16
: t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 32
: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 64
@@ -27,6 +28,7 @@ constexpr uint gemm_tiling_block_n(GemmTiling t) {
: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64
: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32
: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 32
+ : t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 32
: t == GemmTiling::Tile16x128x256_Simdgroups1x4 ? 128
: t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 64
: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 32
@@ -42,6 +44,7 @@ constexpr uint gemm_tiling_block_k(GemmTiling t) {
: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 32
: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32
: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 256
+ : t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 256
: t == GemmTiling::Tile16x128x256_Simdgroups1x4 ? 256
: t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 256
: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 256
@@ -57,6 +60,7 @@ constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {
: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2
: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2
: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 1
+ : t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 1
: t == GemmTiling::Tile16x128x256_Simdgroups1x4 ? 1
: t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 2
: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 4
@@ -72,6 +76,7 @@ constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {
: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2
: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2
: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 1
+ : t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 1
: t == GemmTiling::Tile16x128x256_Simdgroups1x4 ? 4
: t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 2
: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 1
@@ -80,5 +85,9 @@ constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {
: 0;
}
+constexpr uint gemm_tiling_simdgroups_per_k(GemmTiling t) {
+ return t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 4 : 1;
+}
+
} // namespace gemm
} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index beee31ab8..d89dba965 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -46,6 +46,8 @@ struct MxuMmaCore {
UZU_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
UZU_CONST ushort SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);
UZU_CONST ushort SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);
+ UZU_CONST ushort SIMDGROUPS_PER_K = gemm_tiling_simdgroups_per_k(GEMM_TILING);
+ static_assert(SIMDGROUPS_PER_K == 1 || SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN == 1, "K slices share one tile");
UZU_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
UZU_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
UZU_CONST ushort SIMDGROUP_BLOCK_K = static_cast<ushort>(MXU_SIMDGROUP_BLOCK_K);
@@ -91,21 +93,24 @@ struct MxuMmaCore {
const size_t block_row = size_t(geometry.block_row_start);
const size_t block_col = size_t(geometry.block_col_start);
- const uint k_offset = partition * params->aligned_inner_iterations * uint(THREADGROUP_BLOCK_K);
+ const ushort k_slice = ushort(thread_context.simdgroup_index) / (SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN);
+ const ushort tile_index = ushort(thread_context.simdgroup_index) % (SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN);
+ const uint k_offset =
+ (partition * SIMDGROUPS_PER_K + k_slice) * params->aligned_inner_iterations * uint(THREADGROUP_BLOCK_K);
- const ushort tile_row_offset = SIMDGROUP_BLOCK_M * (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);
- const ushort tile_col_offset = SIMDGROUP_BLOCK_N * (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
+ const ushort tile_row_offset = SIMDGROUP_BLOCK_M * (tile_index / SIMDGROUPS_PER_COLUMN);
+ const ushort tile_col_offset = SIMDGROUP_BLOCK_N * (tile_index % SIMDGROUPS_PER_COLUMN);
device OutputElementType* d_simdgroup = d + size_t(partition) * size_t(params->M) * size_t(params->N) +
block_row * params->leading_dimension_d + block_col +
tile_row_offset * params->leading_dimension_d + tile_col_offset;
const short simdgroup_limit_m =
- alignment.contains(GemmAlignment::M)
+ !Left::QUANTIZED && alignment.contains(GemmAlignment::M)
? SIMDGROUP_BLOCK_M
: short(min(int(SIMDGROUP_BLOCK_M), int(params->M) - int(geometry.block_row_start + tile_row_offset)));
const short simdgroup_limit_n =
- alignment.contains(GemmAlignment::N)
+ !Left::QUANTIZED && alignment.contains(GemmAlignment::N)
? SIMDGROUP_BLOCK_N
: short(min(int(SIMDGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start + tile_col_offset)));
@@ -127,10 +132,11 @@ struct MxuMmaCore {
const device RightElementType* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);
dispatch_bool(
- alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),
+ (alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M)) && simdgroup_limit_m > 0,
[&](auto aligned_m) {
dispatch_bool(
- alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),
+ (alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N)) &&
+ simdgroup_limit_n > 0,
[&](auto aligned_n) {
AccumFragment accumulator_tile =
Schedule::template launch<MxuMmaCore, aligned_m.value, aligned_n.value>(
@@ -143,6 +149,29 @@ struct MxuMmaCore {
thread_context
);
+ if constexpr (SIMDGROUPS_PER_K > 1) {
+ threadgroup float* k_partials = reinterpret_cast<threadgroup float*>(b_shared);
+ thread AccumulatorType* values = accumulator_tile.elements();
+ const ushort lane = ushort(thread_context.simd_lane_id);
+ if (k_slice != 0) {
+ for (ushort element = 0; element < AccumFragment::ELEMENTS_PER_FRAGMENT; ++element) {
+ k_partials
+ [((k_slice - 1) * AccumFragment::ELEMENTS_PER_FRAGMENT + element) * METAL_SIMD_SIZE + lane] =
+ values[element];
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (k_slice != 0) {
+ return;
+ }
+ for (ushort slice = 1; slice < SIMDGROUPS_PER_K; ++slice) {
+ for (ushort element = 0; element < AccumFragment::ELEMENTS_PER_FRAGMENT; ++element) {
+ values[element] += k_partials
+ [((slice - 1) * AccumFragment::ELEMENTS_PER_FRAGMENT + element) * METAL_SIMD_SIZE + lane];
+ }
+ }
+ }
+
if (apply_scale) {
const AccumulatorType scale = AccumulatorType(params->ab_scale);
accumulator_tile.map([&](auto value) { return value * scale; });
@@ -176,7 +205,7 @@ struct MxuMmaCore {
if constexpr (aligned_m.value && aligned_n.value) {
accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));
} else {
- accumulator_tile.store_safe(
+ accumulator_tile.template store_safe<aligned_m.value, aligned_n.value>(
thread_context.simd_lane_id,
d_simdgroup,
int(params->leading_dimension_d),
@@ -188,7 +217,34 @@ struct MxuMmaCore {
}
);
+ if (k_slice != 0) {
+ return;
+ }
+
if (output_transform.contains(GemmDTransform::RHT)) {
+ if constexpr (
+ schedules::IntegerSchedule<Left, Right>::CENTER_RIGHT_ZERO_POINTS &&
+ SIMDGROUP_BLOCK_N == HADAMARD_TRANSFORM_BLOCK_SIZE
+ ) {
+ if (alignment.contains(GemmAlignment::M) && alignment.contains(GemmAlignment::N)) {
+ if (simdgroup_limit_m <= 0 || simdgroup_limit_n <= 0) {
+ return;
+ }
+ simdgroup_barrier(mem_flags::mem_device);
+ apply_output_random_hadamard_transform<OutputElementType, RightElementType, true>(
+ d_simdgroup,
+ rht_factors + block_col + tile_col_offset,
+ bias_simdgroup,
+ apply_bias,
+ SIMDGROUP_BLOCK_M,
+ SIMDGROUP_BLOCK_N,
+ params->leading_dimension_d,
+ 1,
+ thread_context
+ );
+ return;
+ }
+ }
// Metal rejects a threadgroup barrier when the threadgroup contains one simdgroup.
if constexpr (SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN == 1) {
simdgroup_barrier(mem_flags::mem_device);
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
index 0197e014d..e2ba61926 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
@@ -42,7 +42,7 @@ struct RightOperand {
UZU_CONST ushort BITS = QUANTIZED ? BITS_ : 0;
UZU_CONST ushort GROUP_SIZE = QUANTIZED ? GROUP_SIZE_ : 0;
UZU_CONST GemmBPrologueKind SCHEME = PROLOGUE;
- UZU_CONST bool NEEDS_CORRECTION = QUANTIZED && PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant;
+ UZU_CONST bool NEEDS_CORRECTION = PROLOGUE == GemmBPrologueKind::ScaleBiasDequant;
UZU_CONST ushort CODE_ORIGIN = (!QUANTIZED || (BITS_ == 4 && PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant))
? 0
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
index 4fe454596..a9f0ff38d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
@@ -54,58 +54,62 @@ load_int8_tile(const device int8_t* src, const int row_stride, const short simdg
}
});
} else {
- auto source = uzu::matmul::fragment_source(src, row_stride);
- if constexpr (!ALIGNED) {
- source = source.bounded(simdgroup_limit, Fragment::COL_FRAGMENTS * Fragment::FragmentOpsType::FRAGMENT_ROWS);
- }
- tile.load_from(simd_lane_id, source);
+ using Ops = typename Fragment::FragmentOpsType;
+ static_assert(
+ Ops::THREAD_ELEMENT_COLS == 4 && sizeof(packed_char4) == 4 && alignof(packed_char4) == 1,
+ "INT8 fragments require four-byte, byte-aligned loads"
+ );
+ const short2 position = Ops::get_position(simd_lane_id);
+ const device int8_t* base = src + int(position.y) * row_stride + int(position.x);
+ const short row_limit = simdgroup_limit - position.y;
+ for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {
+ METAL_PRAGMA_UNROLL
+ for (ushort fragment_column = 0; fragment_column < Fragment::COL_FRAGMENTS; ++fragment_column) {
+ char4 codes(0);
+ if (ALIGNED || row_offset < row_limit) {
+ codes = *reinterpret_cast<const device packed_char4*>(
+ base + int(row_offset) * row_stride + int(fragment_column) * Ops::FRAGMENT_COLS
+ );
+ }
+ reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, fragment_column))[row_slot] =
+ as_type<uint>(codes);
+ }
+ });
}
return tile;
}
-template <typename Fragment, bool ALIGNED, bool CODES_GROUPED_BY_NIBBLE, bool HOISTED>
+template <typename Fragment, bool ALIGNED, bool CODES_GROUPED_BY_NIBBLE>
struct Int8Cursor {
using Ops = typename Fragment::FragmentOpsType;
UZU_CONST short BLOCK_K = short(Fragment::COL_FRAGMENTS * Ops::FRAGMENT_ROWS);
struct PackedChunk {};
- const device int8_t* origin;
const device int8_t* address;
int row_stride;
short simdgroup_limit;
ushort simd_lane_id;
- METAL_FUNC Fragment load(const uint chunk_index) const thread {
- const device int8_t* source = address;
- if constexpr (!HOISTED) {
- source += chunk_index * uint(BLOCK_K);
- }
+ METAL_FUNC Fragment load(const uint) const thread {
return load_int8_tile<Fragment, ALIGNED, CODES_GROUPED_BY_NIBBLE>(
- source,
+ address,
row_stride,
simdgroup_limit,
simd_lane_id
);
}
- METAL_FUNC void advance() thread {
- if constexpr (HOISTED) {
- address += BLOCK_K;
- }
- }
+ METAL_FUNC void advance() thread { address += BLOCK_K; }
- METAL_FUNC void begin_k_group(const uint k_offset) thread {
- if constexpr (!HOISTED) {
- address = origin + k_offset;
- }
- }
+ METAL_FUNC void begin_k_group(const uint) thread {}
};
-template <typename Fragment, bool ALIGNED, ushort CODE_ORIGIN>
+template <typename Fragment, typename Operand, bool ALIGNED>
struct W4Cursor {
using Ops = typename Fragment::FragmentOpsType;
UZU_CONST short BLOCK_K = short(Fragment::COL_FRAGMENTS * Ops::FRAGMENT_ROWS);
+ UZU_CONST bool HAS_ZERO_POINTS = Operand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant;
UZU_CONST uint CHUNK_BYTES = uint(BLOCK_K) * uint(get_bytes_per_pack<W4_BITS>()) / uint(get_pack_factor<W4_BITS>());
@@ -123,10 +127,12 @@ struct W4Cursor {
short tile_row_limit;
short2 position;
bool signed_codes;
+ const device uint8_t* zero_points;
+ uint zero_point_group_stride_bytes;
METAL_FUNC PackedChunk fetch(const uint chunk_index) const thread {
// Padding decodes to zero after the optional sign flip and code-origin adjustment.
- const uint padding_word = (CODE_ORIGIN == 0) == signed_codes ? W4_SIGN_MASK : 0u;
+ const uint padding_word = (Operand::CODE_ORIGIN == 0) == signed_codes ? W4_SIGN_MASK : 0u;
const device uint8_t* base =
current + chunk_index * CHUNK_BYTES + int(position.y) * row_stride_bytes + int(position.x);
const short row_limit = tile_row_limit - position.y;
@@ -143,14 +149,25 @@ struct W4Cursor {
METAL_FUNC Fragment decode(const thread PackedChunk& packed_chunk) const thread {
Fragment tile;
- for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short) {
+ for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {
const uint word =
packed_chunk.words[PackedChunk::word_index(fragment_row, row_slot)] ^ (signed_codes ? W4_SIGN_MASK : 0u);
uint low = word & W4_NIBBLE_MASK;
uint high = (word >> W4_BITS) & W4_NIBBLE_MASK;
- if constexpr (CODE_ORIGIN != 0) {
- low = as_type<uint>(as_type<char4>(low) - char4(char(CODE_ORIGIN)));
- high = as_type<uint>(as_type<char4>(high) - char4(char(CODE_ORIGIN)));
+ if constexpr (HAS_ZERO_POINTS) {
+ const uint column = uint(position.y + row_offset);
+ uint zero_point = 0;
+ if (ALIGNED || short(column) < tile_row_limit) {
+ // Hoisting these loads introduces register spills in the M5 fused-RHT specialization.
+ const device volatile uint8_t* group = zero_points;
+ zero_point = decode_zero_point<W4_BITS>(group[column / get_pack_factor<W4_BITS>()], column);
+ }
+ const uint origin_offset = (symmetric_zero_point<8>() - zero_point) * 0x01010101u;
+ low = (low + origin_offset) ^ W8_SIGN_MASK;
+ high = (high + origin_offset) ^ W8_SIGN_MASK;
+ } else if constexpr (Operand::CODE_ORIGIN != 0) {
+ low = as_type<uint>(as_type<char4>(low) - char4(char(Operand::CODE_ORIGIN)));
+ high = as_type<uint>(as_type<char4>(high) - char4(char(Operand::CODE_ORIGIN)));
}
reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, 0))[row_slot] = low;
reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, 1))[row_slot] = high;
@@ -162,10 +179,16 @@ struct W4Cursor {
METAL_FUNC void advance() thread { current += CHUNK_BYTES; }
- METAL_FUNC void begin_k_group(const uint) thread {}
+ METAL_FUNC void begin_k_group(const uint k_offset) thread {
+ if constexpr (HAS_ZERO_POINTS) {
+ if (k_offset != 0) {
+ zero_points += zero_point_group_stride_bytes;
+ }
+ }
+ }
};
-template <bool HOIST_OPERAND_ADDRESSING, typename Core, typename LeftOperand, bool ALIGNED>
+template <typename Core, typename LeftOperand, bool ALIGNED>
static METAL_FUNC auto make_left_cursor(
const typename Core::LeftStorage source,
const constant uzu::matmul::GemmParams* params,
@@ -174,8 +197,7 @@ static METAL_FUNC auto make_left_cursor(
) {
using Fragment = uzu::matmul::Fragment<int8_t, Core::TILES_M, Core::TILES_K, typename Core::FragmentOps>;
const device int8_t* origin = source.codes + size_t(tile.abs_row_base) * params->leading_dimension_a + tile.k_offset;
- return Int8Cursor<Fragment, ALIGNED, LeftOperand::GROUPED_BY_NIBBLE, HOIST_OPERAND_ADDRESSING>{
- origin,
+ return Int8Cursor<Fragment, ALIGNED, LeftOperand::GROUPED_BY_NIBBLE>{
origin,
int(params->leading_dimension_a),
tile.simdgroup_limit_m,
@@ -183,7 +205,7 @@ static METAL_FUNC auto make_left_cursor(
};
}
-template <bool HOIST_OPERAND_ADDRESSING, typename Core, typename Operand, bool ALIGNED>
+template <typename Core, typename Operand, bool ALIGNED>
static METAL_FUNC auto make_right_cursor(
const typename Core::RightStorage source,
const constant uzu::matmul::GemmParams* params,
@@ -200,18 +222,26 @@ static METAL_FUNC auto make_right_cursor(
if constexpr (Operand::BITS == 4) {
static_assert(Core::TILES_K == get_pack_factor<Operand::BITS>(), "W4 requires two K fragments");
- return W4Cursor<Fragment, ALIGNED, Operand::CODE_ORIGIN>{
+ using Cursor = W4Cursor<Fragment, Operand, ALIGNED>;
+ const device uint8_t* zero_points = nullptr;
+ const uint zero_point_group_stride_bytes = params->zero_point_group_stride / get_pack_factor<W4_BITS>();
+ if constexpr (Cursor::HAS_ZERO_POINTS) {
+ zero_points = source.zp() + (tile.k_offset / Operand::GROUP_SIZE) * zero_point_group_stride_bytes +
+ tile.absolute_column_base() / get_pack_factor<W4_BITS>();
+ }
+ return Cursor{
current,
row_stride_bytes,
tile.simdgroup_limit_n,
Ops::get_position(thread_context.simd_lane_id),
- source.signed_codes
+ source.signed_codes,
+ zero_points,
+ zero_point_group_stride_bytes
};
} else {
static_assert(Operand::BITS == 8, "integer tile cursors support 4-bit and 8-bit codes");
const device int8_t* origin_int8 = reinterpret_cast<const device int8_t*>(current);
- return Int8Cursor<Fragment, ALIGNED, false, HOIST_OPERAND_ADDRESSING>{
- origin_int8,
+ return Int8Cursor<Fragment, ALIGNED, false>{
origin_int8,
row_stride_bytes,
tile.simdgroup_limit_n,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
index de9f50c10..044de6e91 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
@@ -55,13 +55,14 @@ struct MetadataContext {
uint left_group_count;
uint right_group_count;
uint right_scale_group_stride;
- uint right_zero_point_group_stride;
};
template <typename LeftOperand, typename RightOperand>
struct IntegerSchedule {
UZU_CONST uint RIGHT_GROUP_SIZE = uint(RightOperand::GROUP_SIZE);
- UZU_CONST bool HAS_ZERO_POINTS = RightOperand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant;
+ UZU_CONST bool CENTER_RIGHT_ZERO_POINTS =
+ LeftOperand::QUANTIZED && RightOperand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant;
+ static_assert(!CENTER_RIGHT_ZERO_POINTS || RightOperand::BITS == 4, "zero points are 4-bit");
UZU_CONST bool HAS_BIAS = RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant;
UZU_CONST uchar RIGHT_CODE_OFFSET = uchar(RightOperand::CODE_ORIGIN);
@@ -72,7 +73,7 @@ struct IntegerSchedule {
UZU_CONST ushort LEFT_VALUE_COUNT = Core::TILES_M * THREAD_ROWS_PER_FRAGMENT;
float scales[LEFT_VALUE_COUNT];
- metal::conditional_t<HAS_ZERO_POINTS, int, float> group_corrections[LEFT_VALUE_COUNT];
+ float group_corrections[LEFT_VALUE_COUNT];
struct LeftSlot {
uint row_index;
@@ -105,7 +106,7 @@ struct IntegerSchedule {
});
}
- if constexpr (HAS_ZERO_POINTS || HAS_BIAS) {
+ if constexpr (HAS_BIAS) {
const uint right_group_index = k_offset / RIGHT_GROUP_SIZE;
for_each_static_index<int(LEFT_VALUE_COUNT)>([&](const ushort index) {
const LeftSlot slot = left_slot(metadata_context, index);
@@ -113,11 +114,7 @@ struct IntegerSchedule {
slot.live
? left.correction_sums()[slot.row_index * metadata_context.right_group_count + right_group_index]
: 0;
- if constexpr (HAS_ZERO_POINTS) {
- group_corrections[index] = code_sum;
- } else {
- group_corrections[index] = slot.live ? (scales[index] * float(code_sum)) : 0.0f;
- }
+ group_corrections[index] = slot.live ? (scales[index] * float(code_sum)) : 0.0f;
});
}
}
@@ -128,7 +125,6 @@ struct IntegerSchedule {
using Ops = typename Core::FragmentOps;
using ScaleElement = typename RightOperand::ScaleElement;
using ScaleVector = vec<ScaleElement, Ops::THREAD_ELEMENT_COLS>;
- using ZeroPointVector = uchar4;
static_assert(
Ops::THREAD_ELEMENT_COLS == uzu::matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT,
"group-major metadata must use the metadata load width"
@@ -136,7 +132,6 @@ struct IntegerSchedule {
ScaleVector scales[Core::TILES_N];
ScaleVector bias_offsets[Core::TILES_N];
- ZeroPointVector zero_points[Core::TILES_N];
METAL_FUNC void load(
const typename Core::RightStorage right,
@@ -159,25 +154,6 @@ struct IntegerSchedule {
);
bias_offsets[tile_n] += scales[tile_n] * ScaleElement(RIGHT_CODE_OFFSET);
}
- if constexpr (HAS_ZERO_POINTS) {
- constexpr uint ZERO_POINT_PACK_FACTOR = 8u / uint(RightOperand::BITS);
- const device uint8_t* zero_point_row =
- right.zp() +
- right_group_index * (metadata_context.right_zero_point_group_stride / ZERO_POINT_PACK_FACTOR);
- uint right_zero_point_column_start = metadata_context.right_column_base + uint(right_column_offset);
- if constexpr (!ALIGNED_N) {
- right_zero_point_column_start = clamp_group_output_column(
- right_zero_point_column_start,
- metadata_context.right_zero_point_group_stride
- );
- }
- static_assert(RightOperand::BITS == 4, "zero points are 4-bit");
- const ushort packed =
- *reinterpret_cast<const device ushort*>(zero_point_row + (right_zero_point_column_start >> 1));
- uint spread = (uint(packed) | (uint(packed) << 8)) & 0x00FF00FFu;
- spread = (spread | (spread << 4)) & 0x0F0F0F0Fu;
- zero_points[tile_n] = as_type<ZeroPointVector>(spread);
- }
if constexpr (!ALIGNED_N) {
const bool4 live =
(short4(right_column_offset) + short4(0, 1, 2, 3)) < short4(metadata_context.right_column_limit);
@@ -185,9 +161,6 @@ struct IntegerSchedule {
if constexpr (HAS_BIAS) {
bias_offsets[tile_n] = select(ScaleVector(0), bias_offsets[tile_n], live);
}
- if constexpr (HAS_ZERO_POINTS) {
- zero_points[tile_n] = select(ZeroPointVector(RIGHT_CODE_OFFSET), zero_points[tile_n], live);
- }
}
});
}
@@ -203,7 +176,14 @@ struct IntegerSchedule {
template <typename Core>
static constexpr bool prefetches_int4_chunks() {
- return Core::TILES_M == 1 && chunks_per_k_group<Core>() > 1 && RightOperand::BITS == 4;
+ return (Core::TILES_M == 1 || (Core::TILES_M == 2 && Core::TILES_N == 2 && CENTER_RIGHT_ZERO_POINTS)) &&
+ chunks_per_k_group<Core>() > 1 && RightOperand::BITS == 4;
+ }
+
+ template <typename Core, bool ALIGNED_M, bool ALIGNED_N>
+ static constexpr bool multiplies_device_groups() {
+ return RightOperand::BITS == 8 && RIGHT_GROUP_SIZE == 64 && Core::TILES_M == 2 && Core::TILES_N == 2 && ALIGNED_M &&
+ ALIGNED_N;
}
template <typename RightCodes, int K_GROUP_COUNT, int CHUNKS_PER_K_GROUP>
@@ -218,6 +198,50 @@ struct IntegerSchedule {
});
}
+ template <typename Core>
+ static METAL_FUNC GroupProducts<Core> multiply_device_group(
+ const typename Core::LeftStorage left,
+ const typename Core::RightStorage right,
+ const constant uzu::matmul::GemmParams* params,
+ const TileContext tile,
+ const uint k_offset
+ ) {
+ using Ops = typename Core::FragmentOps;
+ constexpr auto descriptor = mpp::tensor_ops::matmul2d_descriptor(
+ Core::SIMDGROUP_BLOCK_M,
+ Core::SIMDGROUP_BLOCK_N,
+ RIGHT_GROUP_SIZE,
+ false,
+ true,
+ true,
+ mpp::tensor_ops::matmul2d_descriptor::mode::multiply
+ );
+ mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> op;
+ const device int8_t* right_origin = reinterpret_cast<const device int8_t*>(right.codes) +
+ size_t(tile.absolute_column_base()) * params->K + k_offset;
+ auto right_tensor = tensor(
+ const_cast<device int8_t*>(right_origin),
+ extents<int, RIGHT_GROUP_SIZE, Core::SIMDGROUP_BLOCK_N>{},
+ array<int, 2>{1, int(params->K)}
+ );
+ auto cooperative_right = op.template get_right_input_cooperative_tensor<int8_t, int8_t, int>();
+ cooperative_right.load(right_tensor);
+ const device int8_t* left_origin = left.codes + size_t(tile.abs_row_base) * params->leading_dimension_a + k_offset;
+ auto left_tensor = tensor(
+ const_cast<device int8_t*>(left_origin),
+ extents<int, RIGHT_GROUP_SIZE, Core::SIMDGROUP_BLOCK_M>{},
+ array<int, 2>{1, int(params->leading_dimension_a)}
+ );
+ auto destination =
+ op.template get_destination_cooperative_tensor<decltype(left_tensor), decltype(cooperative_right), int>();
+ op.run(left_tensor, cooperative_right, destination);
+ GroupProducts<Core> product;
+ for_each_static_index<int(Core::TILES_M) * int(Core::TILES_N) * int(Ops::ELEMENTS_PER_THREAD)>(
+ [&](const ushort element) { product.elements()[element] = destination[element]; }
+ );
+ return product;
+ }
+
template <typename Core, typename LeftCodes, typename RightCodes, int CHUNKS_PER_K_GROUP>
static METAL_FUNC GroupProducts<Core> multiply_k_group(
thread LeftCodes& left_codes,
@@ -235,14 +259,16 @@ struct IntegerSchedule {
right_codes.advance();
});
} else {
- METAL_PRAGMA_NO_UNROLL
- for (int chunk = 0; chunk < CHUNKS_PER_K_GROUP; ++chunk) {
+ decltype(right_codes.load(0u)) right_tiles[CHUNKS_PER_K_GROUP];
+ for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
+ right_tiles[chunk] = right_codes.load(uint(chunk));
+ right_codes.advance();
+ });
+ for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
auto left_tile = left_codes.load(uint(chunk));
- auto right_tile = right_codes.load(uint(chunk));
- uzu::matmul::fragment_mma(group_product, left_tile, right_tile);
+ uzu::matmul::fragment_mma(group_product, left_tile, right_tiles[chunk]);
left_codes.advance();
- right_codes.advance();
- }
+ });
}
return group_product;
}
@@ -265,13 +291,10 @@ struct IntegerSchedule {
const ushort right_column = element % Ops::THREAD_ELEMENT_COLS;
const ushort left_row = tile_m * Ops::THREAD_ELEMENT_ROWS + element / Ops::THREAD_ELEMENT_COLS;
const float right_scale = float(right_metadata.scales[tile_n][right_column]);
- int centered_product = group_product_value[element];
- if constexpr (HAS_ZERO_POINTS) {
- centered_product += (int(RIGHT_CODE_OFFSET) - int(right_metadata.zero_points[tile_n][right_column])) *
- left_metadata.group_corrections[left_row];
- }
accumulated[element] =
- fma(left_metadata.scales[left_row] * right_scale, float(centered_product), accumulated[element]);
+ fma(left_metadata.scales[left_row] * right_scale,
+ float(group_product_value[element]),
+ accumulated[element]);
if constexpr (HAS_BIAS) {
accumulated[element] =
fma(right_metadata.bias_offsets[tile_n][right_column],
@@ -300,23 +323,18 @@ struct IntegerSchedule {
LeftOperand::GROUP_SIZE % RightOperand::GROUP_SIZE == 0,
"the left group must hold a whole number of right groups"
);
+ if constexpr (!ALIGNED_M || !ALIGNED_N) {
+ if (tile.simdgroup_limit_m <= 0 || tile.simdgroup_limit_n <= 0) {
+ typename Core::AccumFragment empty;
+ empty.clear();
+ return empty;
+ }
+ }
- constexpr bool HOIST_OPERAND_ADDRESSING =
- !(RightOperand::SCHEME == GemmBPrologueKind::ScaleSymmetricDequant &&
- Core::TILING == GemmTiling::Tile128x128x256_Simdgroups4x4);
-
- auto left_codes = quantized::make_left_cursor<HOIST_OPERAND_ADDRESSING, Core, LeftOperand, ALIGNED_M>(
- left_storage,
- params,
- tile,
- thread_context
- );
- auto right_codes = quantized::make_right_cursor<HOIST_OPERAND_ADDRESSING, Core, RightOperand, ALIGNED_N>(
- right_storage,
- params,
- tile,
- thread_context
- );
+ auto left_codes =
+ quantized::make_left_cursor<Core, LeftOperand, ALIGNED_M>(left_storage, params, tile, thread_context);
+ auto right_codes =
+ quantized::make_right_cursor<Core, RightOperand, ALIGNED_N>(right_storage, params, tile, thread_context);
const short2 position = Core::FragmentOps::get_position(thread_context.simd_lane_id);
const MetadataContext metadata_context = {
@@ -327,7 +345,6 @@ struct IntegerSchedule {
uint(params->K) / uint(LeftOperand::GROUP_SIZE),
uint(params->K) / RIGHT_GROUP_SIZE,
params->scale_group_stride,
- params->zero_point_group_stride,
};
const uint first_right_group = tile.k_offset / RIGHT_GROUP_SIZE;
@@ -340,11 +357,36 @@ struct IntegerSchedule {
constexpr int CHUNKS_PER_K_GROUP = chunks_per_k_group<Core>();
constexpr bool PREFETCH_INT4_CHUNKS = prefetches_int4_chunks<Core>();
constexpr bool IS_ALIGNED_M_INT4_TILE16X32 =
- PREFETCH_INT4_CHUNKS && ALIGNED_M && Core::TILING == GemmTiling::Tile16x32x256_Simdgroups1x1;
+ PREFETCH_INT4_CHUNKS && ALIGNED_M && Core::THREADGROUP_BLOCK_M == 16 && Core::THREADGROUP_BLOCK_N == 32;
constexpr int K_GROUPS_PER_FETCH = IS_ALIGNED_M_INT4_TILE16X32 ? 4 : 1;
constexpr int MIN_K_GROUPS_PER_THREADGROUP_FOR_BATCHED_FETCH = 16;
using PackedRightChunk = typename decltype(right_codes)::PackedChunk;
+ auto multiply_group = [&](const int right_group_index,
+ const thread PackedRightChunk(&packed_right_chunks)[CHUNKS_PER_K_GROUP]) {
+ const uint right_group_offset = uint(right_group_index) * RIGHT_GROUP_SIZE;
+ left_codes.begin_k_group(right_group_offset);
+ right_codes.begin_k_group(right_group_offset);
+ auto load_metadata = [&]() {
+ left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);
+ right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));
+ };
+ if constexpr (!PREFETCH_INT4_CHUNKS) {
+ load_metadata();
+ }
+ GroupProducts<Core> group_product;
+ if constexpr (multiplies_device_groups<Core, ALIGNED_M, ALIGNED_N>()) {
+ group_product =
+ multiply_device_group<Core>(left_storage, right_storage, params, tile, tile.k_offset + right_group_offset);
+ } else {
+ group_product = multiply_k_group<Core>(left_codes, right_codes, packed_right_chunks);
+ }
+ if constexpr (PREFETCH_INT4_CHUNKS) {
+ load_metadata();
+ }
+ accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);
+ };
+
int right_group_index = 0;
if constexpr (K_GROUPS_PER_FETCH > 1) {
const int batched_fetch_right_group_limit =
@@ -355,14 +397,7 @@ struct IntegerSchedule {
PackedRightChunk packed_right_chunks[K_GROUPS_PER_FETCH][CHUNKS_PER_K_GROUP];
fetch_k_groups(right_codes, packed_right_chunks);
for_each_static_index<K_GROUPS_PER_FETCH>([&](const ushort k_group) {
- const int fetched_right_group_index = right_group_index + int(k_group);
- const uint right_group_offset = uint(fetched_right_group_index) * RIGHT_GROUP_SIZE;
- left_codes.begin_k_group(right_group_offset);
- right_codes.begin_k_group(right_group_offset);
- auto group_product = multiply_k_group<Core>(left_codes, right_codes, packed_right_chunks[k_group]);
- left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);
- right_metadata.load(right_storage, metadata_context, first_right_group + uint(fetched_right_group_index));
- accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);
+ multiply_group(right_group_index + int(k_group), packed_right_chunks[k_group]);
});
}
}
@@ -372,13 +407,7 @@ struct IntegerSchedule {
if constexpr (PREFETCH_INT4_CHUNKS) {
fetch_k_groups(right_codes, packed_right_chunks);
}
- const uint right_group_offset = uint(right_group_index) * RIGHT_GROUP_SIZE;
- left_codes.begin_k_group(right_group_offset);
- right_codes.begin_k_group(right_group_offset);
- auto group_product = multiply_k_group<Core>(left_codes, right_codes, packed_right_chunks[0]);
- left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);
- right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));
- accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);
+ multiply_group(right_group_index, packed_right_chunks[0]);
}
return accumulator;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
index db6f59292..d1d234faa 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -11,14 +11,20 @@ using namespace metal;
using namespace uzu::gemm;
#define A_IS_INT8 (A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)
-#define NEEDS_ASYMMETRIC_WEIGHT_CORRECTION (A_IS_INT8 && B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant)
+#define NEEDS_ASYMMETRIC_WEIGHT_CORRECTION (A_IS_INT8 && B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant)
#define GEMM_MXU_QUANT (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision && !A_IS_INT8)
#define GEMM_TGA_ELEMENTS \
((USE_MXU) ? 1 : (gemm_tiling_block_m(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))
+#define GEMM_K_PARTIAL_ELEMENTS \
+ (gemm_tiling_simdgroups_per_k(GEMM_TILING) == 1 \
+ ? 1 \
+ : (gemm_tiling_simdgroups_per_k(GEMM_TILING) - 1) * gemm_tiling_block_m(GEMM_TILING) * \
+ gemm_tiling_block_n(GEMM_TILING) * uint(sizeof(float) / sizeof(BT)))
// The A8 MXU route reads its scales into registers, so it asks for no
// threadgroup B memory.
#define GEMM_TGB_ELEMENTS \
- ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) : 1) \
+ ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) \
+ : GEMM_K_PARTIAL_ELEMENTS) \
: (gemm_tiling_block_n(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))
template <
@@ -45,6 +51,7 @@ VARIANTS(
GemmTiling::Tile64x64x32_Simdgroups2x2,
GemmTiling::Tile32x32x32_Simdgroups2x2,
GemmTiling::Tile16x32x256_Simdgroups1x1,
+ GemmTiling::Tile16x32x256_Simdgroups1x1x4,
GemmTiling::Tile16x128x256_Simdgroups1x4,
GemmTiling::Tile32x64x256_Simdgroups2x2,
GemmTiling::Tile64x32x256_Simdgroups4x1,
@@ -68,6 +75,7 @@ VARIANTS(A_GROUP_SIZE, 0, 128)
CONSTRAINT(
USE_MXU ==
(GEMM_TILING == GemmTiling::Tile16x32x256_Simdgroups1x1 ||
+ GEMM_TILING == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ||
GEMM_TILING == GemmTiling::Tile16x128x256_Simdgroups1x4 ||
GEMM_TILING == GemmTiling::Tile32x64x256_Simdgroups2x2 ||
GEMM_TILING == GemmTiling::Tile64x32x256_Simdgroups4x1 ||
@@ -93,6 +101,9 @@ CONSTRAINT(
(TRANSPOSE_B &&
(B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)))
+CONSTRAINT(
+ GEMM_TILING != GemmTiling::Tile16x32x256_Simdgroups1x1x4 ||
+ A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)
CONSTRAINT(
GEMM_TILING != GemmTiling::Tile16x128x256_Simdgroups1x4 ||
(TRANSPOSE_B && B_PROLOGUE == GemmBPrologueKind::FullPrecision))
@@ -100,7 +111,7 @@ CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || USE_MXU)
CONSTRAINT(
A_PROLOGUE == GemmAPrologueKind::FullPrecision ||
(TRANSPOSE_B && B_PROLOGUE != GemmBPrologueKind::FullPrecision))
-CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || (AT == "bfloat" && DT == "bfloat"))
+CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || AT == "bfloat")
CONSTRAINT((A_PROLOGUE == GemmAPrologueKind::FullPrecision) == (A_GROUP_SIZE == 0))
KERNEL(Gemm)(
const device AT* a OPTIONAL(A_PROLOGUE == GemmAPrologueKind::FullPrecision),
@@ -133,7 +144,7 @@ KERNEL(Gemm)(
const uint group_z GROUPS(group_count_z),
const uint thread_x THREADS(METAL_SIMD_SIZE),
const uint thread_y THREADS(gemm_tiling_simdgroups_per_column(GEMM_TILING)),
- const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING)),
+ const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING) * gemm_tiling_simdgroups_per_k(GEMM_TILING)),
const ThreadContext thread_context
) {
(void)group_x;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
index 7faab6ed8..553259e9d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
@@ -5,10 +5,12 @@
using namespace metal;
using namespace uzu::gemm;
-template <typename T>
+template <typename Input, typename T>
+VARIANTS(Input, float, bfloat)
VARIANTS(T, float, bfloat)
+CONSTRAINT(Input == "float" || Input == T)
KERNEL(GemmSplitKReduce)(
- const device T* partial_sums,
+ const device Input* partial_sums,
device T* output,
const device T* output_bias
OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),
@@ -36,7 +38,7 @@ KERNEL(GemmSplitKReduce)(
}
device vec<T, 4>* output_vectors = reinterpret_cast<device vec<T, 4>*>(output);
- const device vec<T, 4>* partial_sum_vectors = reinterpret_cast<const device vec<T, 4>*>(partial_sums);
+ const device vec<Input, 4>* partial_sum_vectors = reinterpret_cast<const device vec<Input, 4>*>(partial_sums);
float4 accumulator = float4(0.0f);
for (uint partition = 0u; partition < partition_count; ++partition) {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
index 137171472..5e5fdaffd 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
@@ -108,7 +108,7 @@ KERNEL(GemmTrellis)(
products.clear();
uzu::dispatch_bool(alignment.contains(GemmAlignment::M) || token_end == Core::SIMDGROUP_BLOCK_M, [&](auto aligned_m) {
uzu::dispatch_bool(alignment.contains(GemmAlignment::N) || row_end == Core::SIMDGROUP_BLOCK_N, [&](auto aligned_n) {
- auto left = quantized::make_left_cursor<true, Core, LeftOperand, aligned_m.value>(
+ auto left = quantized::make_left_cursor<Core, LeftOperand, aligned_m.value>(
left_storage,
params,
tile_context,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index fe4ae2bdf..158082bbf 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -36,11 +36,15 @@ pub struct GemmKernel {
input_data_type: DataType,
output_data_type: DataType,
trellis: TrellisGemm,
- kernels: HashMap<GemmSpecialization, GemmMetalKernel>,
- split_k_reduce: HashMap<GemmDTransform, GemmSplitKReduceMetalKernel>,
+ kernels: HashMap<(GemmSpecialization, DataType), GemmMetalKernel>,
+ split_k_reduce: HashMap<(GemmDTransform, DataType), GemmSplitKReduceMetalKernel>,
}
impl GemmKernel {
+ pub fn requires_activation_group_sums(shape: &MatmulShape) -> bool {
+ shape.b_prologue == GemmBPrologueKind::ScaleBiasDequant
+ }
+
pub fn new(
weights_data_type: DataType,
input_data_type: DataType,
@@ -60,15 +64,16 @@ impl GemmKernel {
&mut self,
context: &MetalContext,
specialization: GemmSpecialization,
+ output_data_type: DataType,
) -> Result<&GemmMetalKernel, MetalError> {
- match self.kernels.entry(specialization) {
+ match self.kernels.entry((specialization, output_data_type)) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
let kernel = GemmMetalKernel::new(
context,
self.input_data_type,
self.weights_data_type,
- self.output_data_type,
+ output_data_type,
specialization.tiling,
specialization.transpose_b,
specialization.use_mxu,
@@ -90,11 +95,17 @@ impl GemmKernel {
&mut self,
context: &MetalContext,
output_transform: GemmDTransform,
+ partial_data_type: DataType,
) -> Result<&GemmSplitKReduceMetalKernel, MetalError> {
- match self.split_k_reduce.entry(output_transform) {
+ match self.split_k_reduce.entry((output_transform, partial_data_type)) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
- let kernel = GemmSplitKReduceMetalKernel::new(context, self.output_data_type, output_transform)?;
+ let kernel = GemmSplitKReduceMetalKernel::new(
+ context,
+ partial_data_type,
+ self.output_data_type,
+ output_transform,
+ )?;
Ok(entry.insert(kernel))
},
}
@@ -288,6 +299,8 @@ impl GemmKernel {
},
};
+ let a_group_sums = a_group_sums.filter(|_| Self::requires_activation_group_sums(&shape));
+
if plan.split_k > 1 {
self.encode_split_k(
a,
@@ -312,8 +325,12 @@ impl GemmKernel {
}
let tiling = plan.tiling;
- let alignment =
- GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);
+ let (alignment_m, alignment_n) = if use_mxu && a_int8.is_some() {
+ (tiling.block_m() / tiling.simdgroups_m(), tiling.block_n() / tiling.simdgroups_n())
+ } else {
+ (tiling.block_m(), tiling.block_n())
+ };
+ let alignment = GemmAlignment::new(m % alignment_m == 0, n % alignment_n == 0, k % tiling.block_k() == 0);
let mut params = gemm_params(shape, plan, ab_scale, scale_strides, zero_point_strides);
let threadgroups_per_row = n.div_ceil(tiling.block_n());
let threadgroups_per_column = m.div_ceil(tiling.block_m());
@@ -349,7 +366,7 @@ impl GemmKernel {
a_prologue,
a_group_size,
)?;
- let kernel = self.get_or_create(command_buffer.context(), specialization)?;
+ let kernel = self.get_or_create(command_buffer.context(), specialization, self.output_data_type)?;
kernel.encode(
a_full_precision,
weights,
@@ -413,6 +430,7 @@ impl GemmKernel {
.into());
},
};
+ let a_group_sums = a_group_sums.filter(|_| Self::requires_activation_group_sums(&shape));
let tiling = plan.tiling;
let split_k = plan.split_k;
let kp = k / split_k;
@@ -432,11 +450,16 @@ impl GemmKernel {
)?;
let elem = (m as usize) * (n as usize);
- let slice_bytes = elem * self.output_data_type.size_in_bytes();
+ let partial_data_type = if a_int8.is_some() {
+ DataType::F32
+ } else {
+ self.output_data_type
+ };
+ let slice_bytes = elem * partial_data_type.size_in_bytes();
let mut temp = command_buffer.allocate_scratch(split_k as usize * slice_bytes)?;
let mut params = gemm_params(shape, plan, 1.0, scale_strides, zero_point_strides);
- params.aligned_inner_iterations = kp / k_step;
- let part_kernel = self.get_or_create(command_buffer.context(), part_spec)?;
+ params.aligned_inner_iterations = kp / k_step / tiling.simdgroups_k();
+ let part_kernel = self.get_or_create(command_buffer.context(), part_spec, partial_data_type)?;
part_kernel.encode(
a_full_precision,
weights,
@@ -470,7 +493,8 @@ impl GemmKernel {
} else {
None
};
- let reduce = self.get_or_create_split_k_reduce(command_buffer.context(), reduce_transform)?;
+ let reduce =
+ self.get_or_create_split_k_reduce(command_buffer.context(), reduce_transform, partial_data_type)?;
reduce.encode(&temp, d.reborrow(), bias_arg, elem as u32, split_k, group_count, n, scale_arg, command_buffer);
Ok(())
@@ -491,9 +515,7 @@ fn validate_int8_left_operand(
}
.into());
}
- let needs_group_sums =
- matches!(shape.b_prologue, GemmBPrologueKind::ScaleBiasDequant | GemmBPrologueKind::ScaleZeroPointDequant);
- if needs_group_sums && !has_group_sums {
+ if GemmKernel::requires_activation_group_sums(&shape) && !has_group_sums {
return Err(MatmulError::IncompatibleA {
path: "Gemm",
reason: "quantized correction requires left group sums",
@@ -542,7 +564,8 @@ fn gemm_params(
leading_dimension_d: n,
threadgroups_per_row: n.div_ceil(tiling.block_n()),
threadgroups_per_column: m.div_ceil(tiling.block_m()),
- aligned_inner_iterations: outer_block_k(shape, plan.engine, plan.tiling).map_or(0, |step| k / step),
+ aligned_inner_iterations: outer_block_k(shape, plan.engine, plan.tiling)
+ .map_or(0, |step| k / step / tiling.simdgroups_k()),
use_morton: false,
ab_scale,
scale_output_stride: scale_strides.output_stride,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
index b7beec182..db4e537a4 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
@@ -11,6 +11,7 @@ use crate::backends::common::gpu_types::gemm::GemmTiling;
const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;
const MXU_SKINNY_M_MAX: u32 = 16;
+const MXU_K_SPLIT_FULL_BLOCK_N_MAX: u32 = 16384;
const SKINNY_SQUARE_K_MAX: u32 = 2560;
const READOUT_N_TO_K_RATIO: u32 = 32;
const WIDE_N_DEEP_K_MIN: u32 = 4096;
@@ -29,6 +30,7 @@ const SIMDGROUP_QUANT_WIDE_N_MIN: u32 = 6144;
const SPLIT_K_TARGET_TILES_FP: u32 = 512;
pub(super) const SPLIT_K_TARGET_TILES_A8: u32 = 256;
pub(super) const SPLIT_K_TARGET_TILES_A8_TILE16X32: u32 = 4 * SPLIT_K_TARGET_TILES_A8;
+const SPLIT_K_TARGET_TILES_A8_TILE16X32_K_SPLIT: u32 = 2 * SPLIT_K_TARGET_TILES_A8;
pub(super) const SPLIT_K_TARGET_TILES_A8_TILE32_W4: u32 = 512;
const SPLIT_K_TARGET_TILES_A8_TILE32_W8: u32 = 1024;
@@ -55,6 +57,19 @@ pub(super) fn mxu_mn_tile(
}
}
+pub(super) fn mxu_int8_gemm_tile(
+ m: u32,
+ n: u32,
+) -> GemmTiling {
+ match mxu_mn_tile(true, m, n) {
+ GemmTiling::Tile128x128x256_Simdgroups4x4 => MXU_DEFAULT_TILE,
+ GemmTiling::Tile16x32x256_Simdgroups1x1 if m < MXU_SKINNY_M_MAX || n <= MXU_K_SPLIT_FULL_BLOCK_N_MAX => {
+ GemmTiling::Tile16x32x256_Simdgroups1x1x4
+ },
+ tiling => tiling,
+ }
+}
+
pub(super) fn mxu_fp_tile(
m: u32,
n: u32,
@@ -135,6 +150,7 @@ pub(super) fn split_k_target_tiles(
(true, GemmTiling::Tile32x64x256_Simdgroups2x2, Some(4)) => SPLIT_K_TARGET_TILES_A8_TILE32_W4,
(true, GemmTiling::Tile32x64x256_Simdgroups2x2, _) => SPLIT_K_TARGET_TILES_A8_TILE32_W8,
(true, GemmTiling::Tile16x32x256_Simdgroups1x1, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32,
+ (true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32_K_SPLIT,
(true, _, _) => SPLIT_K_TARGET_TILES_A8,
(false, _, _) => SPLIT_K_TARGET_TILES_FP,
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index 4f8921c8e..86fb18c70 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -117,7 +117,7 @@ impl GemmProblem {
let Some(step) = outer_block_k(shape, engine, tiling) else {
return 1;
};
- let align = step.max(ACTIVATION_SCALE_GROUP_SIZE);
+ let align = step.max(ACTIVATION_SCALE_GROUP_SIZE) * tiling.simdgroups_k();
let target_tiles = policy::split_k_target_tiles(!shape.a_full_precision, tiling, shape.b_bits);
let mut split_k = (target_tiles / base_tiles).max(1).min((shape.k / align).max(1));
if !shape.a_full_precision && engine == GemmEngine::Mxu && tiling.block_k() != 0 {
@@ -203,13 +203,21 @@ fn select_tiling(
match engine {
GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m, shape.n, apple_gpu_family),
GemmEngine::Simdgroup => policy::simdgroup_fp_tile(shape.m, shape.n, shape.k),
- GemmEngine::Mxu if !shape.a_full_precision || shape.is_quant() => {
- policy::mxu_mn_tile(!shape.a_full_precision, shape.m, shape.n)
+ GemmEngine::Mxu if !shape.a_full_precision => match policy::mxu_int8_gemm_tile(shape.m, shape.n) {
+ tiling
+ if tiling.simdgroups_k() > 1
+ && !shape.k.is_multiple_of(tiling.simdgroups_k() * ACTIVATION_SCALE_GROUP_SIZE) =>
+ {
+ GemmTiling::Tile16x32x256_Simdgroups1x1
+ },
+ tiling => tiling,
},
+ GemmEngine::Mxu if shape.is_quant() => policy::mxu_mn_tile(false, shape.m, shape.n),
GemmEngine::Mxu if shape.b_transpose => policy::mxu_fp_tile(shape.m, shape.n, shape.k),
GemmEngine::Mxu => policy::mxu_mn_tile(false, shape.m, shape.n),
}
}
+
#[cfg(test)]
#[path = "../../../../../../unit/backends/metal/kernel/matmul/gemm/selection_test.rs"]
mod tests;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h
index abae2e1a5..e6bab1f36 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h
@@ -69,7 +69,6 @@ private:
bool signed_codes
) {
constexpr uint CODES_PER_WORD = WORD_BITS / BITS;
- constexpr uint W8_SIGN_BITS = 0x80808080u;
if constexpr (BITS == 4) {
word0 ^= signed_codes ? W4_SIGN_MASK : 0u;
METAL_PRAGMA_UNROLL
@@ -77,7 +76,7 @@ private:
values[i] = as_type<float>(extract_bits(word0, BITS * i, BITS) | MANTISSA_BASE_BITS) - origin;
}
} else {
- const uint mask = signed_codes ? W8_SIGN_BITS : 0u;
+ const uint mask = signed_codes ? W8_SIGN_MASK : 0u;
word0 ^= mask;
word1 ^= mask;
METAL_PRAGMA_UNROLL
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_dispatch.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_dispatch.rs
new file mode 100644
index 000000000..8e7fb7cf8
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_dispatch.rs
@@ -0,0 +1,6 @@
+use super::{gemm::GemmPlan, gemv::GemvSpecialization};
+
+pub enum MatmulDispatch {
+ Gemv(GemvSpecialization),
+ Gemm(GemmPlan),
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
new file mode 100644
index 000000000..e40b744f7
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
@@ -0,0 +1,274 @@
+use metal::MTLGPUFamily;
+
+use super::{
+ GemmKernel,
+ gemm::{self, GemmPlan, GemmProblem},
+ gemv::{GemvKernel, GemvSpecialization},
+ matmul_dispatch::MatmulDispatch,
+ matmul_output_work::MatmulOutputWork,
+ qmv::{self, QmvRoute},
+};
+use crate::{
+ backends::{
+ common::{
+ BufferMut, BufferRef, CommandBufferEncoding,
+ gpu_types::gemm::GemmTiling,
+ kernel::{
+ ActivationQuantization,
+ activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
+ matmul::{
+ ActivationFormat, Int8CodeLayout, MatmulArguments, MatmulError, MatmulKernel, MatmulShape,
+ QuantParamsLayout,
+ },
+ },
+ },
+ metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},
+ },
+ data_type::DataType,
+};
+
+pub struct MatmulMetalKernel {
+ gemv: GemvKernel,
+ pub gemm: GemmKernel,
+ output_work: MatmulOutputWork,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+}
+
+pub fn supports_integer_right_operand(shape: &MatmulShape) -> bool {
+ matches!((shape.b_bits, shape.signed_codes), (Some(4), _) | (Some(8), true))
+}
+
+impl MatmulMetalKernel {
+ pub fn prefer_gemm_over_gemv(
+ shape: MatmulShape,
+ plan: GemmPlan,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ ) -> bool {
+ if shape.gathered || plan.engine != gemm::GemmEngine::Mxu {
+ return false;
+ }
+ match (shape.m, shape.n == shape.k, (weights_data_type, input_data_type, output_data_type)) {
+ (4, true, (DataType::F32, DataType::F32, DataType::F32))
+ | (5, _, (DataType::BF16, DataType::BF16, DataType::BF16)) => return false,
+ _ => {},
+ }
+ match shape.m {
+ 0..=3 => return false,
+ 4 => {
+ let small_enough_for_mxu = shape.n <= 6144 && shape.k <= 9728;
+ let k_dominates = shape.k > 3_u32.saturating_mul(shape.n);
+ if !(small_enough_for_mxu || k_dominates) {
+ return false;
+ }
+ },
+ _ => {},
+ }
+ matches!(
+ plan.tiling,
+ GemmTiling::Tile16x32x256_Simdgroups1x1
+ | GemmTiling::Tile16x32x256_Simdgroups1x1x4
+ | GemmTiling::Tile16x128x256_Simdgroups1x4
+ )
+ }
+
+ pub fn choose_dispatch(
+ shape: &MatmulShape,
+ device_name: &str,
+ gpu_core_count: u32,
+ apple_gpu_family: MTLGPUFamily,
+ supports_mxu: bool,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ ) -> MatmulDispatch {
+ let all_bf16 = weights_data_type == DataType::BF16
+ && input_data_type == DataType::BF16
+ && output_data_type == DataType::BF16;
+ let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);
+ if shape.b_is_trellis {
+ return MatmulDispatch::Gemm(problem.select_plan());
+ }
+ if let Some(route) = qmv::route(device_name, apple_gpu_family, supports_mxu, shape, all_bf16) {
+ return match route {
+ QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(
+ GemvSpecialization::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile)
+ .expect("typed QMV route must contain a legal GEMV tile"),
+ ),
+ QmvRoute::MainGemm(plan) => MatmulDispatch::Gemm(plan),
+ };
+ }
+ let gemv = GemvSpecialization::select_shape(
+ shape,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
+ gpu_core_count,
+ apple_gpu_family,
+ );
+ let plan = problem.select_plan();
+ match gemv {
+ None => MatmulDispatch::Gemm(plan),
+ Some(_)
+ if Self::prefer_gemm_over_gemv(*shape, plan, weights_data_type, input_data_type, output_data_type) =>
+ {
+ MatmulDispatch::Gemm(plan)
+ },
+ Some(gemv) => MatmulDispatch::Gemv(gemv),
+ }
+ }
+
+ fn select_dispatch(
+ &self,
+ shape: &MatmulShape,
+ context: &MetalContext,
+ ) -> MatmulDispatch {
+ Self::choose_dispatch(
+ shape,
+ &context.device_name,
+ context.gpu_core_count,
+ context.apple_gpu_family,
+ context.supports_mxu,
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
+ )
+ }
+}
+
+impl MatmulKernel for MatmulMetalKernel {
+ type Backend = Metal;
+
+ fn new(
+ context: &MetalContext,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ ) -> Result<Self, MetalError> {
+ for data_type in [weights_data_type, input_data_type, output_data_type] {
+ if !matches!(data_type, DataType::BF16 | DataType::F32) {
+ return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());
+ }
+ }
+
+ let output_work = MatmulOutputWork::new(context, weights_data_type, output_data_type)?;
+ let gemm = GemmKernel::new(weights_data_type, input_data_type, output_data_type);
+ let gemv = GemvKernel::new(weights_data_type, input_data_type, output_data_type);
+
+ Ok(Self {
+ gemv,
+ gemm,
+ output_work,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
+ })
+ }
+
+ fn select_activation_quantization(
+ &self,
+ shape: &MatmulShape,
+ context: &MetalContext,
+ ) -> Option<ActivationQuantization> {
+ let weight_group_size = shape.b_group_size?;
+ let emit_group_sums = GemmKernel::requires_activation_group_sums(shape);
+ let code_layout = shape.b_bits.and_then(Int8CodeLayout::for_right_bits)?;
+ let quantization =
+ ActivationQuantization::new(ACTIVATION_SCALE_GROUP_SIZE, weight_group_size, emit_group_sums, code_layout)?;
+ if !context.supports_mxu
+ || self.input_data_type != DataType::BF16
+ || self.output_data_type != DataType::BF16
+ || shape.a_full_precision
+ || !shape.is_quant()
+ || shape.params_layout != Some(QuantParamsLayout::GroupOutput)
+ || !supports_integer_right_operand(shape)
+ || !shape.b_transpose
+ || shape.b_leading_dimension.is_some()
+ || !shape.k.is_multiple_of(ACTIVATION_SCALE_GROUP_SIZE)
+ || !shape.k.is_multiple_of(weight_group_size)
+ {
+ return None;
+ }
+
+ Some(quantization)
+ }
+
+ fn select_activation_format(
+ &self,
+ bf16_shape: &MatmulShape,
+ context: &MetalContext,
+ ) -> ActivationFormat {
+ if bf16_shape.params_layout != Some(QuantParamsLayout::GroupOutput)
+ || !supports_integer_right_operand(bf16_shape)
+ || matches!(self.select_dispatch(bf16_shape, context), MatmulDispatch::Gemv(_))
+ {
+ return ActivationFormat::Bf16;
+ }
+
+ let a8_shape = MatmulShape {
+ a_full_precision: false,
+ ..*bf16_shape
+ };
+ match self.select_dispatch(&a8_shape, context) {
+ MatmulDispatch::Gemm(plan) if plan.engine == gemm::GemmEngine::Mxu => ActivationFormat::Int8,
+ _ => ActivationFormat::Bf16,
+ }
+ }
+
+ fn encode(
+ &mut self,
+ arguments: MatmulArguments<
+ '_,
+ Metal,
+ impl BufferRef<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
+ impl BufferMut<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
+ >,
+ command_buffer: &mut MetalCommandBufferEncoding,
+ ) -> Result<(), MetalError> {
+ let shape = MatmulShape::from_arguments(&arguments);
+ let plan = match self.select_dispatch(&shape, command_buffer.context()) {
+ MatmulDispatch::Gemv(gemv) => {
+ return self.gemv.encode(arguments, gemv, &self.output_work, command_buffer).map_err(MetalError::from);
+ },
+ MatmulDispatch::Gemm(plan) => plan,
+ };
+
+ // TODO: remove after GatherGEMM is supported
+ if arguments.gather_indices.is_some() {
+ return Err(MetalError::KernelDispatchFailed(
+ format!(
+ "gathered readout requires the GEMV path, but shape (m={}, n={}) routes to GEMM",
+ arguments.m, arguments.n
+ )
+ .into(),
+ ));
+ }
+ self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)
+ }
+}
+
+#[cfg(test)]
+impl MatmulMetalKernel {
+ pub fn encode_with_gemm_engine(
+ &mut self,
+ arguments: MatmulArguments<
+ '_,
+ Metal,
+ impl BufferRef<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
+ impl BufferMut<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
+ >,
+ engine: gemm::GemmEngine,
+ command_buffer: &mut MetalCommandBufferEncoding,
+ ) -> Result<(), MetalError> {
+ let shape = MatmulShape::from_arguments(&arguments);
+ let plan = self.gemm.select_plan_for_engine(&shape, engine, command_buffer.context())?;
+ self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_output_work.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_output_work.rs
new file mode 100644
index 000000000..a46201fd9
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_output_work.rs
@@ -0,0 +1,47 @@
+use crate::{
+ backends::{
+ common::{Backend, BufferMut, BufferRef, kernel::ActivationTransform},
+ metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},
+ },
+ data_type::DataType,
+};
+
+pub struct MatmulOutputWork {
+ output_rht: ActivationTransform<Metal>,
+ output_rht_with_bias: ActivationTransform<Metal>,
+}
+
+impl MatmulOutputWork {
+ pub fn new(
+ context: &MetalContext,
+ weights_data_type: DataType,
+ output_data_type: DataType,
+ ) -> Result<Self, MetalError> {
+ Ok(Self {
+ output_rht: ActivationTransform::output_rht(context, output_data_type, None, true)?,
+ output_rht_with_bias: ActivationTransform::output_rht(
+ context,
+ output_data_type,
+ Some(weights_data_type),
+ true,
+ )?,
+ })
+ }
+
+ pub fn apply(
+ &self,
+ output: impl BufferMut<Backend = Metal>,
+ factors: impl BufferRef<Backend = Metal>,
+ bias: Option<&<Metal as Backend>::GlobalBuffer>,
+ m: u32,
+ n: u32,
+ command_buffer: &mut MetalCommandBufferEncoding,
+ ) {
+ let transform = if bias.is_some() {
+ &self.output_rht_with_bias
+ } else {
+ &self.output_rht
+ };
+ transform.encode_fp_in_place(output, factors, bias, m, n, command_buffer);
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
index 4257c9dda..6121db708 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
@@ -1,308 +1,13 @@
pub mod gemm;
mod gemv;
+mod matmul_dispatch;
+mod matmul_metal_kernel;
+mod matmul_output_work;
mod qmv;
-use metal::MTLGPUFamily;
-
-pub use self::gemm::GemmKernel;
-use self::{
- gemm::{GemmPlan, GemmProblem},
- gemv::{GemvKernel, GemvSpecialization},
- qmv::QmvRoute,
-};
-use crate::{
- backends::{
- common::{
- Backend, BufferMut, BufferRef, CommandBufferEncoding,
- gpu_types::gemm::{GemmBPrologueKind, GemmTiling},
- kernel::{
- ActivationQuantization, ActivationTransform,
- activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
- matmul::{
- ActivationFormat, Int8CodeLayout, MatmulArguments, MatmulError, MatmulKernel, MatmulShape,
- QuantParamsLayout,
- },
- },
- },
- metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},
- },
- data_type::DataType,
-};
-
-pub struct MatmulMetalKernel {
- gemv: GemvKernel,
- gemm: GemmKernel,
- output_work: MatmulOutputWork,
- weights_data_type: DataType,
- input_data_type: DataType,
- output_data_type: DataType,
-}
-
-enum MatmulDispatch {
- Gemv(GemvSpecialization),
- Gemm(GemmPlan),
-}
-
-pub struct MatmulOutputWork {
- output_rht: ActivationTransform<Metal>,
- output_rht_with_bias: ActivationTransform<Metal>,
-}
-
-impl MatmulOutputWork {
- fn new(
- context: &MetalContext,
- weights_data_type: DataType,
- output_data_type: DataType,
- ) -> Result<Self, MetalError> {
- Ok(Self {
- output_rht: ActivationTransform::output_rht(context, output_data_type, None, true)?,
- output_rht_with_bias: ActivationTransform::output_rht(
- context,
- output_data_type,
- Some(weights_data_type),
- true,
- )?,
- })
- }
-
- fn apply(
- &self,
- output: impl BufferMut<Backend = Metal>,
- factors: impl BufferRef<Backend = Metal>,
- bias: Option<&<Metal as Backend>::GlobalBuffer>,
- m: u32,
- n: u32,
- command_buffer: &mut MetalCommandBufferEncoding,
- ) {
- let transform = if bias.is_some() {
- &self.output_rht_with_bias
- } else {
- &self.output_rht
- };
- transform.encode_fp_in_place(output, factors, bias, m, n, command_buffer);
- }
-}
-
-fn supports_integer_right_operand(shape: &MatmulShape) -> bool {
- matches!((shape.b_bits, shape.signed_codes), (Some(4), _) | (Some(8), true))
-}
-
-impl MatmulMetalKernel {
- fn prefer_gemm_over_gemv(
- shape: MatmulShape,
- plan: GemmPlan,
- weights_data_type: DataType,
- input_data_type: DataType,
- output_data_type: DataType,
- ) -> bool {
- if shape.gathered || plan.engine != gemm::GemmEngine::Mxu {
- return false;
- }
- match (shape.m, shape.n == shape.k, (weights_data_type, input_data_type, output_data_type)) {
- (4, true, (DataType::F32, DataType::F32, DataType::F32))
- | (5, _, (DataType::BF16, DataType::BF16, DataType::BF16)) => return false,
- _ => {},
- }
- match shape.m {
- 0..=3 => return false,
- 4 => {
- let small_enough_for_mxu = shape.n <= 6144 && shape.k <= 9728;
- let k_dominates = shape.k > 3_u32.saturating_mul(shape.n);
- if !(small_enough_for_mxu || k_dominates) {
- return false;
- }
- },
- _ => {},
- }
- matches!(plan.tiling, GemmTiling::Tile16x32x256_Simdgroups1x1 | GemmTiling::Tile16x128x256_Simdgroups1x4)
- }
-
- fn choose_dispatch(
- shape: &MatmulShape,
- device_name: &str,
- gpu_core_count: u32,
- apple_gpu_family: MTLGPUFamily,
- supports_mxu: bool,
- weights_data_type: DataType,
- input_data_type: DataType,
- output_data_type: DataType,
- ) -> MatmulDispatch {
- let all_bf16 = weights_data_type == DataType::BF16
- && input_data_type == DataType::BF16
- && output_data_type == DataType::BF16;
- let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);
- if shape.b_is_trellis {
- return MatmulDispatch::Gemm(problem.select_plan());
- }
- if let Some(route) = qmv::route(device_name, apple_gpu_family, supports_mxu, shape, all_bf16) {
- return match route {
- QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(
- GemvSpecialization::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile)
- .expect("typed QMV route must contain a legal GEMV tile"),
- ),
- QmvRoute::MainGemm(plan) => MatmulDispatch::Gemm(plan),
- };
- }
- let gemv = GemvSpecialization::select_shape(
- shape,
- weights_data_type,
- input_data_type,
- output_data_type,
- gpu_core_count,
- apple_gpu_family,
- );
- let plan = problem.select_plan();
- match gemv {
- None => MatmulDispatch::Gemm(plan),
- Some(_)
- if Self::prefer_gemm_over_gemv(*shape, plan, weights_data_type, input_data_type, output_data_type) =>
- {
- MatmulDispatch::Gemm(plan)
- },
- Some(gemv) => MatmulDispatch::Gemv(gemv),
- }
- }
-
- fn select_dispatch(
- &self,
- shape: &MatmulShape,
- context: &MetalContext,
- ) -> MatmulDispatch {
- Self::choose_dispatch(
- shape,
- &context.device_name,
- context.gpu_core_count,
- context.apple_gpu_family,
- context.supports_mxu,
- self.weights_data_type,
- self.input_data_type,
- self.output_data_type,
- )
- }
-}
-
-impl MatmulKernel for MatmulMetalKernel {
- type Backend = Metal;
-
- fn new(
- context: &MetalContext,
- weights_data_type: DataType,
- input_data_type: DataType,
- output_data_type: DataType,
- ) -> Result<Self, MetalError> {
- for data_type in [weights_data_type, input_data_type, output_data_type] {
- if !matches!(data_type, DataType::BF16 | DataType::F32) {
- return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());
- }
- }
-
- let output_work = MatmulOutputWork::new(context, weights_data_type, output_data_type)?;
- let gemm = GemmKernel::new(weights_data_type, input_data_type, output_data_type);
- let gemv = GemvKernel::new(weights_data_type, input_data_type, output_data_type);
-
- Ok(Self {
- gemv,
- gemm,
- output_work,
- weights_data_type,
- input_data_type,
- output_data_type,
- })
- }
-
- fn select_activation_quantization(
- &self,
- shape: &MatmulShape,
- context: &MetalContext,
- ) -> Option<ActivationQuantization> {
- let weight_group_size = shape.b_group_size?;
- let emit_group_sums = match shape.b_prologue {
- GemmBPrologueKind::ScaleSymmetricDequant => false,
- GemmBPrologueKind::ScaleBiasDequant | GemmBPrologueKind::ScaleZeroPointDequant => true,
- GemmBPrologueKind::FullPrecision => return None,
- };
- let code_layout = shape.b_bits.and_then(Int8CodeLayout::for_right_bits)?;
- let quantization =
- ActivationQuantization::new(ACTIVATION_SCALE_GROUP_SIZE, weight_group_size, emit_group_sums, code_layout)?;
- if !context.supports_mxu
- || self.input_data_type != DataType::BF16
- || self.output_data_type != DataType::BF16
- || shape.a_full_precision
- || !shape.is_quant()
- || shape.params_layout != Some(QuantParamsLayout::GroupOutput)
- || !supports_integer_right_operand(shape)
- || !shape.b_transpose
- || shape.b_leading_dimension.is_some()
- || !shape.k.is_multiple_of(ACTIVATION_SCALE_GROUP_SIZE)
- || !shape.k.is_multiple_of(weight_group_size)
- {
- return None;
- }
-
- Some(quantization)
- }
-
- fn select_activation_format(
- &self,
- bf16_shape: &MatmulShape,
- context: &MetalContext,
- ) -> ActivationFormat {
- if bf16_shape.params_layout != Some(QuantParamsLayout::GroupOutput)
- || !supports_integer_right_operand(bf16_shape)
- || matches!(self.select_dispatch(bf16_shape, context), MatmulDispatch::Gemv(_))
- {
- return ActivationFormat::Bf16;
- }
-
- let a8_shape = MatmulShape {
- a_full_precision: false,
- ..*bf16_shape
- };
- match self.select_dispatch(&a8_shape, context) {
- MatmulDispatch::Gemm(plan) if plan.engine == gemm::GemmEngine::Mxu => ActivationFormat::Int8,
- _ => ActivationFormat::Bf16,
- }
- }
-
- fn encode(
- &mut self,
- arguments: MatmulArguments<
- '_,
- Metal,
- impl BufferRef<Backend = Metal>,
- impl BufferRef<Backend = Metal>,
- impl BufferMut<Backend = Metal>,
- impl BufferRef<Backend = Metal>,
- >,
- command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(), MetalError> {
- let shape = MatmulShape::from_arguments(&arguments);
- match self.select_dispatch(&shape, command_buffer.context()) {
- MatmulDispatch::Gemv(gemv) => {
- self.gemv.encode(arguments, gemv, &self.output_work, command_buffer).map_err(MetalError::from)
- },
- MatmulDispatch::Gemm(plan) => self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer),
- }
- }
-}
-
+pub use gemm::GemmKernel;
#[cfg(test)]
-impl MatmulMetalKernel {
- pub fn encode_with_gemm_engine(
- &mut self,
- arguments: MatmulArguments<
- '_,
- Metal,
- impl BufferRef<Backend = Metal>,
- impl BufferRef<Backend = Metal>,
- impl BufferMut<Backend = Metal>,
- impl BufferRef<Backend = Metal>,
- >,
- engine: gemm::GemmEngine,
- command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(), MetalError> {
- let shape = MatmulShape::from_arguments(&arguments);
- let plan = self.gemm.select_plan_for_engine(&shape, engine, command_buffer.context())?;
- self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)
- }
-}
+use matmul_dispatch::MatmulDispatch;
+pub use matmul_metal_kernel::MatmulMetalKernel;
+use matmul_metal_kernel::supports_integer_right_operand;
+pub use matmul_output_work::MatmulOutputWork;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/mod.rs
index 9c0d00097..30c7939fb 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/mod.rs
@@ -3,7 +3,7 @@ use crate::backends::{
Kernels,
gpu_types::{
HADAMARD_TRANSFORM_BLOCK_SIZE,
- gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_row},
+ gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_k, gemm_tiling_simdgroups_per_row},
weaver::{FRONTIER_SELECT_THREADS, TOP_CHILDREN_THREADS},
},
},
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index 7b7ff7826..ac3233c8b 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -118,9 +118,11 @@ fn selection_fallbacks_and_split_k_are_preserved() {
assert_eq!(problem(quant(shape(64, 4096, 4095)), DataType::BF16).select_plan().engine, Simdgroup);
for (m, n, k, expected_tiling, expected_split_k) in [
- (16, 4096, 4096, Tile16x32x256_Simdgroups1x1, 8),
- (16, 1024, 4096, Tile16x32x256_Simdgroups1x1, 16),
+ (16, 4096, 4096, Tile16x32x256_Simdgroups1x1x4, 4),
+ (16, 1024, 4096, Tile16x32x256_Simdgroups1x1x4, 8),
(16, 34816, 4096, Tile16x32x256_Simdgroups1x1, 1),
+ (8, 34816, 4096, Tile16x32x256_Simdgroups1x1x4, 1),
+ (8, 4096, 384, Tile16x32x256_Simdgroups1x1, 1),
] {
let mut a8 = quant(shape(m, n, k));
a8.a_full_precision = false;