Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

some trellis gemm optimizations (#998)

CCYeh2229bf8521ec
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
index 5e5fdaffd..83855456b 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
@@ -41,6 +41,7 @@ template <GemmTiling GEMM_TILING>
VARIANTS(
GEMM_TILING,
GemmTiling::Tile16x32x256_Simdgroups1x1,
+ GemmTiling::Tile32x64x256_Simdgroups2x2,
GemmTiling::Tile64x64x256_Simdgroups2x2,
GemmTiling::Tile128x128x256_Simdgroups4x4)
KERNEL(GemmTrellis)(
@@ -125,13 +126,41 @@ KERNEL(GemmTrellis)(
const int k_chunks = token_end > 0 && row_end > 0
? int(params->aligned_inner_iterations) * int(trellis::K_STEP / Core::SIMDGROUP_BLOCK_K)
: 0;
- METAL_PRAGMA_NO_UNROLL
- for (int chunk = 0; chunk < k_chunks; ++chunk) {
- auto left_tile = left.load(0u);
- auto right_tile = trellis_cursor.load(0u);
- uzu::matmul::fragment_mma(products, left_tile, right_tile);
- left.advance();
- trellis_cursor.advance();
+ if constexpr (GEMM_TILING == GemmTiling::Tile128x128x256_Simdgroups4x4) {
+ if (trellis_vector_width == trellis::V4_VECTOR_WIDTH && trellis_transition_bits == 8 &&
+ trellis_restart_columns == 64 && params->aligned_inner_iterations > 1) {
+ static_assert(trellis::K_STEP % (2 * Core::SIMDGROUP_BLOCK_K) == 0, "paired K chunks must be complete");
+ auto run_chunk = [&]() {
+ auto left_tile = left.load(0u);
+ auto right_tile = trellis_cursor.load(0u);
+ uzu::matmul::fragment_mma(products, left_tile, right_tile);
+ left.advance();
+ trellis_cursor.advance();
+ };
+ METAL_PRAGMA_NO_UNROLL
+ for (int chunk = 0; chunk < k_chunks; chunk += 2) {
+ run_chunk();
+ run_chunk();
+ }
+ } else {
+ METAL_PRAGMA_NO_UNROLL
+ for (int chunk = 0; chunk < k_chunks; ++chunk) {
+ auto left_tile = left.load(0u);
+ auto right_tile = trellis_cursor.load(0u);
+ uzu::matmul::fragment_mma(products, left_tile, right_tile);
+ left.advance();
+ trellis_cursor.advance();
+ }
+ }
+ } else {
+ METAL_PRAGMA_NO_UNROLL
+ for (int chunk = 0; chunk < k_chunks; ++chunk) {
+ auto left_tile = left.load(0u);
+ auto right_tile = trellis_cursor.load(0u);
+ uzu::matmul::fragment_mma(products, left_tile, right_tile);
+ left.advance();
+ trellis_cursor.advance();
+ }
}
});
});
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index 3b45fe82f..1c4894164 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -154,18 +154,24 @@ pub(super) fn select_trellis_plan(
n: u32,
k: u32,
) -> GemmPlan {
- let (tiling, target_workgroups) = match policy::mxu_mn_tile(true, m, n) {
- GemmTiling::Tile16x32x256_Simdgroups1x1 => {
- (GemmTiling::Tile16x32x256_Simdgroups1x1, policy::SPLIT_K_TARGET_TILES_A8_TILE16X32)
- },
- GemmTiling::Tile32x64x256_Simdgroups2x2 => {
- (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8_TILE32_W4)
- },
- GemmTiling::Tile128x128x256_Simdgroups4x4 => {
- (GemmTiling::Tile128x128x256_Simdgroups4x4, policy::SPLIT_K_TARGET_TILES_A8)
- },
- _ => (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8),
- };
+ // The M16 wide finalist favored a 32x64 tile and a 256-workgroup target.
+ let (tiling, target_workgroups) =
+ if m == 16 && n >= 4096 && (2048..=5120).contains(&k) && k.is_multiple_of(TRELLIS_K_STEP) {
+ (GemmTiling::Tile32x64x256_Simdgroups2x2, 256)
+ } else {
+ match policy::mxu_mn_tile(true, m, n) {
+ GemmTiling::Tile16x32x256_Simdgroups1x1 => {
+ (GemmTiling::Tile16x32x256_Simdgroups1x1, policy::SPLIT_K_TARGET_TILES_A8_TILE16X32)
+ },
+ GemmTiling::Tile32x64x256_Simdgroups2x2 => {
+ (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8_TILE32_W4)
+ },
+ GemmTiling::Tile128x128x256_Simdgroups4x4 => {
+ (GemmTiling::Tile128x128x256_Simdgroups4x4, policy::SPLIT_K_TARGET_TILES_A8)
+ },
+ _ => (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8),
+ }
+ };
let output_workgroups = (n.div_ceil(tiling.block_n()) * m.div_ceil(tiling.block_m())).max(1);
let k_steps = (k / TRELLIS_K_STEP).max(1);
let mut split_k = (target_workgroups / output_workgroups).clamp(1, k_steps);
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index 71a8e68d2..9474db934 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -155,6 +155,12 @@ fn trellis_plan_matches_projection_cases() {
for (m, n, k, tiling, split_k) in [
(16, 128, 64, Tile16x32x256_Simdgroups1x1, 1),
+ (16, 4096, 2048, Tile32x64x256_Simdgroups2x2, 4),
+ (16, 5120, 5120, Tile32x64x256_Simdgroups2x2, 2),
+ (16, 5120, 5184, Tile16x32x256_Simdgroups1x1, 3),
+ (16, 4096, 1984, Tile16x32x256_Simdgroups1x1, 1),
+ (16, 32, 5120, Tile64x64x256_Simdgroups2x2, 80),
+ (17, 12_288, 2048, Tile64x64x256_Simdgroups2x2, 2),
(17, 80, 64, Tile64x64x256_Simdgroups2x2, 1),
(2048, 2048, 128, Tile128x128x256_Simdgroups4x4, 1),
(1, 80, 5120, Tile16x32x256_Simdgroups1x1, 80),
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs
index 1f4acba36..e3654dc8b 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs
@@ -8,10 +8,15 @@ use crate::{
common::{
Backend, Context, Kernels,
kernel::matmul::{
- MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput, TrellisFormat,
+ MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput, MatmulShape, TrellisFormat,
},
},
- metal::{Metal, context::MetalContext},
+ metal::{
+ Metal,
+ context::MetalContext,
+ gpu_types::gemm::GemmTiling,
+ kernel::matmul::{GemmKernel, MatmulOutputWork, gemm::GemmEngine},
+ },
},
data_type::DataType,
tests::{
@@ -81,13 +86,22 @@ const PADDING: usize = 24;
const SENTINEL: u16 = 0x7F7F;
const RELATIVE_TOLERANCE: f32 = 1e-2;
// Cases are ordered as (m, n, k).
-const CASES: [(usize, usize, usize); 7] =
- [(1, 80, 5120), (17, 80, 5120), (512, 80, 5120), (16, 128, 64), (32, 128, 64), (1, 6, 64), (2048, 2048, 128)];
+const CASES: [(usize, usize, usize); 8] = [
+ (1, 80, 5120),
+ (17, 80, 5120),
+ (512, 80, 5120),
+ (16, 4096, 2048),
+ (16, 128, 64),
+ (32, 128, 64),
+ (1, 6, 64),
+ (2048, 2048, 128),
+];
fn run_projection(
context: &MetalContext,
format: TrellisFormat,
dimensions: (usize, usize, usize),
+ forced_plan: Option<(GemmTiling, u32)>,
) {
let (m, n, k) = dimensions;
let mut rng = SmallRng::seed_from_u64((m * k + n) as u64);
@@ -98,6 +112,15 @@ fn run_projection(
// Exercise the zero-activation path in the epilogue.
values[k..2 * k].fill(0);
}
+ if m > 3 {
+ for (column, value) in values[3 * k..4 * k].iter_mut().enumerate() {
+ *value = if column % 2 == 0 {
+ -128
+ } else {
+ 127
+ };
+ }
+ }
let group_sums_host: Vec<f32> = values
.chunks_exact(k)
.flat_map(|row| {
@@ -136,10 +159,20 @@ fn run_projection(
n: n as u32,
k: k as u32,
};
- let mut matmul =
- <MetalMatmul as MatmulKernel>::new(context, DataType::BF16, DataType::BF16, DataType::BF16).unwrap();
let mut command_buffer = context.create_command_buffer(None, None).unwrap();
- matmul.encode(arguments, &mut command_buffer).unwrap();
+ if let Some((tiling, split_k)) = forced_plan {
+ let shape = MatmulShape::from_arguments(&arguments);
+ let mut gemm = GemmKernel::new(DataType::BF16, DataType::BF16, DataType::BF16);
+ let mut plan = gemm.select_plan_for_engine(&shape, GemmEngine::Mxu, context).unwrap();
+ plan.tiling = tiling;
+ plan.split_k = split_k;
+ let output_work = MatmulOutputWork::new(context, DataType::BF16, DataType::BF16).unwrap();
+ gemm.encode_plan(arguments, plan, &output_work, &mut command_buffer).unwrap();
+ } else {
+ let mut matmul =
+ <MetalMatmul as MatmulKernel>::new(context, DataType::BF16, DataType::BF16, DataType::BF16).unwrap();
+ matmul.encode(arguments, &mut command_buffer).unwrap();
+ }
submit_command_buffer(command_buffer);
let decoded_levels: Vec<i32> = stored
.chunks_exact(row_bytes(format, k))
@@ -181,7 +214,10 @@ fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {
if !context.supports_mxu {
return;
}
- for (m, n, k) in CASES {
- run_projection(&context, format, (m, n, k));
+ for dimensions in CASES {
+ run_projection(&context, format, dimensions, None);
}
+ run_projection(&context, format, (129, 132, 1024), Some((GemmTiling::Tile128x128x256_Simdgroups4x4, 1)));
+ run_projection(&context, format, (129, 132, 64), Some((GemmTiling::Tile128x128x256_Simdgroups4x4, 1)));
+ run_projection(&context, format, (17, 80, 192), Some((GemmTiling::Tile32x64x256_Simdgroups2x2, 1)));
}