A high-performance inference engine for AI models
some trellis gemm optimizations (#998)
CCYeh2229bf8521ec
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metalindex 5e5fdaffd..83855456b 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal@@ -41,6 +41,7 @@ template <GemmTiling GEMM_TILING>VARIANTS(GEMM_TILING,GemmTiling::Tile16x32x256_Simdgroups1x1,+ GemmTiling::Tile32x64x256_Simdgroups2x2,GemmTiling::Tile64x64x256_Simdgroups2x2,GemmTiling::Tile128x128x256_Simdgroups4x4)KERNEL(GemmTrellis)(@@ -125,13 +126,41 @@ KERNEL(GemmTrellis)(const int k_chunks = token_end > 0 && row_end > 0? int(params->aligned_inner_iterations) * int(trellis::K_STEP / Core::SIMDGROUP_BLOCK_K): 0;- METAL_PRAGMA_NO_UNROLL- for (int chunk = 0; chunk < k_chunks; ++chunk) {- auto left_tile = left.load(0u);- auto right_tile = trellis_cursor.load(0u);- uzu::matmul::fragment_mma(products, left_tile, right_tile);- left.advance();- trellis_cursor.advance();+ if constexpr (GEMM_TILING == GemmTiling::Tile128x128x256_Simdgroups4x4) {+ if (trellis_vector_width == trellis::V4_VECTOR_WIDTH && trellis_transition_bits == 8 &&+ trellis_restart_columns == 64 && params->aligned_inner_iterations > 1) {+ static_assert(trellis::K_STEP % (2 * Core::SIMDGROUP_BLOCK_K) == 0, "paired K chunks must be complete");+ auto run_chunk = [&]() {+ auto left_tile = left.load(0u);+ auto right_tile = trellis_cursor.load(0u);+ uzu::matmul::fragment_mma(products, left_tile, right_tile);+ left.advance();+ trellis_cursor.advance();+ };+ METAL_PRAGMA_NO_UNROLL+ for (int chunk = 0; chunk < k_chunks; chunk += 2) {+ run_chunk();+ run_chunk();+ }+ } else {+ METAL_PRAGMA_NO_UNROLL+ for (int chunk = 0; chunk < k_chunks; ++chunk) {+ auto left_tile = left.load(0u);+ auto right_tile = trellis_cursor.load(0u);+ uzu::matmul::fragment_mma(products, left_tile, right_tile);+ left.advance();+ trellis_cursor.advance();+ }+ }+ } else {+ METAL_PRAGMA_NO_UNROLL+ for (int chunk = 0; chunk < k_chunks; ++chunk) {+ auto left_tile = left.load(0u);+ auto right_tile = trellis_cursor.load(0u);+ uzu::matmul::fragment_mma(products, left_tile, right_tile);+ left.advance();+ trellis_cursor.advance();+ }}});});diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex 3b45fe82f..1c4894164 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -154,18 +154,24 @@ pub(super) fn select_trellis_plan(n: u32,k: u32,) -> GemmPlan {- let (tiling, target_workgroups) = match policy::mxu_mn_tile(true, m, n) {- GemmTiling::Tile16x32x256_Simdgroups1x1 => {- (GemmTiling::Tile16x32x256_Simdgroups1x1, policy::SPLIT_K_TARGET_TILES_A8_TILE16X32)- },- GemmTiling::Tile32x64x256_Simdgroups2x2 => {- (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8_TILE32_W4)- },- GemmTiling::Tile128x128x256_Simdgroups4x4 => {- (GemmTiling::Tile128x128x256_Simdgroups4x4, policy::SPLIT_K_TARGET_TILES_A8)- },- _ => (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8),- };+ // The M16 wide finalist favored a 32x64 tile and a 256-workgroup target.+ let (tiling, target_workgroups) =+ if m == 16 && n >= 4096 && (2048..=5120).contains(&k) && k.is_multiple_of(TRELLIS_K_STEP) {+ (GemmTiling::Tile32x64x256_Simdgroups2x2, 256)+ } else {+ match policy::mxu_mn_tile(true, m, n) {+ GemmTiling::Tile16x32x256_Simdgroups1x1 => {+ (GemmTiling::Tile16x32x256_Simdgroups1x1, policy::SPLIT_K_TARGET_TILES_A8_TILE16X32)+ },+ GemmTiling::Tile32x64x256_Simdgroups2x2 => {+ (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8_TILE32_W4)+ },+ GemmTiling::Tile128x128x256_Simdgroups4x4 => {+ (GemmTiling::Tile128x128x256_Simdgroups4x4, policy::SPLIT_K_TARGET_TILES_A8)+ },+ _ => (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8),+ }+ };let output_workgroups = (n.div_ceil(tiling.block_n()) * m.div_ceil(tiling.block_m())).max(1);let k_steps = (k / TRELLIS_K_STEP).max(1);let mut split_k = (target_workgroups / output_workgroups).clamp(1, k_steps);diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex 71a8e68d2..9474db934 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -155,6 +155,12 @@ fn trellis_plan_matches_projection_cases() {for (m, n, k, tiling, split_k) in [(16, 128, 64, Tile16x32x256_Simdgroups1x1, 1),+ (16, 4096, 2048, Tile32x64x256_Simdgroups2x2, 4),+ (16, 5120, 5120, Tile32x64x256_Simdgroups2x2, 2),+ (16, 5120, 5184, Tile16x32x256_Simdgroups1x1, 3),+ (16, 4096, 1984, Tile16x32x256_Simdgroups1x1, 1),+ (16, 32, 5120, Tile64x64x256_Simdgroups2x2, 80),+ (17, 12_288, 2048, Tile64x64x256_Simdgroups2x2, 2),(17, 80, 64, Tile64x64x256_Simdgroups2x2, 1),(2048, 2048, 128, Tile128x128x256_Simdgroups4x4, 1),(1, 80, 5120, Tile16x32x256_Simdgroups1x1, 80),diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rsindex 1f4acba36..e3654dc8b 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs@@ -8,10 +8,15 @@ use crate::{common::{Backend, Context, Kernels,kernel::matmul::{- MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput, TrellisFormat,+ MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput, MatmulShape, TrellisFormat,},},- metal::{Metal, context::MetalContext},+ metal::{+ Metal,+ context::MetalContext,+ gpu_types::gemm::GemmTiling,+ kernel::matmul::{GemmKernel, MatmulOutputWork, gemm::GemmEngine},+ },},data_type::DataType,tests::{@@ -81,13 +86,22 @@ const PADDING: usize = 24;const SENTINEL: u16 = 0x7F7F;const RELATIVE_TOLERANCE: f32 = 1e-2;// Cases are ordered as (m, n, k).-const CASES: [(usize, usize, usize); 7] =- [(1, 80, 5120), (17, 80, 5120), (512, 80, 5120), (16, 128, 64), (32, 128, 64), (1, 6, 64), (2048, 2048, 128)];+const CASES: [(usize, usize, usize); 8] = [+ (1, 80, 5120),+ (17, 80, 5120),+ (512, 80, 5120),+ (16, 4096, 2048),+ (16, 128, 64),+ (32, 128, 64),+ (1, 6, 64),+ (2048, 2048, 128),+];fn run_projection(context: &MetalContext,format: TrellisFormat,dimensions: (usize, usize, usize),+ forced_plan: Option<(GemmTiling, u32)>,) {let (m, n, k) = dimensions;let mut rng = SmallRng::seed_from_u64((m * k + n) as u64);@@ -98,6 +112,15 @@ fn run_projection(// Exercise the zero-activation path in the epilogue.values[k..2 * k].fill(0);}+ if m > 3 {+ for (column, value) in values[3 * k..4 * k].iter_mut().enumerate() {+ *value = if column % 2 == 0 {+ -128+ } else {+ 127+ };+ }+ }let group_sums_host: Vec<f32> = values.chunks_exact(k).flat_map(|row| {@@ -136,10 +159,20 @@ fn run_projection(n: n as u32,k: k as u32,};- let mut matmul =- <MetalMatmul as MatmulKernel>::new(context, DataType::BF16, DataType::BF16, DataType::BF16).unwrap();let mut command_buffer = context.create_command_buffer(None, None).unwrap();- matmul.encode(arguments, &mut command_buffer).unwrap();+ if let Some((tiling, split_k)) = forced_plan {+ let shape = MatmulShape::from_arguments(&arguments);+ let mut gemm = GemmKernel::new(DataType::BF16, DataType::BF16, DataType::BF16);+ let mut plan = gemm.select_plan_for_engine(&shape, GemmEngine::Mxu, context).unwrap();+ plan.tiling = tiling;+ plan.split_k = split_k;+ let output_work = MatmulOutputWork::new(context, DataType::BF16, DataType::BF16).unwrap();+ gemm.encode_plan(arguments, plan, &output_work, &mut command_buffer).unwrap();+ } else {+ let mut matmul =+ <MetalMatmul as MatmulKernel>::new(context, DataType::BF16, DataType::BF16, DataType::BF16).unwrap();+ matmul.encode(arguments, &mut command_buffer).unwrap();+ }submit_command_buffer(command_buffer);let decoded_levels: Vec<i32> = stored.chunks_exact(row_bytes(format, k))@@ -181,7 +214,10 @@ fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {if !context.supports_mxu {return;}- for (m, n, k) in CASES {- run_projection(&context, format, (m, n, k));+ for dimensions in CASES {+ run_projection(&context, format, dimensions, None);}+ run_projection(&context, format, (129, 132, 1024), Some((GemmTiling::Tile128x128x256_Simdgroups4x4, 1)));+ run_projection(&context, format, (129, 132, 64), Some((GemmTiling::Tile128x128x256_Simdgroups4x4, 1)));+ run_projection(&context, format, (17, 80, 192), Some((GemmTiling::Tile32x64x256_Simdgroups2x2, 1)));}