A high-performance inference engine for AI models
adjust tiling for non-8 aligned batch sizes
eugene968a25e245ee
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 2700cad70..eb69e31b5 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -21,6 +21,7 @@ use crate::{metal::{Metal,context::MetalContext,+ device_profile::DeviceProfile,error::MetalError,kernel::{GemmMetalKernel, GemmSplitKReduceMetalKernel, TensorAddBiasMetalKernel},},@@ -109,8 +110,9 @@ impl GemmKernel {&self,shape: MatmulShape,supports_mxu: bool,+ profile: DeviceProfile,) -> GemmProblem {- GemmProblem::new(shape, self.weights_data_type, self.output_data_type, supports_mxu)+ GemmProblem::new(shape, self.weights_data_type, self.output_data_type, supports_mxu, profile)}#[cfg(test)]@@ -120,7 +122,7 @@ impl GemmKernel {engine: GemmEngine,context: &MetalContext,) -> Result<GemmPlan, MetalError> {- self.problem(*shape, context.supports_mxu())+ self.problem(*shape, context.supports_mxu(), context.device_profile()).select_plan_for_engine(engine).map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))}@@ -144,7 +146,7 @@ impl GemmKernel {encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {let shape = MatmulShape::from_arguments(&arguments);- self.problem(shape, encoder.context().supports_mxu())+ self.problem(shape, encoder.context().supports_mxu(), encoder.context().device_profile()).validate_engine(plan.engine).map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))?;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/policy.rsindex 45ee8aa2f..31b855317 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/policy.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/policy.rs@@ -4,7 +4,10 @@//! features to choices. Eligibility, fallbacks, and legality stay in//! `selection.rs`.-use crate::backends::common::gpu_types::gemm::GemmTiling;+use crate::backends::{+ common::gpu_types::gemm::GemmTiling,+ metal::device_profile::{DeviceGeneration, DeviceProfile},+};pub(super) const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;@@ -20,6 +23,7 @@ const MXU_M_BUCKET_MAXES: [u32; 4] = [16, 63, 255, 511];const MXU_N_BUCKET_MAXES: [u32; 2] = [63, 127];const SIMDGROUP_QUANT_SMALL_M_MAX: u32 = 32;+const SIMDGROUP_QUANT_NARROW_BLOCK_M: u32 = 8;const SIMDGROUP_QUANT_LARGE_M_MIN: u32 = 64;const SIMDGROUP_QUANT_WIDE_N_MIN: u32 = 6144;@@ -95,15 +99,29 @@ pub(super) fn simdgroup_fp_tile(}}+/// A partial trailing M block costs a second pass over the weights. Older GPUs are bound by that;+/// Apple9 and newer would rather keep the narrow tile's parallelism.+fn prefers_wide_partial_m_tile(profile: DeviceProfile) -> bool {+ matches!(profile.generation(), DeviceGeneration::Legacy | DeviceGeneration::Apple8)+}+pub(super) fn simdgroup_quant_tile(m: u32,n: u32,group_size: u32,+ profile: DeviceProfile,) -> GemmTiling {if group_size < 32 {GemmTiling::Tile64x64x16_Simdgroups2x2} else if m < SIMDGROUP_QUANT_SMALL_M_MAX {- GemmTiling::Tile8x32x32_Simdgroups1x1+ if !prefers_wide_partial_m_tile(profile)+ || m <= SIMDGROUP_QUANT_NARROW_BLOCK_M+ || m.is_multiple_of(SIMDGROUP_QUANT_NARROW_BLOCK_M)+ {+ GemmTiling::Tile8x32x32_Simdgroups1x1+ } else {+ GemmTiling::Tile32x32x32_Simdgroups2x2+ }} else if m >= SIMDGROUP_QUANT_LARGE_M_MIN && n >= SIMDGROUP_QUANT_WIDE_N_MIN && n.is_multiple_of(64) {GemmTiling::Tile64x64x32_Simdgroups2x2} else {diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/selection.rsindex 8c341ac44..a28f92136 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -2,9 +2,12 @@ use thiserror::Error;use super::{GemmEngine, GemmPlan, policy};use crate::{- backends::common::{- gpu_types::gemm::{GemmBPrologueKind, GemmTiling},- kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},+ backends::{+ common::{+ gpu_types::gemm::{GemmBPrologueKind, GemmTiling},+ kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},+ },+ metal::device_profile::DeviceProfile,},data_type::DataType,};@@ -15,6 +18,7 @@ pub struct GemmProblem {weights_data_type: DataType,output_data_type: DataType,supports_mxu: bool,+ profile: DeviceProfile,}#[derive(Debug, Clone, Copy, PartialEq, Error)]@@ -31,12 +35,14 @@ impl GemmProblem {weights_data_type: DataType,output_data_type: DataType,supports_mxu: bool,+ profile: DeviceProfile,) -> Self {Self {shape,weights_data_type,output_data_type,supports_mxu,+ profile,}}@@ -46,7 +52,7 @@ impl GemmProblem {} else {GemmEngine::Simdgroup};- self.finish_plan(engine, select_tiling(self.shape, engine))+ self.finish_plan(engine, select_tiling(self.shape, engine, self.profile))}#[cfg(test)]@@ -55,7 +61,7 @@ impl GemmProblem {engine: GemmEngine,) -> Result<GemmPlan, GemmPlanError> {self.validate_engine(engine)?;- Ok(self.finish_plan(engine, select_tiling(self.shape, engine)))+ Ok(self.finish_plan(engine, select_tiling(self.shape, engine, self.profile)))}pub(super) fn validate_engine(@@ -160,10 +166,11 @@ fn mxu_is_eligible(shape: MatmulShape) -> bool {fn select_tiling(shape: MatmulShape,engine: GemmEngine,+ profile: DeviceProfile,) -> GemmTiling {match engine {GemmEngine::Simdgroup if shape.is_quant() => {- policy::simdgroup_quant_tile(shape.m, shape.n, shape.b_group_size.unwrap_or(0))+ policy::simdgroup_quant_tile(shape.m, shape.n, shape.b_group_size.unwrap_or(0), profile)},GemmEngine::Simdgroup => policy::simdgroup_fp_tile(shape.m, shape.n, shape.k),GemmEngine::Mxu if !shape.a_full_precision || shape.is_quant() => select_mxu_quant_tiling(shape),diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex c07e956eb..e90e9346c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -76,7 +76,13 @@ impl MatmulMetalKernel {self.output_data_type,context.device_profile(),);- let problem = GemmProblem::new(*shape, self.weights_data_type, self.output_data_type, context.supports_mxu());+ let problem = GemmProblem::new(+ *shape,+ self.weights_data_type,+ self.output_data_type,+ context.supports_mxu(),+ context.device_profile(),+ );let plan = problem.select_plan();match gemv {None => MatmulDispatch::Gemm(plan),diff --git a/crates/backend-uzu/src/tests/matmul/shape.rs b/crates/backend-uzu/src/tests/matmul/shape.rsindex 0c7a7e866..9ee092cf1 100644--- a/crates/backend-uzu/src/tests/matmul/shape.rs+++ b/crates/backend-uzu/src/tests/matmul/shape.rs@@ -58,7 +58,8 @@ pub fn bench_quant_gemm_shapes(bits: u32) -> impl Iterator<Item = Shape> {} else {256};- let ms = &[4u32, 5, 6, 7, 8, 16, 32, 48, 64];+ // The tile policy only differs for m that is not a multiple of 8, so keep 4..16 contiguous.+ let ms = &[4u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 32, 48, 64];BENCH_NK.iter().filter(move |&&(n, k)| n % 32 == 0 && k % block_size == 0)diff --git a/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex ed59d14d3..10d0884d4 100644--- a/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -3,9 +3,15 @@ use proc_macros::uzu_test;use super::{super::specialization::GemmSpecialization, *};use crate::backends::{common::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmDTransform},- metal::kernel::matmul::MatmulMetalKernel,+ metal::{+ device_profile::{DeviceGeneration, DeviceProfile},+ kernel::matmul::MatmulMetalKernel,+ },};+const LEGACY_PROFILE: DeviceProfile = DeviceProfile::new(8, DeviceGeneration::Legacy);+const APPLE9_PROFILE: DeviceProfile = DeviceProfile::new(10, DeviceGeneration::Apple9);+fn shape(m: u32,n: u32,@@ -39,7 +45,7 @@ fn problem(shape: MatmulShape,output_data_type: DataType,) -> GemmProblem {- GemmProblem::new(shape, DataType::BF16, output_data_type, true)+ GemmProblem::new(shape, DataType::BF16, output_data_type, true, LEGACY_PROFILE)}fn plan(split_k: u32) -> GemmPlan {@@ -78,11 +84,24 @@ fn policy_boundaries_are_preserved() {for (m, n, group_size, expected) in [(16, 4096, 31, Tile64x64x16_Simdgroups2x2),- (31, 4096, 32, Tile8x32x32_Simdgroups1x1),+ // Full M blocks keep the narrow tile.+ (4, 4096, 32, Tile8x32x32_Simdgroups1x1),+ (8, 4096, 32, Tile8x32x32_Simdgroups1x1),+ (16, 4096, 32, Tile8x32x32_Simdgroups1x1),+ (24, 4096, 32, Tile8x32x32_Simdgroups1x1),+ // A partial trailing block takes the wide tile.+ (9, 4096, 32, Tile32x32x32_Simdgroups2x2),+ (15, 4096, 32, Tile32x32x32_Simdgroups2x2),+ (31, 4096, 32, Tile32x32x32_Simdgroups2x2),(64, 6143, 32, Tile32x32x32_Simdgroups2x2),(64, 6144, 32, Tile64x64x32_Simdgroups2x2),] {- assert_eq!(policy::simdgroup_quant_tile(m, n, group_size), expected);+ assert_eq!(policy::simdgroup_quant_tile(m, n, group_size, LEGACY_PROFILE), expected);+ }++ // Apple9 and newer keep the narrow tile regardless.+ for m in [9, 15, 31] {+ assert_eq!(policy::simdgroup_quant_tile(m, 4096, 32, APPLE9_PROFILE), Tile8x32x32_Simdgroups1x1);}}@@ -110,7 +129,7 @@ fn selection_fallbacks_and_split_k_are_preserved() {let mut biased = quant(shape(16, 4096, 4096));biased.a_full_precision = false;biased.d_transform = GemmDTransform::BIAS;- assert_eq!(GemmProblem::new(biased, DataType::BF16, DataType::F32, true).select_plan().split_k, 1);+ assert_eq!(GemmProblem::new(biased, DataType::BF16, DataType::F32, true, LEGACY_PROFILE).select_plan().split_k, 1);let mut zero = quant(shape(0, 1, 1));zero.b_prologue = GemmBPrologueKind::ScaleZeroPointDequant;@@ -122,7 +141,8 @@ fn selection_fallbacks_and_split_k_are_preserved() {fn forced_engine_errors_are_preserved() {let huge = shape(u32::MAX, u32::MAX, u32::MAX);assert_eq!(- GemmProblem::new(huge, DataType::BF16, DataType::BF16, false).select_plan_for_engine(GemmEngine::Mxu),+ GemmProblem::new(huge, DataType::BF16, DataType::BF16, false, LEGACY_PROFILE)+ .select_plan_for_engine(GemmEngine::Mxu),Err(GemmPlanError::MxuUnavailable));@@ -167,7 +187,7 @@ fn gemv_gemm_route_boundaries_are_preserved() {(shape(3, 4096, 8192), DataType::BF16, false),(shape(5, 4096, 8192), DataType::BF16, false),] {- let plan = GemmProblem::new(shape, data_type, data_type, true).select_plan();+ let plan = GemmProblem::new(shape, data_type, data_type, true, LEGACY_PROFILE).select_plan();assert_eq!(MatmulMetalKernel::prefer_gemm_over_gemv(shape, plan, data_type, data_type, data_type), prefer_gemm);}let mut gathered = shape(4, 4096, 8192);