Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

adjust tiling for non-8 aligned batch sizes

eugene968a25e245ee
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 2700cad70..eb69e31b5 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -21,6 +21,7 @@ use crate::{
metal::{
Metal,
context::MetalContext,
+ device_profile::DeviceProfile,
error::MetalError,
kernel::{GemmMetalKernel, GemmSplitKReduceMetalKernel, TensorAddBiasMetalKernel},
},
@@ -109,8 +110,9 @@ impl GemmKernel {
&self,
shape: MatmulShape,
supports_mxu: bool,
+ profile: DeviceProfile,
) -> GemmProblem {
- GemmProblem::new(shape, self.weights_data_type, self.output_data_type, supports_mxu)
+ GemmProblem::new(shape, self.weights_data_type, self.output_data_type, supports_mxu, profile)
}
#[cfg(test)]
@@ -120,7 +122,7 @@ impl GemmKernel {
engine: GemmEngine,
context: &MetalContext,
) -> Result<GemmPlan, MetalError> {
- self.problem(*shape, context.supports_mxu())
+ self.problem(*shape, context.supports_mxu(), context.device_profile())
.select_plan_for_engine(engine)
.map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))
}
@@ -144,7 +146,7 @@ impl GemmKernel {
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
let shape = MatmulShape::from_arguments(&arguments);
- self.problem(shape, encoder.context().supports_mxu())
+ self.problem(shape, encoder.context().supports_mxu(), encoder.context().device_profile())
.validate_engine(plan.engine)
.map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))?;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/policy.rs
index 45ee8aa2f..31b855317 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/policy.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/policy.rs
@@ -4,7 +4,10 @@
//! features to choices. Eligibility, fallbacks, and legality stay in
//! `selection.rs`.
-use crate::backends::common::gpu_types::gemm::GemmTiling;
+use crate::backends::{
+ common::gpu_types::gemm::GemmTiling,
+ metal::device_profile::{DeviceGeneration, DeviceProfile},
+};
pub(super) const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;
@@ -20,6 +23,7 @@ const MXU_M_BUCKET_MAXES: [u32; 4] = [16, 63, 255, 511];
const MXU_N_BUCKET_MAXES: [u32; 2] = [63, 127];
const SIMDGROUP_QUANT_SMALL_M_MAX: u32 = 32;
+const SIMDGROUP_QUANT_NARROW_BLOCK_M: u32 = 8;
const SIMDGROUP_QUANT_LARGE_M_MIN: u32 = 64;
const SIMDGROUP_QUANT_WIDE_N_MIN: u32 = 6144;
@@ -95,15 +99,29 @@ pub(super) fn simdgroup_fp_tile(
}
}
+/// A partial trailing M block costs a second pass over the weights. Older GPUs are bound by that;
+/// Apple9 and newer would rather keep the narrow tile's parallelism.
+fn prefers_wide_partial_m_tile(profile: DeviceProfile) -> bool {
+ matches!(profile.generation(), DeviceGeneration::Legacy | DeviceGeneration::Apple8)
+}
+
pub(super) fn simdgroup_quant_tile(
m: u32,
n: u32,
group_size: u32,
+ profile: DeviceProfile,
) -> GemmTiling {
if group_size < 32 {
GemmTiling::Tile64x64x16_Simdgroups2x2
} else if m < SIMDGROUP_QUANT_SMALL_M_MAX {
- GemmTiling::Tile8x32x32_Simdgroups1x1
+ if !prefers_wide_partial_m_tile(profile)
+ || m <= SIMDGROUP_QUANT_NARROW_BLOCK_M
+ || m.is_multiple_of(SIMDGROUP_QUANT_NARROW_BLOCK_M)
+ {
+ GemmTiling::Tile8x32x32_Simdgroups1x1
+ } else {
+ GemmTiling::Tile32x32x32_Simdgroups2x2
+ }
} else if m >= SIMDGROUP_QUANT_LARGE_M_MIN && n >= SIMDGROUP_QUANT_WIDE_N_MIN && n.is_multiple_of(64) {
GemmTiling::Tile64x64x32_Simdgroups2x2
} else {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/selection.rs
index 8c341ac44..a28f92136 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -2,9 +2,12 @@ use thiserror::Error;
use super::{GemmEngine, GemmPlan, policy};
use crate::{
- backends::common::{
- gpu_types::gemm::{GemmBPrologueKind, GemmTiling},
- kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},
+ backends::{
+ common::{
+ gpu_types::gemm::{GemmBPrologueKind, GemmTiling},
+ kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},
+ },
+ metal::device_profile::DeviceProfile,
},
data_type::DataType,
};
@@ -15,6 +18,7 @@ pub struct GemmProblem {
weights_data_type: DataType,
output_data_type: DataType,
supports_mxu: bool,
+ profile: DeviceProfile,
}
#[derive(Debug, Clone, Copy, PartialEq, Error)]
@@ -31,12 +35,14 @@ impl GemmProblem {
weights_data_type: DataType,
output_data_type: DataType,
supports_mxu: bool,
+ profile: DeviceProfile,
) -> Self {
Self {
shape,
weights_data_type,
output_data_type,
supports_mxu,
+ profile,
}
}
@@ -46,7 +52,7 @@ impl GemmProblem {
} else {
GemmEngine::Simdgroup
};
- self.finish_plan(engine, select_tiling(self.shape, engine))
+ self.finish_plan(engine, select_tiling(self.shape, engine, self.profile))
}
#[cfg(test)]
@@ -55,7 +61,7 @@ impl GemmProblem {
engine: GemmEngine,
) -> Result<GemmPlan, GemmPlanError> {
self.validate_engine(engine)?;
- Ok(self.finish_plan(engine, select_tiling(self.shape, engine)))
+ Ok(self.finish_plan(engine, select_tiling(self.shape, engine, self.profile)))
}
pub(super) fn validate_engine(
@@ -160,10 +166,11 @@ fn mxu_is_eligible(shape: MatmulShape) -> bool {
fn select_tiling(
shape: MatmulShape,
engine: GemmEngine,
+ profile: DeviceProfile,
) -> GemmTiling {
match engine {
GemmEngine::Simdgroup if shape.is_quant() => {
- policy::simdgroup_quant_tile(shape.m, shape.n, shape.b_group_size.unwrap_or(0))
+ policy::simdgroup_quant_tile(shape.m, shape.n, shape.b_group_size.unwrap_or(0), profile)
},
GemmEngine::Simdgroup => policy::simdgroup_fp_tile(shape.m, shape.n, shape.k),
GemmEngine::Mxu if !shape.a_full_precision || shape.is_quant() => select_mxu_quant_tiling(shape),
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index c07e956eb..e90e9346c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -76,7 +76,13 @@ impl MatmulMetalKernel {
self.output_data_type,
context.device_profile(),
);
- let problem = GemmProblem::new(*shape, self.weights_data_type, self.output_data_type, context.supports_mxu());
+ let problem = GemmProblem::new(
+ *shape,
+ self.weights_data_type,
+ self.output_data_type,
+ context.supports_mxu(),
+ context.device_profile(),
+ );
let plan = problem.select_plan();
match gemv {
None => MatmulDispatch::Gemm(plan),
diff --git a/crates/backend-uzu/src/tests/matmul/shape.rs b/crates/backend-uzu/src/tests/matmul/shape.rs
index 0c7a7e866..9ee092cf1 100644
--- a/crates/backend-uzu/src/tests/matmul/shape.rs
+++ b/crates/backend-uzu/src/tests/matmul/shape.rs
@@ -58,7 +58,8 @@ pub fn bench_quant_gemm_shapes(bits: u32) -> impl Iterator<Item = Shape> {
} else {
256
};
- let ms = &[4u32, 5, 6, 7, 8, 16, 32, 48, 64];
+ // The tile policy only differs for m that is not a multiple of 8, so keep 4..16 contiguous.
+ let ms = &[4u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 32, 48, 64];
BENCH_NK
.iter()
.filter(move |&&(n, k)| n % 32 == 0 && k % block_size == 0)
diff --git a/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index ed59d14d3..10d0884d4 100644
--- a/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -3,9 +3,15 @@ use proc_macros::uzu_test;
use super::{super::specialization::GemmSpecialization, *};
use crate::backends::{
common::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmDTransform},
- metal::kernel::matmul::MatmulMetalKernel,
+ metal::{
+ device_profile::{DeviceGeneration, DeviceProfile},
+ kernel::matmul::MatmulMetalKernel,
+ },
};
+const LEGACY_PROFILE: DeviceProfile = DeviceProfile::new(8, DeviceGeneration::Legacy);
+const APPLE9_PROFILE: DeviceProfile = DeviceProfile::new(10, DeviceGeneration::Apple9);
+
fn shape(
m: u32,
n: u32,
@@ -39,7 +45,7 @@ fn problem(
shape: MatmulShape,
output_data_type: DataType,
) -> GemmProblem {
- GemmProblem::new(shape, DataType::BF16, output_data_type, true)
+ GemmProblem::new(shape, DataType::BF16, output_data_type, true, LEGACY_PROFILE)
}
fn plan(split_k: u32) -> GemmPlan {
@@ -78,11 +84,24 @@ fn policy_boundaries_are_preserved() {
for (m, n, group_size, expected) in [
(16, 4096, 31, Tile64x64x16_Simdgroups2x2),
- (31, 4096, 32, Tile8x32x32_Simdgroups1x1),
+ // Full M blocks keep the narrow tile.
+ (4, 4096, 32, Tile8x32x32_Simdgroups1x1),
+ (8, 4096, 32, Tile8x32x32_Simdgroups1x1),
+ (16, 4096, 32, Tile8x32x32_Simdgroups1x1),
+ (24, 4096, 32, Tile8x32x32_Simdgroups1x1),
+ // A partial trailing block takes the wide tile.
+ (9, 4096, 32, Tile32x32x32_Simdgroups2x2),
+ (15, 4096, 32, Tile32x32x32_Simdgroups2x2),
+ (31, 4096, 32, Tile32x32x32_Simdgroups2x2),
(64, 6143, 32, Tile32x32x32_Simdgroups2x2),
(64, 6144, 32, Tile64x64x32_Simdgroups2x2),
] {
- assert_eq!(policy::simdgroup_quant_tile(m, n, group_size), expected);
+ assert_eq!(policy::simdgroup_quant_tile(m, n, group_size, LEGACY_PROFILE), expected);
+ }
+
+ // Apple9 and newer keep the narrow tile regardless.
+ for m in [9, 15, 31] {
+ assert_eq!(policy::simdgroup_quant_tile(m, 4096, 32, APPLE9_PROFILE), Tile8x32x32_Simdgroups1x1);
}
}
@@ -110,7 +129,7 @@ fn selection_fallbacks_and_split_k_are_preserved() {
let mut biased = quant(shape(16, 4096, 4096));
biased.a_full_precision = false;
biased.d_transform = GemmDTransform::BIAS;
- assert_eq!(GemmProblem::new(biased, DataType::BF16, DataType::F32, true).select_plan().split_k, 1);
+ assert_eq!(GemmProblem::new(biased, DataType::BF16, DataType::F32, true, LEGACY_PROFILE).select_plan().split_k, 1);
let mut zero = quant(shape(0, 1, 1));
zero.b_prologue = GemmBPrologueKind::ScaleZeroPointDequant;
@@ -122,7 +141,8 @@ fn selection_fallbacks_and_split_k_are_preserved() {
fn forced_engine_errors_are_preserved() {
let huge = shape(u32::MAX, u32::MAX, u32::MAX);
assert_eq!(
- GemmProblem::new(huge, DataType::BF16, DataType::BF16, false).select_plan_for_engine(GemmEngine::Mxu),
+ GemmProblem::new(huge, DataType::BF16, DataType::BF16, false, LEGACY_PROFILE)
+ .select_plan_for_engine(GemmEngine::Mxu),
Err(GemmPlanError::MxuUnavailable)
);
@@ -167,7 +187,7 @@ fn gemv_gemm_route_boundaries_are_preserved() {
(shape(3, 4096, 8192), DataType::BF16, false),
(shape(5, 4096, 8192), DataType::BF16, false),
] {
- let plan = GemmProblem::new(shape, data_type, data_type, true).select_plan();
+ let plan = GemmProblem::new(shape, data_type, data_type, true, LEGACY_PROFILE).select_plan();
assert_eq!(MatmulMetalKernel::prefer_gemm_over_gemv(shape, plan, data_type, data_type, data_type), prefer_gemm);
}
let mut gathered = shape(4, 4096, 8192);