Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

fix preheat

eugene2a8ef0255e7d
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
index 451a89f43..8318cbb04 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
@@ -26,6 +26,8 @@ pub trait MatmulKernel: Sized {
&mut self,
_context: &<Self::Backend as Backend>::Context,
_combo: MatmulQuantCombo,
+ _output_dim: u32,
+ _input_dim: u32,
) -> Result<(), <Self::Backend as Backend>::Error> {
Ok(())
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index d042df1c2..0c2ff6500 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -73,8 +73,15 @@ impl GemmKernel {
&mut self,
context: &MetalContext,
combo: MatmulQuantCombo,
+ n: u32,
+ k: u32,
) -> Result<(), MetalError> {
- for specialization in GemmSpecialization::quant_combo_specs(self.weights_data_type, combo) {
+ let all_float = [self.weights_data_type, self.input_data_type, self.output_data_type]
+ .into_iter()
+ .all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32));
+ let use_mxu = context.device.supports_mxu() && all_float && k.is_multiple_of(select_mxu_tiling(n, n).block_k());
+
+ for specialization in GemmSpecialization::quant_combo_specs(self.weights_data_type, combo, n, k, use_mxu) {
self.get_or_create(context, specialization)?;
}
Ok(())
@@ -668,7 +675,7 @@ fn select_mxu_tiling(
}
}
-fn select_mxu_quant_tiling(
+pub(crate) fn select_mxu_quant_tiling(
m: u32,
n: u32,
group_size: u32,
@@ -684,7 +691,7 @@ fn select_mxu_quant_tiling(
}
}
-fn select_quant_tiling(
+pub(crate) fn select_quant_tiling(
m: u32,
n: u32,
group_size: u32,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 82869aa87..aae3b1b99 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -1,4 +1,7 @@
-use super::error::GemmSpecializationError;
+use super::{
+ error::GemmSpecializationError,
+ kernel::{select_mxu_quant_tiling, select_quant_tiling},
+};
use crate::{
backends::common::{
gpu_types::{
@@ -10,6 +13,8 @@ use crate::{
data_type::DataType,
};
+const QUANT_PREHEAT_PROBE_MS: [u32; 4] = [4096, 128, 48, 8];
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(crate) struct GemmSpecialization {
pub(crate) weights_data_type: DataType,
@@ -130,6 +135,9 @@ impl GemmSpecialization {
pub(crate) fn quant_combo_specs(
weights_data_type: DataType,
combo: MatmulQuantCombo,
+ n: u32,
+ k: u32,
+ use_mxu: bool,
) -> Vec<Self> {
let bits = DataType::from(combo.mode).size_in_bits() as u32;
let group_size = combo.group_size;
@@ -138,60 +146,38 @@ impl GemmSpecialization {
QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,
QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,
};
- let mut out = Vec::new();
- for &tiling in quant_tiling_set(weights_data_type) {
- if tiling.simdgroup_block_k() > group_size {
- continue;
+
+ let mut tilings: Vec<GemmTiling> = Vec::new();
+ for &m in &QUANT_PREHEAT_PROBE_MS {
+ let tiling = if use_mxu {
+ select_mxu_quant_tiling(m, n, group_size)
+ } else {
+ select_quant_tiling(m, n, group_size)
+ };
+ if !tilings.contains(&tiling) {
+ tilings.push(tiling);
}
- for align_n in [true, false] {
- for output_transform in [
- GemmDTransform::empty(),
- GemmDTransform::BIAS,
- GemmDTransform::RHT,
- GemmDTransform::BIAS | GemmDTransform::RHT,
- ] {
- out.push(Self {
+ }
+
+ let mut out = Vec::new();
+ for tiling in tilings {
+ let align_n = n.is_multiple_of(tiling.block_n());
+ let align_k = k.is_multiple_of(tiling.block_k());
+ for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS, GemmDTransform::RHT] {
+ for align_m in [true, false] {
+ let spec = Self {
weights_data_type,
tiling,
- use_mxu: false,
+ use_mxu,
output_transform,
- alignment: GemmAlignment::new(true, align_n, true),
+ alignment: GemmAlignment::new(align_m, align_n, align_k),
transpose_b: true,
b_prologue,
bits_per_b: Some(bits),
group_size: Some(group_size),
- });
- }
- }
- }
- for &tiling in mxu_tiling_set(weights_data_type) {
- if !group_size.is_multiple_of(32) {
- continue;
- }
- if !tiling.fits_quant_group_size(group_size) {
- continue;
- }
- for align_m in [true, false] {
- for align_n in [true, false] {
- for output_transform in [
- GemmDTransform::empty(),
- GemmDTransform::SCALE,
- GemmDTransform::BIAS,
- GemmDTransform::RHT,
- GemmDTransform::BIAS | GemmDTransform::RHT,
- GemmDTransform::BIAS | GemmDTransform::SCALE,
- ] {
- out.push(Self {
- weights_data_type,
- tiling,
- use_mxu: true,
- output_transform,
- alignment: GemmAlignment::new(align_m, align_n, true),
- transpose_b: true,
- b_prologue,
- bits_per_b: Some(bits),
- group_size: Some(group_size),
- });
+ };
+ if spec.validate().is_ok() {
+ out.push(spec);
}
}
}
@@ -221,14 +207,45 @@ fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
]
}
-pub(crate) fn quant_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
- match data_type {
- DataType::BF16 => &[
- GemmTiling::Tile8x32x32_Simdgroups1x1,
- GemmTiling::Tile32x32x32_Simdgroups2x2,
- GemmTiling::Tile64x32x32_Simdgroups2x2,
- GemmTiling::Tile64x64x32_Simdgroups2x2,
- ],
- _ => &[],
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::backends::common::gpu_types::QuantizationMode;
+
+ fn combo() -> MatmulQuantCombo {
+ MatmulQuantCombo {
+ method: QuantizationMethod::ScaleZeroPoint,
+ mode: QuantizationMode::U4,
+ group_size: 32,
+ }
+ }
+
+ #[test]
+ fn quant_specs_are_shape_specific_and_tight() {
+ let specs = GemmSpecialization::quant_combo_specs(DataType::BF16, combo(), 3584, 1024, false);
+ assert!(!specs.is_empty());
+ assert!(specs.len() <= 16, "expected a tight preheat set, got {}", specs.len());
+ for spec in &specs {
+ assert!(!spec.use_mxu);
+ assert_eq!(spec.bits_per_b, Some(4));
+ assert_eq!(spec.group_size, Some(32));
+ assert_ne!(spec.b_prologue, GemmBPrologueKind::FullPrecision);
+ assert!(spec.validate().is_ok());
+ }
+ let tilings: std::collections::HashSet<_> = specs.iter().map(|s| s.tiling).collect();
+ assert!(tilings.contains(&GemmTiling::Tile32x32x32_Simdgroups2x2));
+ assert!(tilings.contains(&GemmTiling::Tile8x32x32_Simdgroups1x1));
+ }
+
+ #[test]
+ fn mxu_path_emits_only_mxu_tilings() {
+ let specs = GemmSpecialization::quant_combo_specs(DataType::BF16, combo(), 3584, 1024, true);
+ assert!(!specs.is_empty());
+ assert!(specs.len() <= 20, "expected a tight preheat set, got {}", specs.len());
+ for spec in &specs {
+ assert!(spec.use_mxu);
+ assert!(spec.tiling.is_mxu_variant());
+ assert!(spec.validate().is_ok());
+ }
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 11b6f572b..1643c7fde 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -105,7 +105,10 @@ impl GemvSpecialization {
})
}
- fn quant_combo_specs(combo: MatmulQuantCombo) -> Vec<GemvSpecialization> {
+ fn quant_combo_specs(
+ combo: MatmulQuantCombo,
+ k: u32,
+ ) -> Vec<GemvSpecialization> {
let bits = DataType::from(combo.mode).size_in_bits() as u32;
let group_size = combo.group_size;
let b_prologue = match combo.method {
@@ -113,26 +116,24 @@ impl GemvSpecialization {
QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,
QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,
};
- let mut out = Vec::new();
- for output_transform in [
- GemmDTransform::empty(),
- GemmDTransform::BIAS,
- GemmDTransform::RHT,
- GemmDTransform::BIAS | GemmDTransform::RHT,
- ] {
- for input_aligned in [true, false] {
- out.push(GemvSpecialization {
- b_prologue,
- group_size,
- bits,
- output_transform,
- input_aligned,
- k_split: 1,
- num_simdgroups: 8,
- });
- }
- }
- out
+ let block_size = if bits == 4 {
+ 512
+ } else {
+ 256
+ };
+ let input_aligned = k.is_multiple_of(block_size);
+ [GemmDTransform::empty(), GemmDTransform::BIAS, GemmDTransform::RHT]
+ .into_iter()
+ .map(|output_transform| GemvSpecialization {
+ b_prologue,
+ group_size,
+ bits,
+ output_transform,
+ input_aligned,
+ k_split: 1,
+ num_simdgroups: 8,
+ })
+ .collect()
}
}
@@ -185,11 +186,12 @@ impl GemvDispatch {
&mut self,
context: &MetalContext,
combo: MatmulQuantCombo,
+ k: u32,
) -> Result<(), MatmulError<Metal>> {
if self.weights_data_type != DataType::BF16 {
return Ok(());
}
- for specialization in GemvSpecialization::quant_combo_specs(combo) {
+ for specialization in GemvSpecialization::quant_combo_specs(combo, k) {
self.get_or_create(context, specialization)?;
}
Ok(())
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index b66dca74d..7c896da22 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -70,8 +70,10 @@ impl MatmulKernel for MatmulMetalKernel {
&mut self,
context: &MetalContext,
combo: MatmulQuantCombo,
+ output_dim: u32,
+ input_dim: u32,
) -> Result<(), MetalError> {
- self.gemv.preheat_quant_combo(context, combo).map_err(MetalError::from)?;
- self.gemm.preheat_quant_combo(context, combo)
+ self.gemv.preheat_quant_combo(context, combo, input_dim).map_err(MetalError::from)?;
+ self.gemm.preheat_quant_combo(context, combo, output_dim, input_dim)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 958a7ad34..452654756 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -190,6 +190,8 @@ impl<B: Backend> Embedding<B> {
embedding_quantization_mode,
quantization_method,
group_size,
+ vocab_size,
+ model_dim,
)?;
(
@@ -254,6 +256,8 @@ impl<B: Backend> Embedding<B> {
embedding_quantization_mode,
quantization_method,
group_size,
+ vocab_size,
+ model_dim,
)?;
(
@@ -385,6 +389,8 @@ impl<B: Backend> Embedding<B> {
embedding_quantization_mode,
quantization_method,
group_size,
+ vocab_size,
+ model_dim,
)?;
UntiedEmbeddingReadoutType::Quantized {
@@ -768,6 +774,8 @@ fn quantized_readout<B: Backend>(
mode: QuantizationMode,
method: QuantizationMethod,
group_size: usize,
+ vocab_size: u32,
+ model_dim: u32,
) -> Result<(RefCell<<B::Kernels as Kernels>::MatmulKernel>, ReadoutQuantConfig), EmbeddingError<B>> {
let mut readout = <B::Kernels as Kernels>::MatmulKernel::new(context, data_type, data_type, data_type)
.map_err(EmbeddingError::BackendError)?;
@@ -779,6 +787,8 @@ fn quantized_readout<B: Backend>(
mode,
group_size: group_size as u32,
},
+ vocab_size,
+ model_dim,
)
.map_err(EmbeddingError::BackendError)?;
Ok((
diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
index ad2a165e9..5c00cc2c6 100644
--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
@@ -181,6 +181,8 @@ impl<B: Backend> LinearMatmul<B> {
mode: weight_quantization_mode,
group_size: group_size as u32,
},
+ output_dim as u32,
+ input_dim as u32,
)
.map_err(LinearMatmulError::BackendError)?;