A high-performance inference engine for AI models
fix preheat
eugene2a8ef0255e7d
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rsindex 451a89f43..8318cbb04 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs@@ -26,6 +26,8 @@ pub trait MatmulKernel: Sized {&mut self,_context: &<Self::Backend as Backend>::Context,_combo: MatmulQuantCombo,+ _output_dim: u32,+ _input_dim: u32,) -> Result<(), <Self::Backend as Backend>::Error> {Ok(())}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex d042df1c2..0c2ff6500 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -73,8 +73,15 @@ impl GemmKernel {&mut self,context: &MetalContext,combo: MatmulQuantCombo,+ n: u32,+ k: u32,) -> Result<(), MetalError> {- for specialization in GemmSpecialization::quant_combo_specs(self.weights_data_type, combo) {+ let all_float = [self.weights_data_type, self.input_data_type, self.output_data_type]+ .into_iter()+ .all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32));+ let use_mxu = context.device.supports_mxu() && all_float && k.is_multiple_of(select_mxu_tiling(n, n).block_k());++ for specialization in GemmSpecialization::quant_combo_specs(self.weights_data_type, combo, n, k, use_mxu) {self.get_or_create(context, specialization)?;}Ok(())@@ -668,7 +675,7 @@ fn select_mxu_tiling(}}-fn select_mxu_quant_tiling(+pub(crate) fn select_mxu_quant_tiling(m: u32,n: u32,group_size: u32,@@ -684,7 +691,7 @@ fn select_mxu_quant_tiling(}}-fn select_quant_tiling(+pub(crate) fn select_quant_tiling(m: u32,n: u32,group_size: u32,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 82869aa87..aae3b1b99 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -1,4 +1,7 @@-use super::error::GemmSpecializationError;+use super::{+ error::GemmSpecializationError,+ kernel::{select_mxu_quant_tiling, select_quant_tiling},+};use crate::{backends::common::{gpu_types::{@@ -10,6 +13,8 @@ use crate::{data_type::DataType,};+const QUANT_PREHEAT_PROBE_MS: [u32; 4] = [4096, 128, 48, 8];+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub(crate) struct GemmSpecialization {pub(crate) weights_data_type: DataType,@@ -130,6 +135,9 @@ impl GemmSpecialization {pub(crate) fn quant_combo_specs(weights_data_type: DataType,combo: MatmulQuantCombo,+ n: u32,+ k: u32,+ use_mxu: bool,) -> Vec<Self> {let bits = DataType::from(combo.mode).size_in_bits() as u32;let group_size = combo.group_size;@@ -138,60 +146,38 @@ impl GemmSpecialization {QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,};- let mut out = Vec::new();- for &tiling in quant_tiling_set(weights_data_type) {- if tiling.simdgroup_block_k() > group_size {- continue;++ let mut tilings: Vec<GemmTiling> = Vec::new();+ for &m in &QUANT_PREHEAT_PROBE_MS {+ let tiling = if use_mxu {+ select_mxu_quant_tiling(m, n, group_size)+ } else {+ select_quant_tiling(m, n, group_size)+ };+ if !tilings.contains(&tiling) {+ tilings.push(tiling);}- for align_n in [true, false] {- for output_transform in [- GemmDTransform::empty(),- GemmDTransform::BIAS,- GemmDTransform::RHT,- GemmDTransform::BIAS | GemmDTransform::RHT,- ] {- out.push(Self {+ }++ let mut out = Vec::new();+ for tiling in tilings {+ let align_n = n.is_multiple_of(tiling.block_n());+ let align_k = k.is_multiple_of(tiling.block_k());+ for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS, GemmDTransform::RHT] {+ for align_m in [true, false] {+ let spec = Self {weights_data_type,tiling,- use_mxu: false,+ use_mxu,output_transform,- alignment: GemmAlignment::new(true, align_n, true),+ alignment: GemmAlignment::new(align_m, align_n, align_k),transpose_b: true,b_prologue,bits_per_b: Some(bits),group_size: Some(group_size),- });- }- }- }- for &tiling in mxu_tiling_set(weights_data_type) {- if !group_size.is_multiple_of(32) {- continue;- }- if !tiling.fits_quant_group_size(group_size) {- continue;- }- for align_m in [true, false] {- for align_n in [true, false] {- for output_transform in [- GemmDTransform::empty(),- GemmDTransform::SCALE,- GemmDTransform::BIAS,- GemmDTransform::RHT,- GemmDTransform::BIAS | GemmDTransform::RHT,- GemmDTransform::BIAS | GemmDTransform::SCALE,- ] {- out.push(Self {- weights_data_type,- tiling,- use_mxu: true,- output_transform,- alignment: GemmAlignment::new(align_m, align_n, true),- transpose_b: true,- b_prologue,- bits_per_b: Some(bits),- group_size: Some(group_size),- });+ };+ if spec.validate().is_ok() {+ out.push(spec);}}}@@ -221,14 +207,45 @@ fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {]}-pub(crate) fn quant_tiling_set(data_type: DataType) -> &'static [GemmTiling] {- match data_type {- DataType::BF16 => &[- GemmTiling::Tile8x32x32_Simdgroups1x1,- GemmTiling::Tile32x32x32_Simdgroups2x2,- GemmTiling::Tile64x32x32_Simdgroups2x2,- GemmTiling::Tile64x64x32_Simdgroups2x2,- ],- _ => &[],+#[cfg(test)]+mod tests {+ use super::*;+ use crate::backends::common::gpu_types::QuantizationMode;++ fn combo() -> MatmulQuantCombo {+ MatmulQuantCombo {+ method: QuantizationMethod::ScaleZeroPoint,+ mode: QuantizationMode::U4,+ group_size: 32,+ }+ }++ #[test]+ fn quant_specs_are_shape_specific_and_tight() {+ let specs = GemmSpecialization::quant_combo_specs(DataType::BF16, combo(), 3584, 1024, false);+ assert!(!specs.is_empty());+ assert!(specs.len() <= 16, "expected a tight preheat set, got {}", specs.len());+ for spec in &specs {+ assert!(!spec.use_mxu);+ assert_eq!(spec.bits_per_b, Some(4));+ assert_eq!(spec.group_size, Some(32));+ assert_ne!(spec.b_prologue, GemmBPrologueKind::FullPrecision);+ assert!(spec.validate().is_ok());+ }+ let tilings: std::collections::HashSet<_> = specs.iter().map(|s| s.tiling).collect();+ assert!(tilings.contains(&GemmTiling::Tile32x32x32_Simdgroups2x2));+ assert!(tilings.contains(&GemmTiling::Tile8x32x32_Simdgroups1x1));+ }++ #[test]+ fn mxu_path_emits_only_mxu_tilings() {+ let specs = GemmSpecialization::quant_combo_specs(DataType::BF16, combo(), 3584, 1024, true);+ assert!(!specs.is_empty());+ assert!(specs.len() <= 20, "expected a tight preheat set, got {}", specs.len());+ for spec in &specs {+ assert!(spec.use_mxu);+ assert!(spec.tiling.is_mxu_variant());+ assert!(spec.validate().is_ok());+ }}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 11b6f572b..1643c7fde 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -105,7 +105,10 @@ impl GemvSpecialization {})}- fn quant_combo_specs(combo: MatmulQuantCombo) -> Vec<GemvSpecialization> {+ fn quant_combo_specs(+ combo: MatmulQuantCombo,+ k: u32,+ ) -> Vec<GemvSpecialization> {let bits = DataType::from(combo.mode).size_in_bits() as u32;let group_size = combo.group_size;let b_prologue = match combo.method {@@ -113,26 +116,24 @@ impl GemvSpecialization {QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,};- let mut out = Vec::new();- for output_transform in [- GemmDTransform::empty(),- GemmDTransform::BIAS,- GemmDTransform::RHT,- GemmDTransform::BIAS | GemmDTransform::RHT,- ] {- for input_aligned in [true, false] {- out.push(GemvSpecialization {- b_prologue,- group_size,- bits,- output_transform,- input_aligned,- k_split: 1,- num_simdgroups: 8,- });- }- }- out+ let block_size = if bits == 4 {+ 512+ } else {+ 256+ };+ let input_aligned = k.is_multiple_of(block_size);+ [GemmDTransform::empty(), GemmDTransform::BIAS, GemmDTransform::RHT]+ .into_iter()+ .map(|output_transform| GemvSpecialization {+ b_prologue,+ group_size,+ bits,+ output_transform,+ input_aligned,+ k_split: 1,+ num_simdgroups: 8,+ })+ .collect()}}@@ -185,11 +186,12 @@ impl GemvDispatch {&mut self,context: &MetalContext,combo: MatmulQuantCombo,+ k: u32,) -> Result<(), MatmulError<Metal>> {if self.weights_data_type != DataType::BF16 {return Ok(());}- for specialization in GemvSpecialization::quant_combo_specs(combo) {+ for specialization in GemvSpecialization::quant_combo_specs(combo, k) {self.get_or_create(context, specialization)?;}Ok(())diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex b66dca74d..7c896da22 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -70,8 +70,10 @@ impl MatmulKernel for MatmulMetalKernel {&mut self,context: &MetalContext,combo: MatmulQuantCombo,+ output_dim: u32,+ input_dim: u32,) -> Result<(), MetalError> {- self.gemv.preheat_quant_combo(context, combo).map_err(MetalError::from)?;- self.gemm.preheat_quant_combo(context, combo)+ self.gemv.preheat_quant_combo(context, combo, input_dim).map_err(MetalError::from)?;+ self.gemm.preheat_quant_combo(context, combo, output_dim, input_dim)}}diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 958a7ad34..452654756 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -190,6 +190,8 @@ impl<B: Backend> Embedding<B> {embedding_quantization_mode,quantization_method,group_size,+ vocab_size,+ model_dim,)?;(@@ -254,6 +256,8 @@ impl<B: Backend> Embedding<B> {embedding_quantization_mode,quantization_method,group_size,+ vocab_size,+ model_dim,)?;(@@ -385,6 +389,8 @@ impl<B: Backend> Embedding<B> {embedding_quantization_mode,quantization_method,group_size,+ vocab_size,+ model_dim,)?;UntiedEmbeddingReadoutType::Quantized {@@ -768,6 +774,8 @@ fn quantized_readout<B: Backend>(mode: QuantizationMode,method: QuantizationMethod,group_size: usize,+ vocab_size: u32,+ model_dim: u32,) -> Result<(RefCell<<B::Kernels as Kernels>::MatmulKernel>, ReadoutQuantConfig), EmbeddingError<B>> {let mut readout = <B::Kernels as Kernels>::MatmulKernel::new(context, data_type, data_type, data_type).map_err(EmbeddingError::BackendError)?;@@ -779,6 +787,8 @@ fn quantized_readout<B: Backend>(mode,group_size: group_size as u32,},+ vocab_size,+ model_dim,).map_err(EmbeddingError::BackendError)?;Ok((diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsindex ad2a165e9..5c00cc2c6 100644--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -181,6 +181,8 @@ impl<B: Backend> LinearMatmul<B> {mode: weight_quantization_mode,group_size: group_size as u32,},+ output_dim as u32,+ input_dim as u32,).map_err(LinearMatmulError::BackendError)?;