Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

warmup session instead of kernel preheat

eugenef0c3f4d45fcd
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
index 95a3c0827..c3eee70dc 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
@@ -1,7 +1,6 @@
use crate::{
backends::common::{
- AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,
- kernel::matmul::{arguments::MatmulArguments, quant_combo::MatmulQuantCombo},
+ AsBufferRangeRef, Backend, Buffer, Encoder, Kernels, kernel::matmul::arguments::MatmulArguments,
},
data_type::DataType,
};
@@ -21,24 +20,4 @@ pub trait MatmulKernel: Sized {
arguments: MatmulArguments<Self::Backend, TB>,
encoder: &mut Encoder<Self::Backend>,
) -> Result<(), <Self::Backend as Backend>::Error>;
-
- fn preheat_quant_combo(
- &mut self,
- _context: &<Self::Backend as Backend>::Context,
- _combo: MatmulQuantCombo,
- _output_dim: u32,
- _input_dim: u32,
- ) -> Result<(), <Self::Backend as Backend>::Error> {
- Ok(())
- }
-
- fn preheat_full_precision(
- &mut self,
- _context: &<Self::Backend as Backend>::Context,
- _output_dim: u32,
- _input_dim: u32,
- _has_bias: bool,
- ) -> Result<(), <Self::Backend as Backend>::Error> {
- Ok(())
- }
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs
index c7ebfca8c..fbe1265af 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs
@@ -3,11 +3,9 @@ mod d_ops;
mod error;
mod kernel;
mod matmul_b;
-mod quant_combo;
pub use arguments::MatmulArguments;
pub use d_ops::MatmulDOps;
pub use error::MatmulError;
pub use kernel::MatmulKernel;
pub use matmul_b::MatmulB;
-pub use quant_combo::MatmulQuantCombo;
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rs
deleted file mode 100644
index 471933b14..000000000
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-use crate::backends::common::gpu_types::{QuantizationMethod, QuantizationMode};
-
-#[derive(Debug, Clone, Copy)]
-pub struct MatmulQuantCombo {
- pub method: QuantizationMethod,
- pub mode: QuantizationMode,
- pub group_size: u32,
-}
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index 26651e852..40d47c2d5 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -70,7 +70,6 @@ impl MetalContext {
}
let pipeline = self.library.compute_pipeline_state(function_name, constants)?;
-
self.pipeline_cache.borrow_mut().insert(cache_key.to_string(), pipeline.clone());
Ok(pipeline)
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index b0fb9f2f9..03e0359a5 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -11,7 +11,7 @@ use crate::{
},
kernel::{
HadamardTransformKernel, Kernels, TensorAddBiasKernel,
- matmul::{MatmulArguments, MatmulB, MatmulError, MatmulQuantCombo},
+ matmul::{MatmulArguments, MatmulB, MatmulError},
},
},
metal::{
@@ -66,43 +66,6 @@ impl GemmKernel {
Ok(kernel)
}
- pub fn preheat_full_precision(
- &mut self,
- context: &MetalContext,
- n: u32,
- k: u32,
- output_transform: GemmDTransform,
- ) -> Result<(), MetalError> {
- let all_float = [self.weights_data_type, self.input_data_type, self.output_data_type]
- .into_iter()
- .all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32));
- let use_mxu = context.device.supports_mxu() && all_float;
- for specialization in
- GemmSpecialization::full_precision_combo_specs(self.weights_data_type, n, k, output_transform, use_mxu)
- {
- self.get_or_create(context, specialization)?;
- }
- Ok(())
- }
-
- pub fn preheat_quant_combo(
- &mut self,
- context: &MetalContext,
- combo: MatmulQuantCombo,
- n: u32,
- k: u32,
- ) -> Result<(), MetalError> {
- let all_float = [self.weights_data_type, self.input_data_type, self.output_data_type]
- .into_iter()
- .all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32));
- let use_mxu = context.device.supports_mxu() && all_float && k.is_multiple_of(select_mxu_tiling(n, n).block_k());
-
- for specialization in GemmSpecialization::quant_combo_specs(self.weights_data_type, combo, n, k, use_mxu) {
- self.get_or_create(context, specialization)?;
- }
- Ok(())
- }
-
fn get_or_create(
&mut self,
context: &MetalContext,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 1ad2c18af..ee6471c6a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -1,20 +1,9 @@
-use super::{
- error::GemmSpecializationError,
- kernel::{select_mxu_quant_tiling, select_mxu_tiling, select_quant_tiling, select_simdgroup_tiling},
-};
+use super::error::GemmSpecializationError;
use crate::{
- backends::common::{
- gpu_types::{
- QuantizationMethod,
- gemm::{GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},
- },
- kernel::matmul::MatmulQuantCombo,
- },
+ backends::common::gpu_types::gemm::{GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},
data_type::DataType,
};
-const QUANT_PREHEAT_PROBE_MS: [u32; 4] = [4096, 128, 48, 8];
-
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(crate) struct GemmSpecialization {
pub(crate) weights_data_type: DataType,
@@ -62,213 +51,4 @@ impl GemmSpecialization {
}
Ok(())
}
-
- pub(crate) fn full_precision_combo_specs(
- weights_data_type: DataType,
- n: u32,
- k: u32,
- output_transform: GemmDTransform,
- use_mxu: bool,
- ) -> Vec<Self> {
- let tiling_supported = if use_mxu {
- !mxu_tiling_set(weights_data_type).is_empty()
- } else {
- !simdgroup_tiling_set(weights_data_type).is_empty()
- };
- if !tiling_supported {
- return Vec::new();
- }
-
- let mut tilings: Vec<GemmTiling> = Vec::new();
- for &m in &QUANT_PREHEAT_PROBE_MS {
- let tiling = if use_mxu {
- select_mxu_tiling(m, n)
- } else {
- select_simdgroup_tiling(m, n, k)
- };
- if !tilings.contains(&tiling) {
- tilings.push(tiling);
- }
- }
-
- let mut transforms = vec![GemmDTransform::empty()];
- if !output_transform.is_empty() {
- transforms.push(output_transform);
- }
-
- let mut out = Vec::new();
- for tiling in tilings {
- let align_n = n.is_multiple_of(tiling.block_n());
- let align_k = k.is_multiple_of(tiling.block_k());
- for &output_transform in &transforms {
- for align_m in [true, false] {
- let spec = Self {
- weights_data_type,
- tiling,
- use_mxu,
- output_transform,
- alignment: GemmAlignment::new(align_m, align_n, align_k),
- transpose_b: true,
- b_prologue: GemmBPrologueKind::FullPrecision,
- bits_per_b: None,
- group_size: None,
- };
- if spec.validate().is_ok() {
- out.push(spec);
- }
- }
- }
- }
- out
- }
-
- pub(crate) fn quant_combo_specs(
- weights_data_type: DataType,
- combo: MatmulQuantCombo,
- n: u32,
- k: u32,
- use_mxu: bool,
- ) -> Vec<Self> {
- let bits = DataType::from(combo.mode).size_in_bits() as u32;
- let group_size = combo.group_size;
- let b_prologue = match combo.method {
- QuantizationMethod::ScaleBias => GemmBPrologueKind::ScaleBiasDequant,
- QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,
- QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,
- };
-
- let mut tilings: Vec<GemmTiling> = Vec::new();
- for &m in &QUANT_PREHEAT_PROBE_MS {
- let tiling = if use_mxu {
- select_mxu_quant_tiling(m, n, group_size)
- } else {
- select_quant_tiling(m, n, group_size)
- };
- if !tilings.contains(&tiling) {
- tilings.push(tiling);
- }
- }
-
- let mut out = Vec::new();
- for tiling in tilings {
- let align_n = n.is_multiple_of(tiling.block_n());
- let align_k = k.is_multiple_of(tiling.block_k());
- for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS, GemmDTransform::RHT] {
- for align_m in [true, false] {
- let spec = Self {
- weights_data_type,
- tiling,
- use_mxu,
- output_transform,
- alignment: GemmAlignment::new(align_m, align_n, align_k),
- transpose_b: true,
- b_prologue,
- bits_per_b: Some(bits),
- group_size: Some(group_size),
- };
- if spec.validate().is_ok() {
- out.push(spec);
- }
- }
- }
- }
- out
- }
-}
-
-fn simdgroup_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
- match data_type {
- DataType::BF16 | DataType::F32 => {
- &[GemmTiling::Tile64x32x32_Simdgroups2x2, GemmTiling::Tile64x64x16_Simdgroups2x2]
- },
- _ => &[],
- }
-}
-
-fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
- if !matches!(data_type, DataType::BF16 | DataType::F32) {
- return &[];
- }
- &[
- GemmTiling::Tile64x64x256_Simdgroups2x2,
- GemmTiling::Tile32x64x256_Simdgroups2x2,
- GemmTiling::Tile64x32x256_Simdgroups4x1,
- GemmTiling::Tile128x128x256_Simdgroups4x4,
- ]
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::backends::common::gpu_types::QuantizationMode;
-
- fn combo() -> MatmulQuantCombo {
- MatmulQuantCombo {
- method: QuantizationMethod::ScaleZeroPoint,
- mode: QuantizationMode::U4,
- group_size: 32,
- }
- }
-
- #[test]
- fn quant_specs_are_shape_specific_and_tight() {
- let specs = GemmSpecialization::quant_combo_specs(DataType::BF16, combo(), 3584, 1024, false);
- assert!(!specs.is_empty());
- assert!(specs.len() <= 16, "expected a tight preheat set, got {}", specs.len());
- for spec in &specs {
- assert!(!spec.use_mxu);
- assert_eq!(spec.bits_per_b, Some(4));
- assert_eq!(spec.group_size, Some(32));
- assert_ne!(spec.b_prologue, GemmBPrologueKind::FullPrecision);
- assert!(spec.validate().is_ok());
- }
- let tilings: std::collections::HashSet<_> = specs.iter().map(|s| s.tiling).collect();
- assert!(tilings.contains(&GemmTiling::Tile32x32x32_Simdgroups2x2));
- assert!(tilings.contains(&GemmTiling::Tile8x32x32_Simdgroups1x1));
- }
-
- #[test]
- fn mxu_path_emits_only_mxu_tilings() {
- let specs = GemmSpecialization::quant_combo_specs(DataType::BF16, combo(), 3584, 1024, true);
- assert!(!specs.is_empty());
- assert!(specs.len() <= 20, "expected a tight preheat set, got {}", specs.len());
- for spec in &specs {
- assert!(spec.use_mxu);
- assert!(spec.tiling.is_mxu_variant());
- assert!(spec.validate().is_ok());
- }
- }
-
- #[test]
- fn full_precision_specs_are_shape_specific_and_tight() {
- let plain =
- GemmSpecialization::full_precision_combo_specs(DataType::BF16, 3584, 1024, GemmDTransform::empty(), false);
- assert!(!plain.is_empty());
- assert!(plain.len() <= 16, "expected a tight preheat set, got {}", plain.len());
- for spec in &plain {
- assert!(!spec.use_mxu);
- assert_eq!(spec.b_prologue, GemmBPrologueKind::FullPrecision);
- assert_eq!(spec.output_transform, GemmDTransform::empty());
- assert!(spec.validate().is_ok());
- }
-
- let biased =
- GemmSpecialization::full_precision_combo_specs(DataType::BF16, 3584, 1024, GemmDTransform::BIAS, false);
- let transforms: std::collections::HashSet<_> = biased.iter().map(|spec| spec.output_transform).collect();
- assert!(transforms.contains(&GemmDTransform::empty()));
- assert!(transforms.contains(&GemmDTransform::BIAS));
- assert_eq!(transforms.len(), 2, "only empty + the layer transform should be preheated");
- }
-
- #[test]
- fn full_precision_mxu_emits_only_mxu_tilings() {
- let specs =
- GemmSpecialization::full_precision_combo_specs(DataType::BF16, 3584, 1024, GemmDTransform::BIAS, true);
- assert!(!specs.is_empty());
- for spec in &specs {
- assert!(spec.use_mxu);
- assert!(spec.tiling.is_mxu_variant());
- assert!(spec.validate().is_ok());
- }
- }
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 7e581482c..2662d767d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -7,11 +7,8 @@ use crate::{
backends::{
common::{
Allocation, AsBufferRangeRef, Buffer, Encoder,
- gpu_types::{
- QuantizationMethod,
- gemm::{GemmBPrologueKind, GemmDTransform},
- },
- kernel::matmul::{MatmulArguments, MatmulB, MatmulError, MatmulQuantCombo},
+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
+ kernel::matmul::{MatmulArguments, MatmulB, MatmulError},
},
metal::{Metal, context::MetalContext, kernel::GemvMetalKernel},
},
@@ -104,37 +101,6 @@ impl GemvSpecialization {
num_simdgroups,
})
}
-
- fn quant_combo_specs(
- combo: MatmulQuantCombo,
- k: u32,
- ) -> Vec<GemvSpecialization> {
- let bits = DataType::from(combo.mode).size_in_bits() as u32;
- let group_size = combo.group_size;
- let b_prologue = match combo.method {
- QuantizationMethod::ScaleBias => GemmBPrologueKind::ScaleBiasDequant,
- QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,
- QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,
- };
- let block_size = if bits == 4 {
- 512
- } else {
- 256
- };
- let input_aligned = k.is_multiple_of(block_size);
- [GemmDTransform::empty(), GemmDTransform::BIAS, GemmDTransform::RHT]
- .into_iter()
- .map(|output_transform| GemvSpecialization {
- b_prologue,
- group_size,
- bits,
- output_transform,
- input_aligned,
- k_split: 1,
- num_simdgroups: 8,
- })
- .collect()
- }
}
fn rows_per_threadgroup(
@@ -182,21 +148,6 @@ impl GemvDispatch {
})
}
- pub(crate) fn preheat_quant_combo(
- &mut self,
- context: &MetalContext,
- combo: MatmulQuantCombo,
- k: u32,
- ) -> Result<(), MatmulError<Metal>> {
- if self.weights_data_type != DataType::BF16 {
- return Ok(());
- }
- for specialization in GemvSpecialization::quant_combo_specs(combo, k) {
- self.get_or_create(context, specialization)?;
- }
- Ok(())
- }
-
fn get_or_create(
&mut self,
context: &MetalContext,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index fae392d80..3b2f87986 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -7,8 +7,7 @@ use crate::{
backends::{
common::{
AsBufferRangeRef, Buffer, Encoder,
- gpu_types::gemm::GemmDTransform,
- kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulQuantCombo},
+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},
},
metal::{Metal, context::MetalContext, error::MetalError},
},
@@ -66,30 +65,4 @@ impl MatmulKernel for MatmulMetalKernel {
None => self.gemm.encode(arguments, encoder),
}
}
-
- fn preheat_quant_combo(
- &mut self,
- context: &MetalContext,
- combo: MatmulQuantCombo,
- output_dim: u32,
- input_dim: u32,
- ) -> Result<(), MetalError> {
- self.gemv.preheat_quant_combo(context, combo, input_dim).map_err(MetalError::from)?;
- self.gemm.preheat_quant_combo(context, combo, output_dim, input_dim)
- }
-
- fn preheat_full_precision(
- &mut self,
- context: &MetalContext,
- output_dim: u32,
- input_dim: u32,
- has_bias: bool,
- ) -> Result<(), MetalError> {
- let output_transform = if has_bias {
- GemmDTransform::BIAS
- } else {
- GemmDTransform::empty()
- };
- self.gemm.preheat_full_precision(context, output_dim, input_dim, output_transform)
- }
}
diff --git a/crates/backend-uzu/src/encodable_block/attention/mod.rs b/crates/backend-uzu/src/encodable_block/attention/mod.rs
index cb7211e04..d16e4ad2f 100644
--- a/crates/backend-uzu/src/encodable_block/attention/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/attention/mod.rs
@@ -174,9 +174,7 @@ impl<B: Backend> Attention<B> {
let mut two_pass_2_kernels = HashMap::new();
let mut fallback_scatter_scores_kernels = HashMap::new();
- for (head_dim, is_trie, is_kv_cache_ring) in
- iproduct!([64u32, 128u32, 256u32, 512u32], [false, true], [false, true])
- {
+ for (head_dim, is_trie, is_kv_cache_ring) in iproduct!([config.head_dim as u32], [false, true], [false, true]) {
let key = KernelKey {
head_dim,
is_trie,
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 953610ad4..fddca9bec 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -9,7 +9,7 @@ use crate::{
gpu_types::{QuantizationMethod, QuantizationMode},
kernel::{
FullPrecisionEmbeddingLookupKernel, LogitSoftCapKernel, QuantizedEmbeddingLookupKernel,
- matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
config::{
@@ -143,16 +143,13 @@ impl<B: Backend> Embedding<B> {
model_shape.data_type,
)
.map_err(EmbeddingError::BackendError)?;
- let mut readout_kernel = <B::Kernels as Kernels>::MatmulKernel::new(
+ let readout_kernel = <B::Kernels as Kernels>::MatmulKernel::new(
context,
model_shape.data_type,
model_shape.data_type,
model_shape.data_type,
)
.map_err(EmbeddingError::BackendError)?;
- readout_kernel
- .preheat_full_precision(context, vocab_size, model_dim, false)
- .map_err(EmbeddingError::BackendError)?;
let readout = RefCell::new(readout_kernel);
(
@@ -192,8 +189,6 @@ impl<B: Backend> Embedding<B> {
embedding_quantization_mode,
quantization_method,
group_size,
- vocab_size,
- model_dim,
)?;
(
@@ -258,8 +253,6 @@ impl<B: Backend> Embedding<B> {
embedding_quantization_mode,
quantization_method,
group_size,
- vocab_size,
- model_dim,
)?;
(
@@ -358,16 +351,13 @@ impl<B: Backend> Embedding<B> {
.leaf("weights")?
.validate(&[vocab_size as usize, model_dim as usize], model_shape.data_type)?
.read_allocation()?;
- let mut readout_kernel = <B::Kernels as Kernels>::MatmulKernel::new(
+ let readout_kernel = <B::Kernels as Kernels>::MatmulKernel::new(
context,
model_shape.data_type,
model_shape.data_type,
model_shape.data_type,
)
.map_err(EmbeddingError::BackendError)?;
- readout_kernel
- .preheat_full_precision(context, vocab_size, model_dim, false)
- .map_err(EmbeddingError::BackendError)?;
let readout = RefCell::new(readout_kernel);
UntiedEmbeddingReadoutType::FullPrecision {
@@ -393,8 +383,6 @@ impl<B: Backend> Embedding<B> {
embedding_quantization_mode,
quantization_method,
group_size,
- vocab_size,
- model_dim,
)?;
UntiedEmbeddingReadoutType::Quantized {
@@ -778,22 +766,8 @@ fn quantized_readout<B: Backend>(
mode: QuantizationMode,
method: QuantizationMethod,
group_size: usize,
- vocab_size: u32,
- model_dim: u32,
) -> Result<(RefCell<<B::Kernels as Kernels>::MatmulKernel>, ReadoutQuantConfig), EmbeddingError<B>> {
- let mut readout = <B::Kernels as Kernels>::MatmulKernel::new(context, data_type, data_type, data_type)
- .map_err(EmbeddingError::BackendError)?;
- readout
- .preheat_quant_combo(
- context,
- MatmulQuantCombo {
- method,
- mode,
- group_size: group_size as u32,
- },
- vocab_size,
- model_dim,
- )
+ let readout = <B::Kernels as Kernels>::MatmulKernel::new(context, data_type, data_type, data_type)
.map_err(EmbeddingError::BackendError)?;
Ok((
RefCell::new(readout),
diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
index 27db0072a..779e04071 100644
--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
@@ -10,7 +10,7 @@ use crate::{
gpu_types::{QuantizationMethod, QuantizationMode},
kernel::{
Kernels,
- matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},
+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
config::weight_matrix::{AnyWeightMatrixSpec, Layout, int_spec::IntSpec, mlx_spec::MLXSpec},
@@ -76,12 +76,9 @@ impl<B: Backend> LinearMatmul<B> {
let biases =
load_biases(weights_data_type, output_data_type, output_dim, has_biases.then_some(parameter_tree))?;
- let mut kernel =
+ let kernel =
<B::Kernels as Kernels>::MatmulKernel::new(context, weights_data_type, input_data_type, output_data_type)
.map_err(LinearMatmulError::BackendError)?;
- kernel
- .preheat_full_precision(context, output_dim as u32, input_dim as u32, biases.is_some())
- .map_err(LinearMatmulError::BackendError)?;
Ok(Self {
kernel: RefCell::new(kernel),
@@ -173,21 +170,9 @@ impl<B: Backend> LinearMatmul<B> {
let biases = load_biases(weights_data_type, output_data_type, output_dim, bias_tree)?;
- let mut kernel =
+ let kernel =
<B::Kernels as Kernels>::MatmulKernel::new(context, weights_data_type, input_data_type, output_data_type)
.map_err(LinearMatmulError::BackendError)?;
- kernel
- .preheat_quant_combo(
- context,
- MatmulQuantCombo {
- method: quantization_method,
- mode: weight_quantization_mode,
- group_size: group_size as u32,
- },
- output_dim as u32,
- input_dim as u32,
- )
- .map_err(LinearMatmulError::BackendError)?;
Ok(Self {
kernel: RefCell::new(kernel),
diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rs
index e9a78b512..235be2bd2 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs
@@ -679,7 +679,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
let context = LanguageModelGeneratorContext::new(model_path, &decoding_config, model_config)?;
- Ok(Self {
+ let mut generator = Self {
decoding_config,
tokens: Vec::new(),
context,
@@ -687,7 +687,31 @@ impl<B: Backend> LanguageModelGenerator<B> {
async_token_ids: None,
registered_prefix_len: 0,
gpu_capture,
- })
+ };
+ generator.warmup();
+ Ok(generator)
+ }
+
+ fn warmup(&mut self) {
+ if let Err(error) = self.run_warmup() {
+ eprintln!("uzu: model warmup skipped (kernels will compile on first use): {error}");
+ }
+ self.reset_state();
+ }
+
+ fn run_warmup(&mut self) -> Result<(), Error> {
+ let context_length = self.context.get_context_length(&self.decoding_config);
+ let first = 64.min(context_length.saturating_sub(1)).max(1);
+ self.prefill(vec![0u64; first], None, SamplingMethod::Greedy, 0, true)?;
+ self.generate(None, SamplingMethod::Greedy)?;
+ for &len in &[65usize, 16] {
+ let offset = self.tokens.len();
+ if offset + len >= context_length {
+ break;
+ }
+ self.prefill(vec![0u64; len], None, SamplingMethod::Greedy, offset, false)?;
+ }
+ Ok(())
}
fn repetition_penalty_config(sampling_method: SamplingMethod) -> Option<(f32, usize)> {