A high-performance inference engine for AI models
warmup session instead of kernel preheat
eugenef0c3f4d45fcd
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rsindex 95a3c0827..c3eee70dc 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs@@ -1,7 +1,6 @@use crate::{backends::common::{- AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,- kernel::matmul::{arguments::MatmulArguments, quant_combo::MatmulQuantCombo},+ AsBufferRangeRef, Backend, Buffer, Encoder, Kernels, kernel::matmul::arguments::MatmulArguments,},data_type::DataType,};@@ -21,24 +20,4 @@ pub trait MatmulKernel: Sized {arguments: MatmulArguments<Self::Backend, TB>,encoder: &mut Encoder<Self::Backend>,) -> Result<(), <Self::Backend as Backend>::Error>;-- fn preheat_quant_combo(- &mut self,- _context: &<Self::Backend as Backend>::Context,- _combo: MatmulQuantCombo,- _output_dim: u32,- _input_dim: u32,- ) -> Result<(), <Self::Backend as Backend>::Error> {- Ok(())- }-- fn preheat_full_precision(- &mut self,- _context: &<Self::Backend as Backend>::Context,- _output_dim: u32,- _input_dim: u32,- _has_bias: bool,- ) -> Result<(), <Self::Backend as Backend>::Error> {- Ok(())- }}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rsindex c7ebfca8c..fbe1265af 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs@@ -3,11 +3,9 @@ mod d_ops;mod error;mod kernel;mod matmul_b;-mod quant_combo;pub use arguments::MatmulArguments;pub use d_ops::MatmulDOps;pub use error::MatmulError;pub use kernel::MatmulKernel;pub use matmul_b::MatmulB;-pub use quant_combo::MatmulQuantCombo;diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rsdeleted file mode 100644index 471933b14..000000000--- a/crates/backend-uzu/src/backends/common/kernel/matmul/quant_combo.rs+++ /dev/null@@ -1,8 +0,0 @@-use crate::backends::common::gpu_types::{QuantizationMethod, QuantizationMode};--#[derive(Debug, Clone, Copy)]-pub struct MatmulQuantCombo {- pub method: QuantizationMethod,- pub mode: QuantizationMode,- pub group_size: u32,-}diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex 26651e852..40d47c2d5 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -70,7 +70,6 @@ impl MetalContext {}let pipeline = self.library.compute_pipeline_state(function_name, constants)?;-self.pipeline_cache.borrow_mut().insert(cache_key.to_string(), pipeline.clone());Ok(pipeline)diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex b0fb9f2f9..03e0359a5 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -11,7 +11,7 @@ use crate::{},kernel::{HadamardTransformKernel, Kernels, TensorAddBiasKernel,- matmul::{MatmulArguments, MatmulB, MatmulError, MatmulQuantCombo},+ matmul::{MatmulArguments, MatmulB, MatmulError},},},metal::{@@ -66,43 +66,6 @@ impl GemmKernel {Ok(kernel)}- pub fn preheat_full_precision(- &mut self,- context: &MetalContext,- n: u32,- k: u32,- output_transform: GemmDTransform,- ) -> Result<(), MetalError> {- let all_float = [self.weights_data_type, self.input_data_type, self.output_data_type]- .into_iter()- .all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32));- let use_mxu = context.device.supports_mxu() && all_float;- for specialization in- GemmSpecialization::full_precision_combo_specs(self.weights_data_type, n, k, output_transform, use_mxu)- {- self.get_or_create(context, specialization)?;- }- Ok(())- }-- pub fn preheat_quant_combo(- &mut self,- context: &MetalContext,- combo: MatmulQuantCombo,- n: u32,- k: u32,- ) -> Result<(), MetalError> {- let all_float = [self.weights_data_type, self.input_data_type, self.output_data_type]- .into_iter()- .all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32));- let use_mxu = context.device.supports_mxu() && all_float && k.is_multiple_of(select_mxu_tiling(n, n).block_k());-- for specialization in GemmSpecialization::quant_combo_specs(self.weights_data_type, combo, n, k, use_mxu) {- self.get_or_create(context, specialization)?;- }- Ok(())- }-fn get_or_create(&mut self,context: &MetalContext,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 1ad2c18af..ee6471c6a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -1,20 +1,9 @@-use super::{- error::GemmSpecializationError,- kernel::{select_mxu_quant_tiling, select_mxu_tiling, select_quant_tiling, select_simdgroup_tiling},-};+use super::error::GemmSpecializationError;use crate::{- backends::common::{- gpu_types::{- QuantizationMethod,- gemm::{GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},- },- kernel::matmul::MatmulQuantCombo,- },+ backends::common::gpu_types::gemm::{GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},data_type::DataType,};-const QUANT_PREHEAT_PROBE_MS: [u32; 4] = [4096, 128, 48, 8];-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub(crate) struct GemmSpecialization {pub(crate) weights_data_type: DataType,@@ -62,213 +51,4 @@ impl GemmSpecialization {}Ok(())}-- pub(crate) fn full_precision_combo_specs(- weights_data_type: DataType,- n: u32,- k: u32,- output_transform: GemmDTransform,- use_mxu: bool,- ) -> Vec<Self> {- let tiling_supported = if use_mxu {- !mxu_tiling_set(weights_data_type).is_empty()- } else {- !simdgroup_tiling_set(weights_data_type).is_empty()- };- if !tiling_supported {- return Vec::new();- }-- let mut tilings: Vec<GemmTiling> = Vec::new();- for &m in &QUANT_PREHEAT_PROBE_MS {- let tiling = if use_mxu {- select_mxu_tiling(m, n)- } else {- select_simdgroup_tiling(m, n, k)- };- if !tilings.contains(&tiling) {- tilings.push(tiling);- }- }-- let mut transforms = vec![GemmDTransform::empty()];- if !output_transform.is_empty() {- transforms.push(output_transform);- }-- let mut out = Vec::new();- for tiling in tilings {- let align_n = n.is_multiple_of(tiling.block_n());- let align_k = k.is_multiple_of(tiling.block_k());- for &output_transform in &transforms {- for align_m in [true, false] {- let spec = Self {- weights_data_type,- tiling,- use_mxu,- output_transform,- alignment: GemmAlignment::new(align_m, align_n, align_k),- transpose_b: true,- b_prologue: GemmBPrologueKind::FullPrecision,- bits_per_b: None,- group_size: None,- };- if spec.validate().is_ok() {- out.push(spec);- }- }- }- }- out- }-- pub(crate) fn quant_combo_specs(- weights_data_type: DataType,- combo: MatmulQuantCombo,- n: u32,- k: u32,- use_mxu: bool,- ) -> Vec<Self> {- let bits = DataType::from(combo.mode).size_in_bits() as u32;- let group_size = combo.group_size;- let b_prologue = match combo.method {- QuantizationMethod::ScaleBias => GemmBPrologueKind::ScaleBiasDequant,- QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,- QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,- };-- let mut tilings: Vec<GemmTiling> = Vec::new();- for &m in &QUANT_PREHEAT_PROBE_MS {- let tiling = if use_mxu {- select_mxu_quant_tiling(m, n, group_size)- } else {- select_quant_tiling(m, n, group_size)- };- if !tilings.contains(&tiling) {- tilings.push(tiling);- }- }-- let mut out = Vec::new();- for tiling in tilings {- let align_n = n.is_multiple_of(tiling.block_n());- let align_k = k.is_multiple_of(tiling.block_k());- for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS, GemmDTransform::RHT] {- for align_m in [true, false] {- let spec = Self {- weights_data_type,- tiling,- use_mxu,- output_transform,- alignment: GemmAlignment::new(align_m, align_n, align_k),- transpose_b: true,- b_prologue,- bits_per_b: Some(bits),- group_size: Some(group_size),- };- if spec.validate().is_ok() {- out.push(spec);- }- }- }- }- out- }-}--fn simdgroup_tiling_set(data_type: DataType) -> &'static [GemmTiling] {- match data_type {- DataType::BF16 | DataType::F32 => {- &[GemmTiling::Tile64x32x32_Simdgroups2x2, GemmTiling::Tile64x64x16_Simdgroups2x2]- },- _ => &[],- }-}--fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {- if !matches!(data_type, DataType::BF16 | DataType::F32) {- return &[];- }- &[- GemmTiling::Tile64x64x256_Simdgroups2x2,- GemmTiling::Tile32x64x256_Simdgroups2x2,- GemmTiling::Tile64x32x256_Simdgroups4x1,- GemmTiling::Tile128x128x256_Simdgroups4x4,- ]-}--#[cfg(test)]-mod tests {- use super::*;- use crate::backends::common::gpu_types::QuantizationMode;-- fn combo() -> MatmulQuantCombo {- MatmulQuantCombo {- method: QuantizationMethod::ScaleZeroPoint,- mode: QuantizationMode::U4,- group_size: 32,- }- }-- #[test]- fn quant_specs_are_shape_specific_and_tight() {- let specs = GemmSpecialization::quant_combo_specs(DataType::BF16, combo(), 3584, 1024, false);- assert!(!specs.is_empty());- assert!(specs.len() <= 16, "expected a tight preheat set, got {}", specs.len());- for spec in &specs {- assert!(!spec.use_mxu);- assert_eq!(spec.bits_per_b, Some(4));- assert_eq!(spec.group_size, Some(32));- assert_ne!(spec.b_prologue, GemmBPrologueKind::FullPrecision);- assert!(spec.validate().is_ok());- }- let tilings: std::collections::HashSet<_> = specs.iter().map(|s| s.tiling).collect();- assert!(tilings.contains(&GemmTiling::Tile32x32x32_Simdgroups2x2));- assert!(tilings.contains(&GemmTiling::Tile8x32x32_Simdgroups1x1));- }-- #[test]- fn mxu_path_emits_only_mxu_tilings() {- let specs = GemmSpecialization::quant_combo_specs(DataType::BF16, combo(), 3584, 1024, true);- assert!(!specs.is_empty());- assert!(specs.len() <= 20, "expected a tight preheat set, got {}", specs.len());- for spec in &specs {- assert!(spec.use_mxu);- assert!(spec.tiling.is_mxu_variant());- assert!(spec.validate().is_ok());- }- }-- #[test]- fn full_precision_specs_are_shape_specific_and_tight() {- let plain =- GemmSpecialization::full_precision_combo_specs(DataType::BF16, 3584, 1024, GemmDTransform::empty(), false);- assert!(!plain.is_empty());- assert!(plain.len() <= 16, "expected a tight preheat set, got {}", plain.len());- for spec in &plain {- assert!(!spec.use_mxu);- assert_eq!(spec.b_prologue, GemmBPrologueKind::FullPrecision);- assert_eq!(spec.output_transform, GemmDTransform::empty());- assert!(spec.validate().is_ok());- }-- let biased =- GemmSpecialization::full_precision_combo_specs(DataType::BF16, 3584, 1024, GemmDTransform::BIAS, false);- let transforms: std::collections::HashSet<_> = biased.iter().map(|spec| spec.output_transform).collect();- assert!(transforms.contains(&GemmDTransform::empty()));- assert!(transforms.contains(&GemmDTransform::BIAS));- assert_eq!(transforms.len(), 2, "only empty + the layer transform should be preheated");- }-- #[test]- fn full_precision_mxu_emits_only_mxu_tilings() {- let specs =- GemmSpecialization::full_precision_combo_specs(DataType::BF16, 3584, 1024, GemmDTransform::BIAS, true);- assert!(!specs.is_empty());- for spec in &specs {- assert!(spec.use_mxu);- assert!(spec.tiling.is_mxu_variant());- assert!(spec.validate().is_ok());- }- }}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 7e581482c..2662d767d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -7,11 +7,8 @@ use crate::{backends::{common::{Allocation, AsBufferRangeRef, Buffer, Encoder,- gpu_types::{- QuantizationMethod,- gemm::{GemmBPrologueKind, GemmDTransform},- },- kernel::matmul::{MatmulArguments, MatmulB, MatmulError, MatmulQuantCombo},+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},+ kernel::matmul::{MatmulArguments, MatmulB, MatmulError},},metal::{Metal, context::MetalContext, kernel::GemvMetalKernel},},@@ -104,37 +101,6 @@ impl GemvSpecialization {num_simdgroups,})}-- fn quant_combo_specs(- combo: MatmulQuantCombo,- k: u32,- ) -> Vec<GemvSpecialization> {- let bits = DataType::from(combo.mode).size_in_bits() as u32;- let group_size = combo.group_size;- let b_prologue = match combo.method {- QuantizationMethod::ScaleBias => GemmBPrologueKind::ScaleBiasDequant,- QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,- QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,- };- let block_size = if bits == 4 {- 512- } else {- 256- };- let input_aligned = k.is_multiple_of(block_size);- [GemmDTransform::empty(), GemmDTransform::BIAS, GemmDTransform::RHT]- .into_iter()- .map(|output_transform| GemvSpecialization {- b_prologue,- group_size,- bits,- output_transform,- input_aligned,- k_split: 1,- num_simdgroups: 8,- })- .collect()- }}fn rows_per_threadgroup(@@ -182,21 +148,6 @@ impl GemvDispatch {})}- pub(crate) fn preheat_quant_combo(- &mut self,- context: &MetalContext,- combo: MatmulQuantCombo,- k: u32,- ) -> Result<(), MatmulError<Metal>> {- if self.weights_data_type != DataType::BF16 {- return Ok(());- }- for specialization in GemvSpecialization::quant_combo_specs(combo, k) {- self.get_or_create(context, specialization)?;- }- Ok(())- }-fn get_or_create(&mut self,context: &MetalContext,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex fae392d80..3b2f87986 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -7,8 +7,7 @@ use crate::{backends::{common::{AsBufferRangeRef, Buffer, Encoder,- gpu_types::gemm::GemmDTransform,- kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulQuantCombo},+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},},metal::{Metal, context::MetalContext, error::MetalError},},@@ -66,30 +65,4 @@ impl MatmulKernel for MatmulMetalKernel {None => self.gemm.encode(arguments, encoder),}}-- fn preheat_quant_combo(- &mut self,- context: &MetalContext,- combo: MatmulQuantCombo,- output_dim: u32,- input_dim: u32,- ) -> Result<(), MetalError> {- self.gemv.preheat_quant_combo(context, combo, input_dim).map_err(MetalError::from)?;- self.gemm.preheat_quant_combo(context, combo, output_dim, input_dim)- }-- fn preheat_full_precision(- &mut self,- context: &MetalContext,- output_dim: u32,- input_dim: u32,- has_bias: bool,- ) -> Result<(), MetalError> {- let output_transform = if has_bias {- GemmDTransform::BIAS- } else {- GemmDTransform::empty()- };- self.gemm.preheat_full_precision(context, output_dim, input_dim, output_transform)- }}diff --git a/crates/backend-uzu/src/encodable_block/attention/mod.rs b/crates/backend-uzu/src/encodable_block/attention/mod.rsindex cb7211e04..d16e4ad2f 100644--- a/crates/backend-uzu/src/encodable_block/attention/mod.rs+++ b/crates/backend-uzu/src/encodable_block/attention/mod.rs@@ -174,9 +174,7 @@ impl<B: Backend> Attention<B> {let mut two_pass_2_kernels = HashMap::new();let mut fallback_scatter_scores_kernels = HashMap::new();- for (head_dim, is_trie, is_kv_cache_ring) in- iproduct!([64u32, 128u32, 256u32, 512u32], [false, true], [false, true])- {+ for (head_dim, is_trie, is_kv_cache_ring) in iproduct!([config.head_dim as u32], [false, true], [false, true]) {let key = KernelKey {head_dim,is_trie,diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 953610ad4..fddca9bec 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -9,7 +9,7 @@ use crate::{gpu_types::{QuantizationMethod, QuantizationMode},kernel::{FullPrecisionEmbeddingLookupKernel, LogitSoftCapKernel, QuantizedEmbeddingLookupKernel,- matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},config::{@@ -143,16 +143,13 @@ impl<B: Backend> Embedding<B> {model_shape.data_type,).map_err(EmbeddingError::BackendError)?;- let mut readout_kernel = <B::Kernels as Kernels>::MatmulKernel::new(+ let readout_kernel = <B::Kernels as Kernels>::MatmulKernel::new(context,model_shape.data_type,model_shape.data_type,model_shape.data_type,).map_err(EmbeddingError::BackendError)?;- readout_kernel- .preheat_full_precision(context, vocab_size, model_dim, false)- .map_err(EmbeddingError::BackendError)?;let readout = RefCell::new(readout_kernel);(@@ -192,8 +189,6 @@ impl<B: Backend> Embedding<B> {embedding_quantization_mode,quantization_method,group_size,- vocab_size,- model_dim,)?;(@@ -258,8 +253,6 @@ impl<B: Backend> Embedding<B> {embedding_quantization_mode,quantization_method,group_size,- vocab_size,- model_dim,)?;(@@ -358,16 +351,13 @@ impl<B: Backend> Embedding<B> {.leaf("weights")?.validate(&[vocab_size as usize, model_dim as usize], model_shape.data_type)?.read_allocation()?;- let mut readout_kernel = <B::Kernels as Kernels>::MatmulKernel::new(+ let readout_kernel = <B::Kernels as Kernels>::MatmulKernel::new(context,model_shape.data_type,model_shape.data_type,model_shape.data_type,).map_err(EmbeddingError::BackendError)?;- readout_kernel- .preheat_full_precision(context, vocab_size, model_dim, false)- .map_err(EmbeddingError::BackendError)?;let readout = RefCell::new(readout_kernel);UntiedEmbeddingReadoutType::FullPrecision {@@ -393,8 +383,6 @@ impl<B: Backend> Embedding<B> {embedding_quantization_mode,quantization_method,group_size,- vocab_size,- model_dim,)?;UntiedEmbeddingReadoutType::Quantized {@@ -778,22 +766,8 @@ fn quantized_readout<B: Backend>(mode: QuantizationMode,method: QuantizationMethod,group_size: usize,- vocab_size: u32,- model_dim: u32,) -> Result<(RefCell<<B::Kernels as Kernels>::MatmulKernel>, ReadoutQuantConfig), EmbeddingError<B>> {- let mut readout = <B::Kernels as Kernels>::MatmulKernel::new(context, data_type, data_type, data_type)- .map_err(EmbeddingError::BackendError)?;- readout- .preheat_quant_combo(- context,- MatmulQuantCombo {- method,- mode,- group_size: group_size as u32,- },- vocab_size,- model_dim,- )+ let readout = <B::Kernels as Kernels>::MatmulKernel::new(context, data_type, data_type, data_type).map_err(EmbeddingError::BackendError)?;Ok((RefCell::new(readout),diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsindex 27db0072a..779e04071 100644--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -10,7 +10,7 @@ use crate::{gpu_types::{QuantizationMethod, QuantizationMode},kernel::{Kernels,- matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},+ matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},config::weight_matrix::{AnyWeightMatrixSpec, Layout, int_spec::IntSpec, mlx_spec::MLXSpec},@@ -76,12 +76,9 @@ impl<B: Backend> LinearMatmul<B> {let biases =load_biases(weights_data_type, output_data_type, output_dim, has_biases.then_some(parameter_tree))?;- let mut kernel =+ let kernel =<B::Kernels as Kernels>::MatmulKernel::new(context, weights_data_type, input_data_type, output_data_type).map_err(LinearMatmulError::BackendError)?;- kernel- .preheat_full_precision(context, output_dim as u32, input_dim as u32, biases.is_some())- .map_err(LinearMatmulError::BackendError)?;Ok(Self {kernel: RefCell::new(kernel),@@ -173,21 +170,9 @@ impl<B: Backend> LinearMatmul<B> {let biases = load_biases(weights_data_type, output_data_type, output_dim, bias_tree)?;- let mut kernel =+ let kernel =<B::Kernels as Kernels>::MatmulKernel::new(context, weights_data_type, input_data_type, output_data_type).map_err(LinearMatmulError::BackendError)?;- kernel- .preheat_quant_combo(- context,- MatmulQuantCombo {- method: quantization_method,- mode: weight_quantization_mode,- group_size: group_size as u32,- },- output_dim as u32,- input_dim as u32,- )- .map_err(LinearMatmulError::BackendError)?;Ok(Self {kernel: RefCell::new(kernel),diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rsindex e9a78b512..235be2bd2 100644--- a/crates/backend-uzu/src/language_model/language_model_generator.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs@@ -679,7 +679,7 @@ impl<B: Backend> LanguageModelGenerator<B> {let context = LanguageModelGeneratorContext::new(model_path, &decoding_config, model_config)?;- Ok(Self {+ let mut generator = Self {decoding_config,tokens: Vec::new(),context,@@ -687,7 +687,31 @@ impl<B: Backend> LanguageModelGenerator<B> {async_token_ids: None,registered_prefix_len: 0,gpu_capture,- })+ };+ generator.warmup();+ Ok(generator)+ }++ fn warmup(&mut self) {+ if let Err(error) = self.run_warmup() {+ eprintln!("uzu: model warmup skipped (kernels will compile on first use): {error}");+ }+ self.reset_state();+ }++ fn run_warmup(&mut self) -> Result<(), Error> {+ let context_length = self.context.get_context_length(&self.decoding_config);+ let first = 64.min(context_length.saturating_sub(1)).max(1);+ self.prefill(vec![0u64; first], None, SamplingMethod::Greedy, 0, true)?;+ self.generate(None, SamplingMethod::Greedy)?;+ for &len in &[65usize, 16] {+ let offset = self.tokens.len();+ if offset + len >= context_length {+ break;+ }+ self.prefill(vec![0u64; len], None, SamplingMethod::Greedy, offset, false)?;+ }+ Ok(())}fn repetition_penalty_config(sampling_method: SamplingMethod) -> Option<(f32, usize)> {