A high-performance inference engine for AI models
add precompile
Eugene Bokhand6c747403a4d
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rsindex 835a07c4d..ec8923202 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs@@ -69,8 +69,9 @@ impl FullPrecisionEmbeddingReadout {let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };- let kernel =+ let mut kernel =MatmulKernel::new(data_type).map_err(EmbeddingError::MetalError)?;+ kernel.precompile(_mtl_context).map_err(EmbeddingError::MetalError)?;Ok(Self {kernel: RefCell::new(kernel),diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rsindex 857b2b592..8b338d5f3 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs@@ -88,7 +88,8 @@ impl FullPrecisionLinear {Err(_) => None,};- let kernel = MatmulKernel::new(precision)?;+ let mut kernel = MatmulKernel::new(precision)?;+ kernel.precompile(_mtl_context)?;Ok(Self {kernel: RefCell::new(kernel),diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex cef66acbd..e41463aa0 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -6,6 +6,7 @@ use metal::{use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration,+ tile_configuration::TileConfiguration,};use crate::{DataType,@@ -37,6 +38,64 @@ impl Kernel {})}+ #[allow(clippy::type_complexity)]+ pub fn precompile(+ &mut self,+ context: &MTLContext,+ ) -> Result<(), MTLError> {+ let tiles_and_alignments: &[(+ TileConfiguration,+ &[(bool, bool, bool)],+ )] = match self.data_type {+ DataType::BF16 => &[+ (+ TileConfiguration::new(64, 32, 32, 2, 2, 0),+ &[(false, true, true), (true, true, true)],+ ),+ (+ TileConfiguration::new(64, 64, 16, 2, 2, 0),+ &[+ (false, true, true),+ (true, false, true),+ (true, true, true),+ ],+ ),+ (+ TileConfiguration::new(64, 64, 16, 1, 2, 0),+ &[(true, true, true)],+ ),+ ],+ DataType::F16 => &[(+ TileConfiguration::new(64, 64, 16, 2, 2, 0),+ &[(true, true, true), (false, true, true)],+ )],+ DataType::F32 => &[(+ TileConfiguration::new(32, 64, 16, 1, 2, 0),+ &[(false, true, true), (true, true, true)],+ )],+ _ => return Ok(()),+ };++ for (tile, alignments) in tiles_and_alignments {+ for &(align_m, align_n, align_k) in *alignments {+ let config = PipelineConfiguration {+ tile: *tile,+ transpose_a: false,+ transpose_b: true,+ align_m,+ align_n,+ align_k,+ has_batch: false,+ use_out_source: false,+ do_axpby: false,+ };+ let _ = self.get_or_compile_pipeline(context, &config);+ }+ }++ Ok(())+ }+fn type_name(&self) -> &'static str {match self.data_type {DataType::F16 => "float16",diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex bcf872da8..b3ba3e768 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -68,6 +68,37 @@ impl Kernel {})}+ pub fn precompile(+ &mut self,+ context: &MTLContext,+ ) -> Result<(), MTLError> {+ let configs: &[(u32, bool)] = match self.data_type {+ DataType::BF16 => &[(4, false), (4, true), (8, false), (8, true)],+ DataType::F16 => &[(8, false)],+ DataType::F32 => &[(8, false)],+ _ => return Ok(()),+ };++ for &(threadgroup_rows, do_axpby) in configs {+ let config = PipelineConfiguration {+ transpose_a: false,+ transpose_b: true,+ transpose_matrix: false,+ threadgroup_rows,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ non_contiguous_batch: false,+ do_axpby,+ };+ let _ = self.get_pipeline(context, &config);+ }++ Ok(())+ }+fn get_pipeline(&mut self,context: &MTLContext,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsindex 4d84aedd4..14c6dee7c 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs@@ -41,6 +41,22 @@ impl MatmulKernel {})}+ pub fn precompile(+ &mut self,+ context: &MTLContext,+ ) -> Result<(), MTLError> {+ let gemm = self.get_or_create_gemm()?;+ gemm.precompile(context)?;++ let gemv = self.get_or_create_gemv()?;+ gemv.precompile(context)?;++ let splitk = self.get_or_create_splitk()?;+ splitk.precompile(context)?;++ Ok(())+ }+fn get_or_create_gemm(&mut self) -> Result<&mut gemm::GemmKernel, MTLError> {diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rsindex 5809da8b6..b25f2c3d1 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs@@ -45,6 +45,65 @@ impl Kernel {})}+ pub fn precompile(+ &mut self,+ context: &MTLContext,+ ) -> Result<(), MTLError> {+ use super::tile_configuration::TileConfiguration;++ if !matches!(self.data_type, DataType::BF16) {+ return Ok(());+ }++ let tile_configs = [+ TileConfiguration {+ tile_rows: 16,+ tile_cols: 16,+ tile_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ },+ TileConfiguration {+ tile_rows: 16,+ tile_cols: 32,+ tile_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ },+ TileConfiguration {+ tile_rows: 32,+ tile_cols: 16,+ tile_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ },+ TileConfiguration {+ tile_rows: 32,+ tile_cols: 32,+ tile_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ },+ ];++ for tile in &tile_configs {+ for &mn_aligned in &[false, true] {+ let config = PipelineConfiguration {+ tile: *tile,+ transpose_a: false,+ transpose_b: true,+ mn_aligned,+ k_aligned: true,+ };+ let _ = self.get_partial_pipeline(context, &config);+ }+ }++ let _ = self.get_accum_pipeline(context);++ Ok(())+ }+fn steel_type_name(&self) -> Result<&'static str, MTLError> {match self.data_type {DataType::F16 => Ok("float16"),