Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add precompile

Eugene Bokhand6c747403a4d
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
index 835a07c4d..ec8923202 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
@@ -69,8 +69,9 @@ impl FullPrecisionEmbeddingReadout {
let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
- let kernel =
+ let mut kernel =
MatmulKernel::new(data_type).map_err(EmbeddingError::MetalError)?;
+ kernel.precompile(_mtl_context).map_err(EmbeddingError::MetalError)?;
Ok(Self {
kernel: RefCell::new(kernel),
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
index 857b2b592..8b338d5f3 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
@@ -88,7 +88,8 @@ impl FullPrecisionLinear {
Err(_) => None,
};
- let kernel = MatmulKernel::new(precision)?;
+ let mut kernel = MatmulKernel::new(precision)?;
+ kernel.precompile(_mtl_context)?;
Ok(Self {
kernel: RefCell::new(kernel),
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index cef66acbd..e41463aa0 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -6,6 +6,7 @@ use metal::{
use super::{
DispatchDescriptor, pipeline_configuration::PipelineConfiguration,
+ tile_configuration::TileConfiguration,
};
use crate::{
DataType,
@@ -37,6 +38,64 @@ impl Kernel {
})
}
+ #[allow(clippy::type_complexity)]
+ pub fn precompile(
+ &mut self,
+ context: &MTLContext,
+ ) -> Result<(), MTLError> {
+ let tiles_and_alignments: &[(
+ TileConfiguration,
+ &[(bool, bool, bool)],
+ )] = match self.data_type {
+ DataType::BF16 => &[
+ (
+ TileConfiguration::new(64, 32, 32, 2, 2, 0),
+ &[(false, true, true), (true, true, true)],
+ ),
+ (
+ TileConfiguration::new(64, 64, 16, 2, 2, 0),
+ &[
+ (false, true, true),
+ (true, false, true),
+ (true, true, true),
+ ],
+ ),
+ (
+ TileConfiguration::new(64, 64, 16, 1, 2, 0),
+ &[(true, true, true)],
+ ),
+ ],
+ DataType::F16 => &[(
+ TileConfiguration::new(64, 64, 16, 2, 2, 0),
+ &[(true, true, true), (false, true, true)],
+ )],
+ DataType::F32 => &[(
+ TileConfiguration::new(32, 64, 16, 1, 2, 0),
+ &[(false, true, true), (true, true, true)],
+ )],
+ _ => return Ok(()),
+ };
+
+ for (tile, alignments) in tiles_and_alignments {
+ for &(align_m, align_n, align_k) in *alignments {
+ let config = PipelineConfiguration {
+ tile: *tile,
+ transpose_a: false,
+ transpose_b: true,
+ align_m,
+ align_n,
+ align_k,
+ has_batch: false,
+ use_out_source: false,
+ do_axpby: false,
+ };
+ let _ = self.get_or_compile_pipeline(context, &config);
+ }
+ }
+
+ Ok(())
+ }
+
fn type_name(&self) -> &'static str {
match self.data_type {
DataType::F16 => "float16",
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index bcf872da8..b3ba3e768 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -68,6 +68,37 @@ impl Kernel {
})
}
+ pub fn precompile(
+ &mut self,
+ context: &MTLContext,
+ ) -> Result<(), MTLError> {
+ let configs: &[(u32, bool)] = match self.data_type {
+ DataType::BF16 => &[(4, false), (4, true), (8, false), (8, true)],
+ DataType::F16 => &[(8, false)],
+ DataType::F32 => &[(8, false)],
+ _ => return Ok(()),
+ };
+
+ for &(threadgroup_rows, do_axpby) in configs {
+ let config = PipelineConfiguration {
+ transpose_a: false,
+ transpose_b: true,
+ transpose_matrix: false,
+ threadgroup_rows,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ non_contiguous_batch: false,
+ do_axpby,
+ };
+ let _ = self.get_pipeline(context, &config);
+ }
+
+ Ok(())
+ }
+
fn get_pipeline(
&mut self,
context: &MTLContext,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
index 4d84aedd4..14c6dee7c 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -41,6 +41,22 @@ impl MatmulKernel {
})
}
+ pub fn precompile(
+ &mut self,
+ context: &MTLContext,
+ ) -> Result<(), MTLError> {
+ let gemm = self.get_or_create_gemm()?;
+ gemm.precompile(context)?;
+
+ let gemv = self.get_or_create_gemv()?;
+ gemv.precompile(context)?;
+
+ let splitk = self.get_or_create_splitk()?;
+ splitk.precompile(context)?;
+
+ Ok(())
+ }
+
fn get_or_create_gemm(
&mut self
) -> Result<&mut gemm::GemmKernel, MTLError> {
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
index 5809da8b6..b25f2c3d1 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
@@ -45,6 +45,65 @@ impl Kernel {
})
}
+ pub fn precompile(
+ &mut self,
+ context: &MTLContext,
+ ) -> Result<(), MTLError> {
+ use super::tile_configuration::TileConfiguration;
+
+ if !matches!(self.data_type, DataType::BF16) {
+ return Ok(());
+ }
+
+ let tile_configs = [
+ TileConfiguration {
+ tile_rows: 16,
+ tile_cols: 16,
+ tile_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ },
+ TileConfiguration {
+ tile_rows: 16,
+ tile_cols: 32,
+ tile_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ },
+ TileConfiguration {
+ tile_rows: 32,
+ tile_cols: 16,
+ tile_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ },
+ TileConfiguration {
+ tile_rows: 32,
+ tile_cols: 32,
+ tile_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ },
+ ];
+
+ for tile in &tile_configs {
+ for &mn_aligned in &[false, true] {
+ let config = PipelineConfiguration {
+ tile: *tile,
+ transpose_a: false,
+ transpose_b: true,
+ mn_aligned,
+ k_aligned: true,
+ };
+ let _ = self.get_partial_pipeline(context, &config);
+ }
+ }
+
+ let _ = self.get_accum_pipeline(context);
+
+ Ok(())
+ }
+
fn steel_type_name(&self) -> Result<&'static str, MTLError> {
match self.data_type {
DataType::F16 => Ok("float16"),