A high-performance inference engine for AI models
add debug matmul
Eugene Bokhan5e5cbddff0e7
diff --git a/Cargo.lock b/Cargo.lockindex 1be372c1f..0c8247c5e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3898,9 +3898,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"[[package]]name = "tracing"-version = "0.1.41"+version = "0.1.44"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"dependencies = ["pin-project-lite","tracing-attributes",@@ -3909,9 +3909,9 @@ dependencies = [[[package]]name = "tracing-attributes"-version = "0.1.29"+version = "0.1.31"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662"+checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"dependencies = ["proc-macro2","quote",@@ -3920,9 +3920,9 @@ dependencies = [[[package]]name = "tracing-core"-version = "0.1.34"+version = "0.1.36"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"dependencies = ["once_cell","valuable",diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex c7e47cf0f..870597b73 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -42,7 +42,6 @@ mpsgraph = "0.2.0"[features]default = ["metal-shaders"]metal-shaders = []-tracing = [][[test]]name = "moe"diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rsindex cdaf8c0bd..e96a0ef11 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs@@ -168,7 +168,6 @@ fn select_gemv_configuration(}}-pub struct GemvKernel {data_type: DataType,lhs_is_transposed: bool,@@ -431,6 +430,21 @@ impl GemvKernel {do_axpby,);+ if crate::utils::env_utils::debug_matmul_enabled() {+ let kernel_name = gemv_kernel_name(self.data_type, &configuration)+ .unwrap_or_default();+ eprintln!(+ "[matmul] GEMV m={} k={} n={} batch={} dtype={:?} transpose={} kernel={}",+ arguments.batch,+ arguments.input_dim,+ arguments.output_dim,+ arguments.batch_count,+ self.data_type,+ configuration.transpose_matrix,+ kernel_name+ );+ }+self.encode_with_configuration(context,encoder,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsindex 929e6cd50..8c36bf0d8 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs@@ -17,6 +17,7 @@ use super::{use crate::{DataType,backends::metal::{MTLContext, MTLError},+ utils::env_utils::debug_matmul_enabled,};#[derive(Debug, Clone, Copy)]@@ -494,13 +495,25 @@ impl MatmulKernel {// Try GEMV fast path for decode shapes.if self.maybe_use_gemv(mtl, enc, &args)? {+ if debug_matmul_enabled() {+ self.log_gemv(&args);+ }return Ok(());}// Try Split-K for small M*N and large K.if self.maybe_use_splitk(mtl, enc, &args)? {+ if debug_matmul_enabled() {+ self.log_splitk(&args);+ }return Ok(());}++ if debug_matmul_enabled() {+ let tile = self.select_tile_configuration(mtl, &args);+ self.log_gemm(&args, &tile);+ }+self.encode_gemm(mtl, enc, &args)}@@ -514,14 +527,25 @@ impl MatmulKernel {self.apply_batch_collapse(&mut args);if self.maybe_use_gemv_with_bias(mtl, enc, &args, bias)? {+ if debug_matmul_enabled() {+ self.log_gemv(&args);+ }return Ok(());}if self.maybe_use_splitk(mtl, enc, &args)? {+ if debug_matmul_enabled() {+ self.log_splitk(&args);+ }self.apply_bias_add(mtl, enc, &args, bias)?;return Ok(());}+ if debug_matmul_enabled() {+ let tile = self.select_tile_configuration(mtl, &args);+ self.log_gemm(&args, &tile);+ }+self.encode_gemm(mtl, enc, &args)?;self.apply_bias_add(mtl, enc, &args, bias)?;Ok(())@@ -570,4 +594,52 @@ impl MatmulKernel {args.batch_count = 1;}}++ fn log_gemv(+ &self,+ args: &MatmulArguments,+ ) {+ eprintln!(+ "[matmul] GEMV m={} k={} n={} batch={} dtype={:?}",+ args.batch,+ args.input_dim,+ args.output_dim,+ args.batch_count,+ self.data_type+ );+ }++ fn log_splitk(+ &self,+ args: &MatmulArguments,+ ) {+ eprintln!(+ "[matmul] SplitK m={} k={} n={} batch={} dtype={:?}",+ args.batch,+ args.input_dim,+ args.output_dim,+ args.batch_count,+ self.data_type+ );+ }++ fn log_gemm(+ &self,+ args: &MatmulArguments,+ tile: &TileSelection,+ ) {+ let kernel_name = self.kernel_name(tile);+ eprintln!(+ "[matmul] GEMM m={} k={} n={} batch={} dtype={:?} tile={}x{}x{} kernel={}",+ args.batch,+ args.input_dim,+ args.output_dim,+ args.batch_count,+ self.data_type,+ tile.block_rows,+ tile.block_cols,+ tile.block_depth,+ kernel_name+ );+ }}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rsindex f468b20de..2e87f0012 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs@@ -199,6 +199,20 @@ impl QuantizedMatmulKernel {}})?;+ if crate::utils::env_utils::debug_matmul_enabled() {+ eprintln!(+ "[qmm] {:?} m={} k={} n={} quant={:?} block={}x{} transposed={}",+ variant,+ args.batch,+ args.input_dim,+ args.output_dim,+ args.quantization_type,+ bm,+ bn,+ self.weights_transposed+ );+ }+encoder.set_compute_pipeline_state(pipeline);// Set buffersdiff --git a/crates/uzu/src/utils/env_utils.rs b/crates/uzu/src/utils/env_utils.rsindex 27593aef8..f87bf1211 100644--- a/crates/uzu/src/utils/env_utils.rs+++ b/crates/uzu/src/utils/env_utils.rs@@ -3,6 +3,7 @@ pub enum MetalEnvVar {DeviceWrapperType,CaptureFirstDecode,CaptureFirstPrefill,+ DebugMatmul,}impl MetalEnvVar {@@ -11,6 +12,7 @@ impl MetalEnvVar {MetalEnvVar::DeviceWrapperType => "METAL_DEVICE_WRAPPER_TYPE",MetalEnvVar::CaptureFirstDecode => "UZU_CAPTURE_FIRST_DECODE",MetalEnvVar::CaptureFirstPrefill => "UZU_CAPTURE_FIRST_PREFILL",+ MetalEnvVar::DebugMatmul => "UZU_DEBUG_MATMUL",}}@@ -23,3 +25,11 @@ impl MetalEnvVar {matches!(upper.as_str(), "1" | "YES" | "TRUE")}}++use std::sync::OnceLock;++static DEBUG_MATMUL_ENABLED: OnceLock<bool> = OnceLock::new();++pub fn debug_matmul_enabled() -> bool {+ *DEBUG_MATMUL_ENABLED.get_or_init(|| MetalEnvVar::DebugMatmul.is_enabled())+}