Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add debug matmul

Eugene Bokhan5e5cbddff0e7
diff --git a/Cargo.lock b/Cargo.lock
index 1be372c1f..0c8247c5e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3898,9 +3898,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
-version = "0.1.41"
+version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
"pin-project-lite",
"tracing-attributes",
@@ -3909,9 +3909,9 @@ dependencies = [
[[package]]
name = "tracing-attributes"
-version = "0.1.29"
+version = "0.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662"
+checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
@@ -3920,9 +3920,9 @@ dependencies = [
[[package]]
name = "tracing-core"
-version = "0.1.34"
+version = "0.1.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
dependencies = [
"once_cell",
"valuable",
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index c7e47cf0f..870597b73 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -42,7 +42,6 @@ mpsgraph = "0.2.0"
[features]
default = ["metal-shaders"]
metal-shaders = []
-tracing = []
[[test]]
name = "moe"
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
index cdaf8c0bd..e96a0ef11 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
@@ -168,7 +168,6 @@ fn select_gemv_configuration(
}
}
-
pub struct GemvKernel {
data_type: DataType,
lhs_is_transposed: bool,
@@ -431,6 +430,21 @@ impl GemvKernel {
do_axpby,
);
+ if crate::utils::env_utils::debug_matmul_enabled() {
+ let kernel_name = gemv_kernel_name(self.data_type, &configuration)
+ .unwrap_or_default();
+ eprintln!(
+ "[matmul] GEMV m={} k={} n={} batch={} dtype={:?} transpose={} kernel={}",
+ arguments.batch,
+ arguments.input_dim,
+ arguments.output_dim,
+ arguments.batch_count,
+ self.data_type,
+ configuration.transpose_matrix,
+ kernel_name
+ );
+ }
+
self.encode_with_configuration(
context,
encoder,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
index 929e6cd50..8c36bf0d8 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -17,6 +17,7 @@ use super::{
use crate::{
DataType,
backends::metal::{MTLContext, MTLError},
+ utils::env_utils::debug_matmul_enabled,
};
#[derive(Debug, Clone, Copy)]
@@ -494,13 +495,25 @@ impl MatmulKernel {
// Try GEMV fast path for decode shapes.
if self.maybe_use_gemv(mtl, enc, &args)? {
+ if debug_matmul_enabled() {
+ self.log_gemv(&args);
+ }
return Ok(());
}
// Try Split-K for small M*N and large K.
if self.maybe_use_splitk(mtl, enc, &args)? {
+ if debug_matmul_enabled() {
+ self.log_splitk(&args);
+ }
return Ok(());
}
+
+ if debug_matmul_enabled() {
+ let tile = self.select_tile_configuration(mtl, &args);
+ self.log_gemm(&args, &tile);
+ }
+
self.encode_gemm(mtl, enc, &args)
}
@@ -514,14 +527,25 @@ impl MatmulKernel {
self.apply_batch_collapse(&mut args);
if self.maybe_use_gemv_with_bias(mtl, enc, &args, bias)? {
+ if debug_matmul_enabled() {
+ self.log_gemv(&args);
+ }
return Ok(());
}
if self.maybe_use_splitk(mtl, enc, &args)? {
+ if debug_matmul_enabled() {
+ self.log_splitk(&args);
+ }
self.apply_bias_add(mtl, enc, &args, bias)?;
return Ok(());
}
+ if debug_matmul_enabled() {
+ let tile = self.select_tile_configuration(mtl, &args);
+ self.log_gemm(&args, &tile);
+ }
+
self.encode_gemm(mtl, enc, &args)?;
self.apply_bias_add(mtl, enc, &args, bias)?;
Ok(())
@@ -570,4 +594,52 @@ impl MatmulKernel {
args.batch_count = 1;
}
}
+
+ fn log_gemv(
+ &self,
+ args: &MatmulArguments,
+ ) {
+ eprintln!(
+ "[matmul] GEMV m={} k={} n={} batch={} dtype={:?}",
+ args.batch,
+ args.input_dim,
+ args.output_dim,
+ args.batch_count,
+ self.data_type
+ );
+ }
+
+ fn log_splitk(
+ &self,
+ args: &MatmulArguments,
+ ) {
+ eprintln!(
+ "[matmul] SplitK m={} k={} n={} batch={} dtype={:?}",
+ args.batch,
+ args.input_dim,
+ args.output_dim,
+ args.batch_count,
+ self.data_type
+ );
+ }
+
+ fn log_gemm(
+ &self,
+ args: &MatmulArguments,
+ tile: &TileSelection,
+ ) {
+ let kernel_name = self.kernel_name(tile);
+ eprintln!(
+ "[matmul] GEMM m={} k={} n={} batch={} dtype={:?} tile={}x{}x{} kernel={}",
+ args.batch,
+ args.input_dim,
+ args.output_dim,
+ args.batch_count,
+ self.data_type,
+ tile.block_rows,
+ tile.block_cols,
+ tile.block_depth,
+ kernel_name
+ );
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
index f468b20de..2e87f0012 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
@@ -199,6 +199,20 @@ impl QuantizedMatmulKernel {
}
})?;
+ if crate::utils::env_utils::debug_matmul_enabled() {
+ eprintln!(
+ "[qmm] {:?} m={} k={} n={} quant={:?} block={}x{} transposed={}",
+ variant,
+ args.batch,
+ args.input_dim,
+ args.output_dim,
+ args.quantization_type,
+ bm,
+ bn,
+ self.weights_transposed
+ );
+ }
+
encoder.set_compute_pipeline_state(pipeline);
// Set buffers
diff --git a/crates/uzu/src/utils/env_utils.rs b/crates/uzu/src/utils/env_utils.rs
index 27593aef8..f87bf1211 100644
--- a/crates/uzu/src/utils/env_utils.rs
+++ b/crates/uzu/src/utils/env_utils.rs
@@ -3,6 +3,7 @@ pub enum MetalEnvVar {
DeviceWrapperType,
CaptureFirstDecode,
CaptureFirstPrefill,
+ DebugMatmul,
}
impl MetalEnvVar {
@@ -11,6 +12,7 @@ impl MetalEnvVar {
MetalEnvVar::DeviceWrapperType => "METAL_DEVICE_WRAPPER_TYPE",
MetalEnvVar::CaptureFirstDecode => "UZU_CAPTURE_FIRST_DECODE",
MetalEnvVar::CaptureFirstPrefill => "UZU_CAPTURE_FIRST_PREFILL",
+ MetalEnvVar::DebugMatmul => "UZU_DEBUG_MATMUL",
}
}
@@ -23,3 +25,11 @@ impl MetalEnvVar {
matches!(upper.as_str(), "1" | "YES" | "TRUE")
}
}
+
+use std::sync::OnceLock;
+
+static DEBUG_MATMUL_ENABLED: OnceLock<bool> = OnceLock::new();
+
+pub fn debug_matmul_enabled() -> bool {
+ *DEBUG_MATMUL_ENABLED.get_or_init(|| MetalEnvVar::DebugMatmul.is_enabled())
+}