A high-performance inference engine for AI models
small matmul cleanup and new api for lora (#296)
uuuvn028fb04b988d
diff --git a/Cargo.lock b/Cargo.lockindex cc1ddc0cc..6b09e1746 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -15,6 +15,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"dependencies = ["cfg-if",+ "const-random","getrandom 0.3.4","once_cell","serde",@@ -658,6 +659,26 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "const-random"+version = "0.1.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"+dependencies = [+ "const-random-macro",+]++[[package]]+name = "const-random-macro"+version = "0.1.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"+dependencies = [+ "getrandom 0.2.17",+ "once_cell",+ "tiny-keccak",+]+[[package]]name = "constant_time_eq"version = "0.4.2"@@ -2531,6 +2552,9 @@ name = "once_cell"version = "1.21.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"+dependencies = [+ "portable-atomic",+][[package]]name = "once_cell_polyfill"@@ -3169,6 +3193,34 @@ dependencies = ["web-sys",]+[[package]]+name = "rhai"+version = "1.24.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1f9ef5dabe4c0b43d8f1187dc6beb67b53fe607fff7e30c5eb7f71b814b8c2c1"+dependencies = [+ "ahash",+ "bitflags",+ "num-traits",+ "once_cell",+ "rhai_codegen",+ "smallvec",+ "smartstring",+ "thin-vec",+ "web-time",+]++[[package]]+name = "rhai_codegen"+version = "3.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d4322a2a4e8cf30771dd9f27f7f37ca9ac8fe812dddd811096a98483080dabe6"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.117",+]+[[package]]name = "ring"version = "0.17.14"@@ -3612,6 +3664,17 @@ version = "1.15.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"+[[package]]+name = "smartstring"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"+dependencies = [+ "autocfg",+ "static_assertions",+ "version_check",+]+[[package]]name = "socket2"version = "0.5.10"@@ -3813,6 +3876,12 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "thin-vec"+version = "0.2.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "144f754d318415ac792f9d69fc87abbbfc043ce2ef041c60f16ad828f638717d"+[[package]]name = "thiserror"version = "1.0.69"@@ -3893,6 +3962,15 @@ dependencies = ["time-core",]+[[package]]+name = "tiny-keccak"+version = "2.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"+dependencies = [+ "crunchy",+]+[[package]]name = "tinystr"version = "0.8.2"@@ -4409,6 +4487,7 @@ dependencies = ["rand_distr","rangemap","regex",+ "rhai","schemars","serde","serde_json",diff --git a/Cargo.toml b/Cargo.tomlindex f9eec995e..4894e1bce 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -58,6 +58,7 @@ quote = "1.0"proc-macro2 = "1.0"syn = "2.0"prettyplease = "0.2"+rhai = "1.24.0"mach2 = "0.6"cbindgen = "0.29"ctrlc = "3.5"diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 873f2595d..cfe806582 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -71,12 +71,14 @@ cbindgen.workspace = true# metal on macos requires clang-ast for dsl build[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.build-dependencies]clang-ast = { workspace = true, optional = true }+rhai = { workspace = true, optional = true }[features]default = ["metal"]metal = ["dep:clang-ast",+ "dep:rhai","dep:objc2","dep:objc2-foundation","dep:bytesize",diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rsindex 8bbc95f8b..c8e789d6a 100644--- a/crates/uzu/build/metal/ast.rs+++ b/crates/uzu/build/metal/ast.rs@@ -327,6 +327,7 @@ pub struct MetalKernelInfo {pub name: Box<str>,pub arguments: Box<[MetalArgument]>,pub variants: Option<Box<[MetalTemplateParameter]>>,+ pub constraints: Box<[Box<str>]>,}impl MetalKernelInfo {@@ -543,6 +544,18 @@ impl MetalKernelInfo {None};+ let constraints: Box<[_]> = annotations+ .iter()+ .filter(|(k, _)| k.as_ref() == "dsl.constraint")+ .map(|(_, v)| {+ let [constraint_expr] = v.as_ref() else {+ bail!("malformed dsl.constraint annotation");+ };++ Ok(constraint_expr.clone())+ })+ .collect::<anyhow::Result<_>>()?;+let arguments = arg_nodes.into_iter().map(|an| MetalArgument::from_ast_node_and_source(an, source))@@ -553,6 +566,7 @@ impl MetalKernelInfo {name,arguments,variants,+ constraints,}))}}diff --git a/crates/uzu/build/metal/wrapper.rs b/crates/uzu/build/metal/wrapper.rsindex 6e63c71df..32e781924 100644--- a/crates/uzu/build/metal/wrapper.rs+++ b/crates/uzu/build/metal/wrapper.rs@@ -82,6 +82,7 @@ fn kernel_wrappers(Vec::new()};+ let engine = rhai::Engine::new();for type_variant in if let Some(variants) = &kernel.variants {variants.iter()@@ -100,6 +101,25 @@ fn kernel_wrappers(} else {vec![None]} {+ if let Some(type_variant) = &type_variant {+ let mut scope = rhai::Scope::with_capacity(type_variant.len());+ for (type_variant_name, type_variant_value) in type_variant {+ scope.push(+ type_variant_name.clone(),+ engine+ .eval_expression::<rhai::Dynamic>(type_variant_value.as_ref())+ .unwrap_or_else(|_| type_variant_value.clone().into()),+ );+ }+ if !kernel+ .constraints+ .iter()+ .all(|constraint_expr| engine.eval_expression_with_scope(&mut scope, constraint_expr.as_ref()).unwrap())+ {+ continue;+ }+ }+let (wrapper_name, underlying_name) = if let Some(type_variant) = &type_variant {(static_mangle(kernel.name.as_ref(), type_variant.iter().map(|(_k, v)| v.as_str())),diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rsindex e0f18747a..5bcd36c47 100644--- a/crates/uzu/src/backends/common/backend.rs+++ b/crates/uzu/src/backends/common/backend.rs@@ -1,13 +1,13 @@use std::{error::Error, fmt::Debug};-use super::{Buffer, CommandBuffer, Context, Event, Kernels, kernel::matmul::MatmulKernels};+use super::{Buffer, CommandBuffer, Context, Event, Kernels, kernel::ManualKernels};pub trait Backend: Debug + Clone + 'static {type Context: Context<Backend = Self>;type CommandBuffer: CommandBuffer<Backend = Self>;type Buffer: Buffer<Backend = Self>;type Event: Event<Backend = Self>;- type Kernels: Kernels<Backend = Self> + MatmulKernels;+ type Kernels: Kernels<Backend = Self> + ManualKernels;type Error: Error + Debug;const MIN_ALLOCATION_ALIGNMENT: usize;diff --git a/crates/uzu/src/backends/common/gpu_types/matmul.rs b/crates/uzu/src/backends/common/gpu_types/matmul.rsindex 96713f917..a9c496065 100644--- a/crates/uzu/src/backends/common/gpu_types/matmul.rs+++ b/crates/uzu/src/backends/common/gpu_types/matmul.rs@@ -3,14 +3,14 @@#[repr(C)]#[derive(Debug, Default, Copy, Clone)]pub struct GemmParams {- pub M: i32,- pub N: i32,- pub K: i32,- pub leading_dimension_a: i32,- pub leading_dimension_b: i32,- pub leading_dimension_d: i32,- pub threadgroups_per_column: i32,- pub threadgroups_per_row: i32,- pub swizzle_log: i32,- pub aligned_inner_iterations: i32,+ pub M: u32,+ pub N: u32,+ pub K: u32,+ pub leading_dimension_a: u32,+ pub leading_dimension_b: u32,+ pub leading_dimension_d: u32,+ pub threadgroups_per_column: u32,+ pub threadgroups_per_row: u32,+ pub swizzle_log: u32,+ pub aligned_inner_iterations: u32,}diff --git a/crates/uzu/src/backends/common/kernel/matmul.rs b/crates/uzu/src/backends/common/kernel/matmul.rsnew file mode 100644index 000000000..ce5b3df03--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul.rs@@ -0,0 +1,61 @@+use thiserror::Error;++use crate::{+ DataType,+ backends::common::{Backend, Encoder, kernel::ManualKernels},+};++#[derive(Debug, Error)]+pub enum MatmulError<B: Backend> {+ #[error("Unsupported data type: {0:?}")]+ UnsupportedDataType(DataType),+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+}++#[derive(Debug)]+pub enum MatmulArgumentC<'a, B: Backend> {+ None,+ /// Accumulate: [M, N]+ Accumulate,+ /// Bias: [N] (broadcasted across M/batch)+ Bias(&'a B::Buffer),+}++// D = ab_scale * (A @ B.T) + C+#[derive(Debug)]+pub struct MatmulArguments<'a, B: Backend> {+ /// A: [M, K]+ pub a: &'a B::Buffer,+ pub a_offset: u64,+ /// B: [N, K]+ pub b: &'a B::Buffer,+ /// AB scale: also known as alpha+ pub ab_scale: f32,+ /// C: behavior depends on enum variant+ pub c: MatmulArgumentC<'a, B>,+ /// D: [M, N]+ pub d: &'a mut B::Buffer,+ /// M dimension: usually batch/number of tokens (rows of A, rows of C)+ pub batch_dim: u32,+ /// K dimension: usually input_dim/reduction dimension (cols of A, rows of B)+ pub input_dim: u32,+ /// N dimension: usually output_dim (cols of B, cols of C)+ pub output_dim: u32,+}++pub trait MatmulKernel: Sized {+ type Backend: Backend<Kernels: ManualKernels<MatmulKernel = Self>>;++ fn new(+ context: &<Self::Backend as Backend>::Context,+ data_type: DataType,+ ) -> Result<Self, MatmulError<Self::Backend>>;++ fn encode(+ &mut self,+ context: &<Self::Backend as Backend>::Context,+ arguments: MatmulArguments<Self::Backend>,+ encoder: &mut Encoder<Self::Backend>,+ );+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rsdeleted file mode 100644index a432f4e0b..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs+++ /dev/null@@ -1,21 +0,0 @@-use crate::backends::common::Backend;--#[derive(Debug)]-pub struct MatmulArguments<'a, B: Backend> {- pub a: &'a B::Buffer,- /// Byte offset into `a` (used for slicing the batch dimension).- pub a_offset: u64,- pub b: &'a B::Buffer,- pub d: &'a mut B::Buffer,- pub bias: Option<&'a B::Buffer>,- /// M dimension - batch/number of tokens (rows of A, rows of D)- pub batch: i32,- /// K dimension - input_dim/reduction dimension (cols of A, rows of B)- pub input_dim: i32,- /// N dimension - output_dim (cols of B, cols of D)- pub output_dim: i32,- pub leading_dimension_a: i32,- pub leading_dimension_b: i32,- pub leading_dimension_d: i32,- pub transpose_b: bool,-}diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rsdeleted file mode 100644index 78a99f7d8..000000000--- a/crates/uzu/src/backends/common/kernel/matmul/mod.rs+++ /dev/null@@ -1,40 +0,0 @@-mod matmul_arguments;--pub use matmul_arguments::MatmulArguments;-use thiserror::Error;--use super::Kernels;-use crate::{- DataType,- backends::common::{Backend, Encoder},-};--pub trait MatmulKernels: Kernels {- type MatmulKernel: MatmulKernel<Backend = Self::Backend>;-}--pub trait MatmulKernel: Sized {- type Backend: Backend<Kernels: MatmulKernels<MatmulKernel = Self>>;-- fn new(- context: &<Self::Backend as Backend>::Context,- data_type: DataType,- ) -> Result<Self, MatmulError<Self::Backend>>;-- fn encode(- &mut self,- context: &<Self::Backend as Backend>::Context,- arguments: MatmulArguments<Self::Backend>,- encoder: &mut Encoder<Self::Backend>,- );-}--#[derive(Debug, Error)]-pub enum MatmulError<B: Backend> {- #[error("Unsupported data type: {0:?}")]- UnsupportedDataType(DataType),- #[error("Threadgroup dimension overflows u32: {0}")]- ThreadgroupOverflow(usize),- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),-}diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rsindex b5a284288..a2659f323 100644--- a/crates/uzu/src/backends/common/kernel/mod.rs+++ b/crates/uzu/src/backends/common/kernel/mod.rs@@ -2,6 +2,10 @@include!(concat!(env!("OUT_DIR"), "/traits.rs"));+pub trait ManualKernels: Kernels {+ type MatmulKernel: matmul::MatmulKernel<Backend = Self::Backend>;+}+pub mod attention;pub mod kv_cache_update;pub mod matmul;diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rsindex 8c71e6dbf..eb9fe033b 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs@@ -9,6 +9,7 @@ use crate::{unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(a: SendPtr<T>,b: SendPtr<T>,+ ab_scale: f32,d: SendPtrMut<T>,row: usize,n: usize,@@ -16,6 +17,7 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(leading_dimension_a: usize,leading_dimension_b: usize,leading_dimension_d: usize,+ is_accumulate: bool,) {unsafe {for col in 0..n {@@ -25,7 +27,12 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(let b_val = *b.0.add(col * leading_dimension_b + i);acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();}- *d.0.add(row * leading_dimension_d + col) = T::from(acc).unwrap();+ let d_element = d.0.add(row * leading_dimension_d + col);+ acc *= ab_scale;+ if is_accumulate {+ acc += (*d_element).to_f32().unwrap();+ }+ *d_element = T::from(acc).unwrap();}}}@@ -33,6 +40,7 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(pub fn matmul_gemm_impl<T: ArrayElement + Float>(a: *const T,b: *const T,+ ab_scale: f32,d: *mut T,m: usize,n: usize,@@ -40,6 +48,7 @@ pub fn matmul_gemm_impl<T: ArrayElement + Float>(leading_dimension_a: usize,leading_dimension_b: usize,leading_dimension_d: usize,+ is_accumulate: bool,) {let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);let a = SendPtr(a);@@ -66,6 +75,7 @@ pub fn matmul_gemm_impl<T: ArrayElement + Float>(matmul_gemm_compute_row(a,b,+ ab_scale,d,row,n,@@ -73,6 +83,7 @@ pub fn matmul_gemm_impl<T: ArrayElement + Float>(leading_dimension_a,leading_dimension_b,leading_dimension_d,+ is_accumulate,);}});diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rsindex 3991c4f8a..99213a5b7 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -5,7 +5,7 @@ use crate::{backends::{common::{Encoder,- kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},+ kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},},cpu::{Cpu, context::CpuContext},},@@ -37,25 +37,32 @@ impl MatmulKernel for MatmulCpuKernel {encoder: &mut Encoder<Cpu>,) {let command_buffer = encoder.as_command_buffer_mut();- let m = arguments.batch as usize;+ let m = arguments.batch_dim as usize;let n = arguments.output_dim as usize;let k = arguments.input_dim as usize;- let lda = arguments.leading_dimension_a as usize;- let ldb = arguments.leading_dimension_b as usize;- let ldd = arguments.leading_dimension_d as usize;+ let lda = k;+ let ldb = k;+ let ldd = n;let a_offset = arguments.a_offset as usize;let data_type = self.data_type;let a_ptr = unsafe { &*arguments.a.get() }.as_ptr().wrapping_byte_add(a_offset);let b_ptr = unsafe { &*arguments.b.get() }.as_ptr();+ let ab_scale = arguments.ab_scale;let d_ptr = arguments.d.get_mut().as_mut_ptr();- let bias_ptr = arguments.bias.map(|b| unsafe { &*b.get() }.as_ptr());++ let (is_accumulate, bias_ptr) = match arguments.c {+ MatmulArgumentC::Accumulate => (true, None),+ MatmulArgumentC::Bias(bias) => (false, Some(unsafe { &*bias.get() }.as_ptr())),+ MatmulArgumentC::None => (false, None),+ };command_buffer.push_command(move || match data_type {DataType::F32 => {super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f32>(a_ptr as *const f32,b_ptr as *const f32,+ ab_scale,d_ptr as *mut f32,m,n,@@ -63,6 +70,7 @@ impl MatmulKernel for MatmulCpuKernel {lda,ldb,ldd,+ is_accumulate,);if let Some(bias) = bias_ptr {apply_bias::<f32>(d_ptr as *mut f32, bias as *const f32, m, n);@@ -72,6 +80,7 @@ impl MatmulKernel for MatmulCpuKernel {super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f16>(a_ptr as *const f16,b_ptr as *const f16,+ ab_scale,d_ptr as *mut f16,m,n,@@ -79,6 +88,7 @@ impl MatmulKernel for MatmulCpuKernel {lda,ldb,ldd,+ is_accumulate,);if let Some(bias) = bias_ptr {apply_bias::<f16>(d_ptr as *mut f16, bias as *const f16, m, n);@@ -88,6 +98,7 @@ impl MatmulKernel for MatmulCpuKernel {super::gemm::shaders::steel_gemm::matmul_gemm_impl::<bf16>(a_ptr as *const bf16,b_ptr as *const bf16,+ ab_scale,d_ptr as *mut bf16,m,n,@@ -95,6 +106,7 @@ impl MatmulKernel for MatmulCpuKernel {lda,ldb,ldd,+ is_accumulate,);if let Some(bias) = bias_ptr {apply_bias::<bf16>(d_ptr as *mut bf16, bias as *const bf16, m, n);diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rsindex 23d8c2f63..bbdde8210 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs@@ -4,8 +4,8 @@ mod matmul;pub use matmul::MatmulCpuKernel;-use crate::backends::{common::kernel::matmul::MatmulKernels, cpu::kernel::CpuKernels};+use crate::backends::{common::kernel::ManualKernels, cpu::kernel::CpuKernels};-impl MatmulKernels for CpuKernels {+impl ManualKernels for CpuKernels {type MatmulKernel = MatmulCpuKernel;}diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rsindex bd242efb9..03fa26a94 100644--- a/crates/uzu/src/backends/metal/backend.rs+++ b/crates/uzu/src/backends/metal/backend.rs@@ -1,7 +1,7 @@use metal::{MTLBuffer, MTLEvent};use objc2::{rc::Retained, runtime::ProtocolObject};-use super::{command_buffer::MetalCommandBuffer, context::MetalContext, error::MetalError, kernel::dsl::MetalKernels};+use super::{command_buffer::MetalCommandBuffer, context::MetalContext, error::MetalError, kernel::MetalKernels};use crate::backends::common::Backend;#[derive(Debug, Clone)]diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/generated/matmul.hindex 86d45c6ec..c3fdf827e 100644--- a/crates/uzu/src/backends/metal/generated/matmul.h+++ b/crates/uzu/src/backends/metal/generated/matmul.h@@ -15,16 +15,16 @@ namespace matmul {#endiftypedef struct {- int32_t M;- int32_t N;- int32_t K;- int32_t leading_dimension_a;- int32_t leading_dimension_b;- int32_t leading_dimension_d;- int32_t threadgroups_per_column;- int32_t threadgroups_per_row;- int32_t swizzle_log;- int32_t aligned_inner_iterations;+ uint32_t M;+ uint32_t N;+ uint32_t K;+ uint32_t leading_dimension_a;+ uint32_t leading_dimension_b;+ uint32_t leading_dimension_d;+ uint32_t threadgroups_per_column;+ uint32_t threadgroups_per_row;+ uint32_t swizzle_log;+ uint32_t aligned_inner_iterations;} GemmParams;#ifdef __METAL_VERSION__diff --git a/crates/uzu/src/backends/metal/kernel/common/dsl.h b/crates/uzu/src/backends/metal/kernel/common/dsl.hindex bae47519e..3bda99637 100644--- a/crates/uzu/src/backends/metal/kernel/common/dsl.h+++ b/crates/uzu/src/backends/metal/kernel/common/dsl.h@@ -20,6 +20,7 @@ using namespace metal;#define VARIANTS(TYPENAME, ...) \DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)+#define CONSTRAINT(C) DSL_META("dsl.constraint", #C)#define PUBLIC DSL_META("dsl.public")#define KERNEL(NAME) DSL_META("dsl.kernel") void NAMEdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.hindex 4e4401177..c441e1f58 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h@@ -68,7 +68,8 @@ struct TransformScaleAccumulate {METAL_FUNC OutT apply(InT x, OutT c) const {return static_cast<OutT>(- x * static_cast<InT>(alpha) + (static_cast<OutT>(beta) * c)+ x * static_cast<InT>(alpha) ++ (beta != 0.0f ? (static_cast<OutT>(beta) * c) : static_cast<OutT>(0.0)));}};diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.hindex 8cee3d832..88130e664 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h@@ -150,6 +150,8 @@ struct ThreadgroupGemm {const device T* b,device U* d,const constant GemmParams* params,+ const constant float& ab_scale,+ const bool is_accumulate,threadgroup T* a_shared,threadgroup T* b_shared,uint simd_lane_id [[thread_index_in_simdgroup]],@@ -199,6 +201,11 @@ struct ThreadgroupGemm {thread ThreadgroupTileType threadgroup_tile(simd_group_id, simd_lane_id);+ TransformScaleAccumulate<AccumType, AccumType> epilogue(+ ab_scale,+ is_accumulate ? 1.0f : 0.0f+ );+int gemm_k_iterations = params->aligned_inner_iterations;///////////////////////////////////////////////////////////////////////////@@ -237,14 +244,18 @@ struct ThreadgroupGemm {threadgroup_tile.multiply_accumulate(a_shared, b_shared);}+ threadgroup_tile+ .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);threadgroup_tile.store_result(d, params->leading_dimension_d);return;}///////////////////////////////////////////////////////////////////////////// MN unaligned loopelse {- short threadgroup_block_rows = min(BLOCK_ROWS, params->M - output_row);- short threadgroup_block_cols = min(BLOCK_COLS, params->N - output_col);+ short threadgroup_block_rows =+ min(BLOCK_ROWS, ((int)params->M) - output_row);+ short threadgroup_block_cols =+ min(BLOCK_COLS, ((int)params->N) - output_col);short leftover_block_depth =params->K - params->aligned_inner_iterations * BLOCK_DEPTH;@@ -262,6 +273,8 @@ struct ThreadgroupGemm {leftover_block_depth);+ threadgroup_tile+ .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);threadgroup_tile.store_result(d, params->leading_dimension_d);return;@@ -278,6 +291,13 @@ struct ThreadgroupGemm {leftover_block_depth);+ threadgroup_tile.apply_epilogue_safe(+ d,+ params->leading_dimension_d,+ 1,+ short2(threadgroup_block_cols, threadgroup_block_rows),+ epilogue+ );threadgroup_tile.store_result_safe(d,params->leading_dimension_d,@@ -298,6 +318,13 @@ struct ThreadgroupGemm {leftover_block_depth);+ threadgroup_tile.apply_epilogue_safe(+ d,+ params->leading_dimension_d,+ 1,+ short2(threadgroup_block_cols, threadgroup_block_rows),+ epilogue+ );threadgroup_tile.store_result_safe(d,params->leading_dimension_d,@@ -318,6 +345,13 @@ struct ThreadgroupGemm {leftover_block_depth);+ threadgroup_tile.apply_epilogue_safe(+ d,+ params->leading_dimension_d,+ 1,+ short2(threadgroup_block_cols, threadgroup_block_rows),+ epilogue+ );threadgroup_tile.store_result_safe(d,params->leading_dimension_d,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm.metalnew file mode 100644index 000000000..9f6d548fb--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal@@ -0,0 +1,65 @@+#include "../common/utils.h"+#include "../common/dsl.h"+#include "../common/thread_context.h"++#include "common/gemm.h"++using namespace uzu::matmul;++#define GEMM_MAX_THREADGROUP_A 2560+#define GEMM_MAX_THREADGROUP_B 1536++template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint BLOCK_DEPTH, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN, bool MN_ALIGNED, bool K_ALIGNED>+VARIANTS(T, float, half, bfloat)+VARIANTS(BLOCK_ROWS, 32, 64)+VARIANTS(BLOCK_COLS, 32, 64)+VARIANTS(BLOCK_DEPTH, 8, 16, 32)+VARIANTS(SIMDGROUPS_PER_ROW, 1, 2, 4)+VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2)+VARIANTS(MN_ALIGNED, false, true)+VARIANTS(K_ALIGNED, false, true)+CONSTRAINT(max(BLOCK_ROWS, BLOCK_COLS) <= 32 * SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN)+KERNEL(MatmulGemm)(+ const device T* a,+ const device T* b,+ device T* d,+ const constant uzu::matmul::GemmParams* params,+ const constant uint& group_count_x,+ const constant uint& group_count_y,+ const constant float& ab_scale,+ const bool is_accumulate SPECIALIZE,+ threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],+ threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],+ const uint group_x GROUPS(group_count_x),+ const uint group_y GROUPS(group_count_y),+ const uint thread_x THREADS(32),+ const uint thread_y THREADS(2),+ const uint thread_z THREADS(2),+ const ThreadContext thread_context+) {+ ThreadgroupGemm<+ T,+ T,+ BLOCK_ROWS,+ BLOCK_COLS,+ BLOCK_DEPTH,+ SIMDGROUPS_PER_ROW,+ SIMDGROUPS_PER_COLUMN,+ false, // transpose_a+ true, // transpose_b+ MN_ALIGNED,+ K_ALIGNED,+ float>::+ run(a,+ b,+ d,+ params,+ ab_scale,+ is_accumulate,+ a_shared,+ b_shared,+ thread_context.simdgroup_index,+ thread_context.threadgroup_index,+ uint2(group_x, group_y),+ uint3(thread_x, thread_y, thread_z));+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm.rssimilarity index 57%rename from crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsrename to crates/uzu/src/backends/metal/kernel/matmul/gemm.rsindex 36e9db557..1d31ba2cd 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm.rs@@ -1,22 +1,24 @@use crate::{DataType,backends::{- common::{Context, kernel::matmul::MatmulArguments},+ common::{+ Context,+ kernel::matmul::{MatmulArgumentC, MatmulArguments},+ },metal::{Metal, context::MetalContext},},};#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct GemmSpecialization {- pub block_rows: i32,- pub block_cols: i32,- pub block_depth: i32,- pub simdgroups_per_row: u64,- pub simdgroups_per_column: u64,- pub swizzle_log2: i32,- pub align_m: bool,- pub align_n: bool,+ pub block_rows: u32,+ pub block_cols: u32,+ pub block_depth: u32,+ pub simdgroups_per_row: u32,+ pub simdgroups_per_column: u32,+ pub align_mn: bool,pub align_k: bool,+ pub is_accumulate: bool,}impl GemmSpecialization {@@ -29,10 +31,9 @@ impl GemmSpecialization {block_depth: 32,simdgroups_per_row: 2,simdgroups_per_column: 2,- swizzle_log2: 0,- align_m: false,- align_n: true,+ align_mn: false,align_k: true,+ is_accumulate: false,},Self {block_rows: 64,@@ -40,10 +41,9 @@ impl GemmSpecialization {block_depth: 32,simdgroups_per_row: 2,simdgroups_per_column: 2,- swizzle_log2: 0,- align_m: true,- align_n: true,+ align_mn: true,align_k: true,+ is_accumulate: false,},Self {block_rows: 64,@@ -51,10 +51,9 @@ impl GemmSpecialization {block_depth: 16,simdgroups_per_row: 2,simdgroups_per_column: 2,- swizzle_log2: 0,- align_m: false,- align_n: true,+ align_mn: false,align_k: true,+ is_accumulate: false,},Self {block_rows: 64,@@ -62,21 +61,9 @@ impl GemmSpecialization {block_depth: 16,simdgroups_per_row: 2,simdgroups_per_column: 2,- swizzle_log2: 0,- align_m: true,- align_n: false,- align_k: true,- },- Self {- block_rows: 64,- block_cols: 64,- block_depth: 16,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- swizzle_log2: 0,- align_m: true,- align_n: true,+ align_mn: true,align_k: true,+ is_accumulate: false,},Self {block_rows: 64,@@ -84,10 +71,9 @@ impl GemmSpecialization {block_depth: 16,simdgroups_per_row: 1,simdgroups_per_column: 2,- swizzle_log2: 0,- align_m: true,- align_n: true,+ align_mn: true,align_k: true,+ is_accumulate: false,},],DataType::F16 => &[@@ -97,10 +83,9 @@ impl GemmSpecialization {block_depth: 16,simdgroups_per_row: 2,simdgroups_per_column: 2,- swizzle_log2: 0,- align_m: true,- align_n: true,+ align_mn: true,align_k: true,+ is_accumulate: false,},Self {block_rows: 64,@@ -108,10 +93,9 @@ impl GemmSpecialization {block_depth: 16,simdgroups_per_row: 2,simdgroups_per_column: 2,- swizzle_log2: 0,- align_m: false,- align_n: true,+ align_mn: false,align_k: true,+ is_accumulate: false,},],DataType::F32 => &[@@ -121,10 +105,9 @@ impl GemmSpecialization {block_depth: 16,simdgroups_per_row: 2,simdgroups_per_column: 2,- swizzle_log2: 0,- align_m: false,- align_n: true,+ align_mn: false,align_k: true,+ is_accumulate: false,},Self {block_rows: 32,@@ -132,10 +115,9 @@ impl GemmSpecialization {block_depth: 16,simdgroups_per_row: 2,simdgroups_per_column: 2,- swizzle_log2: 0,- align_m: true,- align_n: true,+ align_mn: true,align_k: true,+ is_accumulate: false,},],_ => &[],@@ -147,51 +129,36 @@ impl GemmSpecialization {data_type: DataType,arguments: &MatmulArguments<Metal>,) -> Self {- let overall_work_elements = (arguments.batch as i64) * (arguments.output_dim as i64);+ let overall_work_elements = arguments.batch_dim * arguments.output_dim;let is_float32 = matches!(data_type, DataType::F32);let prefer_half_or_tf32 = !is_float32 || MetalContext::tf32_enabled();- let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column, swizzle_log2) =- if context.is_high_performance() && overall_work_elements >= (1_i64 << 20) {+ let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column) =+ if context.is_high_performance() && overall_work_elements >= (1 << 20) {if prefer_half_or_tf32 {- if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {- (64, 64, 16, 2, 2, 0)- } else if arguments.transpose_b {- (64, 32, 32, 2, 2, 0)+ if 2 * std::cmp::max(arguments.batch_dim, arguments.output_dim) > arguments.input_dim {+ (64, 64, 16, 2, 2)} else {- (32, 64, 16, 2, 2, 0)+ (64, 32, 32, 2, 2)}- } else if arguments.transpose_b {- (32, 64, 16, 2, 2, 0)} else {- (64, 32, 32, 2, 2, 0)+ (32, 64, 16, 2, 2)}} else if prefer_half_or_tf32 {- if arguments.transpose_b {- (64, 32, 32, 2, 2, 0)- } else {- (64, 64, 16, 2, 2, 0)- }- } else if arguments.transpose_b {- (32, 64, 16, 2, 2, 0)+ (64, 32, 32, 2, 2)} else {- (64, 32, 32, 2, 2, 0)+ (32, 64, 16, 2, 2)};- let m = arguments.batch;- let n = arguments.output_dim;- let k = arguments.input_dim;-Self {block_rows,block_cols,block_depth,simdgroups_per_row,simdgroups_per_column,- swizzle_log2,- align_m: (m % block_rows) == 0,- align_n: (n % block_cols) == 0,- align_k: (k % block_depth) == 0,+ align_mn: (arguments.batch_dim % block_rows) == 0 && (arguments.output_dim % block_cols) == 0,+ align_k: (arguments.input_dim % block_depth) == 0,+ is_accumulate: matches!(arguments.c, MatmulArgumentC::Accumulate),}}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rsdeleted file mode 100644index 14f9855ce..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs+++ /dev/null@@ -1,3 +0,0 @@-mod specialization;--pub use specialization::GemmSpecialization;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metaldeleted file mode 100644index d020bced4..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metal+++ /dev/null@@ -1,82 +0,0 @@-// clang-format off-#include "../../../common/utils.h"-#include "../../../common/dsl.h"-#include "../../../common/thread_context.h"--#include "../../common/gemm.h"--using namespace uzu::matmul;--template <typename T, int BLOCK_ROWS, int BLOCK_COLS, int BLOCK_DEPTH, int SIMDGROUPS_PER_ROW, int SIMDGROUPS_PER_COLUMN, bool MN_aligned, bool K_aligned>-METAL_FUNC void gemm_dispatch(- const device T* a,- const device T* b,- device T* d,- const constant GemmParams* params,- threadgroup T* a_shared,- threadgroup T* b_shared,- uint simd_lane_id,- uint simd_group_id,- uint2 tid,- uint3 lid-) {- using ThreadgroupGemmType = ThreadgroupGemm<T, T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true, MN_aligned, K_aligned, float>;- ThreadgroupGemmType::run(a, b, d, params, a_shared, b_shared, simd_lane_id, simd_group_id, tid, lid);-}--#define GEMM_DISPATCH(T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN) \- if (block_rows == BLOCK_ROWS && block_cols == BLOCK_COLS && block_depth == BLOCK_DEPTH && \- simdgroups_per_row == SIMDGROUPS_PER_ROW && simdgroups_per_column == SIMDGROUPS_PER_COLUMN) { \- if (align_m && align_n) { \- if (align_k) \- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, true>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \- else \- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, false>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \- } else { \- if (align_k) \- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \- else \- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, false>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \- } \- return; \- }--#define GEMM_MAX_THREADGROUP_A 2560-#define GEMM_MAX_THREADGROUP_B 1536--template <typename T>-VARIANTS(T, float, half, bfloat)-KERNEL(MatmulGemm)(- const device T* a,- const device T* b,- device T* d,- const constant uzu::matmul::GemmParams* params,- const constant uint& group_count_x,- const constant uint& group_count_y,- threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],- threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],- const uint block_rows SPECIALIZE,- const uint block_cols SPECIALIZE,- const uint block_depth SPECIALIZE,- const uint simdgroups_per_row SPECIALIZE,- const uint simdgroups_per_column SPECIALIZE,- const bool align_m SPECIALIZE,- const bool align_n SPECIALIZE,- const bool align_k SPECIALIZE,- const uint group_x GROUPS(group_count_x),- const uint group_y GROUPS(group_count_y),- const uint thread_x THREADS(32),- const uint thread_y THREADS(2),- const uint thread_z THREADS(2),- const ThreadContext thread_context-) {- GEMM_DISPATCH(T, 64, 64, 16, 2, 2)- GEMM_DISPATCH(T, 64, 64, 16, 1, 2)- GEMM_DISPATCH(T, 64, 32, 32, 2, 2)- GEMM_DISPATCH(T, 32, 64, 16, 1, 2)- GEMM_DISPATCH(T, 32, 64, 16, 2, 2)- GEMM_DISPATCH(T, 32, 32, 16, 2, 2)- GEMM_DISPATCH(T, 64, 32, 8, 4, 1)-}--// clang-format ondiff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metalsimilarity index 71%rename from crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metalrename to crates/uzu/src/backends/metal/kernel/matmul/gemv.metalindex 169715ce6..3187bee36 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal@@ -1,6 +1,6 @@-#include "../../../common/utils.h"-#include "../../../common/dsl.h"-#include "../../../common/thread_context.h"+#include "../common/utils.h"+#include "../common/dsl.h"+#include "../common/thread_context.h"#include <metal_simdgroup>@@ -16,16 +16,14 @@ VARIANTS(T, float, half, bfloat)KERNEL(MatmulGemv)(const device T* matrix,const device T* input_vector,- const device T* output_source OPTIONAL(apply_output_scale_and_accumulate),+ const device T* output_bias OPTIONAL(is_bias),device T* output_vector,- const constant int& input_dimension,- const constant int& output_dimension,- const constant int& matrix_leading_dimension,+ const constant uint& input_dimension,+ const constant uint& output_dimension,+ const constant uint& matrix_leading_dimension,const constant float& output_scale,- const constant float& output_accumulate_scale,- const constant int& output_source_stride,- const constant int& batch_rows,- const constant int& output_rows_per_threadgroup,+ const constant uint& batch_rows,+ const constant uint& output_rows_per_threadgroup,threadgroup float threadgroup_memory[GEMV_MAX_THREADGROUP_MEMORY],const uint tg_simd_rows SPECIALIZE,const uint tg_simd_cols SPECIALIZE,@@ -33,7 +31,8 @@ KERNEL(MatmulGemv)(const uint sg_thread_cols SPECIALIZE,const uint thread_out_rows SPECIALIZE,const uint thread_out_cols SPECIALIZE,- const bool apply_output_scale_and_accumulate SPECIALIZE,+ const bool is_accumulate SPECIALIZE,+ const bool is_bias SPECIALIZE,const uint threadgroup_index_x GROUPS((output_dimension + output_rows_per_threadgroup - 1) / output_rows_per_threadgroup),const uint threadgroup_index_y GROUPS(batch_rows),const uint thread_index_x THREADS(32),@@ -51,11 +50,11 @@ KERNEL(MatmulGemv)(}// Derived constants from SPECIALIZE params- const int threads_per_threadgroup_col = tg_simd_cols * sg_thread_cols;- const int input_columns_per_threadgroup =+ const uint threads_per_threadgroup_col = tg_simd_cols * sg_thread_cols;+ const uint input_columns_per_threadgroup =threads_per_threadgroup_col * thread_out_cols;- const int batch_row = static_cast<int>(threadgroup_index_y);+ const uint batch_row = static_cast<uint>(threadgroup_index_y);if (batch_row >= batch_rows) {return;}@@ -65,40 +64,40 @@ KERNEL(MatmulGemv)(thread T matrix_values[4];thread float vector_coefficients[4];- const int thread_row_in_simdgroup =+ const uint thread_row_in_simdgroup =sg_thread_cols != 32- ? int(thread_context.simdgroup_index) / int(sg_thread_cols)+ ? uint(thread_context.simdgroup_index) / uint(sg_thread_cols): 0;const int thread_col_in_simdgroup =sg_thread_cols != 32- ? int(thread_context.simdgroup_index) % int(sg_thread_cols)- : int(thread_context.simdgroup_index);+ ? uint(thread_context.simdgroup_index) % uint(sg_thread_cols)+ : uint(thread_context.simdgroup_index);const int simdgroup_column_index =- tg_simd_cols != 1 ? int(thread_context.threadgroup_index % tg_simd_cols)+ tg_simd_cols != 1 ? uint(thread_context.threadgroup_index % tg_simd_cols): 0;- const int simdgroup_row_thread_base =+ const uint simdgroup_row_thread_base =tg_simd_cols != 1- ? int(sg_thread_rows) *- int(thread_context.threadgroup_index / tg_simd_cols)- : int(sg_thread_rows) * int(thread_context.threadgroup_index);- const int simdgroup_col_thread_base =+ ? uint(sg_thread_rows) *+ uint(thread_context.threadgroup_index / tg_simd_cols)+ : uint(sg_thread_rows) * uint(thread_context.threadgroup_index);+ const uint simdgroup_col_thread_base =tg_simd_cols != 1- ? int(sg_thread_cols) *- int(thread_context.threadgroup_index % tg_simd_cols)+ ? uint(sg_thread_cols) *+ uint(thread_context.threadgroup_index % tg_simd_cols): 0;- int output_block_row_offset =+ uint output_block_row_offset =(simdgroup_row_thread_base + thread_row_in_simdgroup) *- int(thread_out_rows);- int input_block_col_offset =+ uint(thread_out_rows);+ uint input_block_col_offset =(simdgroup_col_thread_base + thread_col_in_simdgroup) *- int(thread_out_cols);+ uint(thread_out_cols);// Block position- int output_row_start =- int(threadgroup_index_x) * int(output_rows_per_threadgroup) ++ uint output_row_start =+ uint(threadgroup_index_x) * uint(output_rows_per_threadgroup) +output_block_row_offset;// Exit simdgroup if rows out of bound@@ -106,23 +105,24 @@ KERNEL(MatmulGemv)(return;// Adjust tail simdgroup to ensure in bound reads- output_row_start = output_row_start + int(thread_out_rows) <= output_dimension- ? output_row_start- : output_dimension - int(thread_out_rows);+ output_row_start =+ output_row_start + uint(thread_out_rows) <= output_dimension+ ? output_row_start+ : output_dimension - uint(thread_out_rows);const device T* thread_matrix =matrix + output_row_start * matrix_leading_dimension;- const uniform<int> input_block_stride =- make_uniform(int(input_columns_per_threadgroup));- const uniform<int> input_vector_length = make_uniform(input_dimension);- const uniform<int> full_input_blocks =+ const uniform<uint> input_block_stride =+ make_uniform(uint(input_columns_per_threadgroup));+ const uniform<uint> input_vector_length = make_uniform(input_dimension);+ const uniform<uint> full_input_blocks =input_vector_length / input_block_stride;- const uniform<int> remaining_input_columns =+ const uniform<uint> remaining_input_columns =input_vector_length - input_block_stride * full_input_blocks;// Loop over input_vector in blocks of input_columns_per_threadgroup- for (int input_block_index = 0; input_block_index < full_input_blocks;+ for (uint input_block_index = 0; input_block_index < full_input_blocks;++input_block_index) {// Load vector coefficients (unchecked){@@ -138,7 +138,7 @@ KERNEL(MatmulGemv)(}// Per thread work loop- int matrix_row_offset = 0;+ uint matrix_row_offset = 0;METAL_PRAGMA_UNROLLfor (uint output_row_offset = 0; output_row_offset < thread_out_rows;output_row_offset++) {@@ -170,7 +170,7 @@ KERNEL(MatmulGemv)({const device T* input_vector_row =input_vector + batch_row * input_dimension;- if (input_block_col_offset + int(thread_out_cols) <=+ if (input_block_col_offset + uint(thread_out_cols) <=input_vector_length) {METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;@@ -184,7 +184,7 @@ KERNEL(MatmulGemv)(for (uint input_col_offset = 0; input_col_offset < thread_out_cols;input_col_offset++) {vector_coefficients[input_col_offset] =- input_block_col_offset + int(input_col_offset) <+ input_block_col_offset + uint(input_col_offset) <input_vector_length? static_cast<float>(input_vector_row@@ -200,7 +200,7 @@ KERNEL(MatmulGemv)(for (uint output_row_offset = 0; output_row_offset < thread_out_rows;output_row_offset++) {// Load matrix row (checked)- if (input_block_col_offset + int(thread_out_cols) <=+ if (input_block_col_offset + uint(thread_out_cols) <=input_vector_length) {METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;@@ -214,7 +214,7 @@ KERNEL(MatmulGemv)(for (uint input_col_offset = 0; input_col_offset < thread_out_cols;input_col_offset++) {matrix_values[input_col_offset] =- input_block_col_offset + int(input_col_offset) <+ input_block_col_offset + uint(input_col_offset) <input_vector_length? thread_matrix[output_row_offset * matrix_leading_dimension +@@ -251,14 +251,14 @@ KERNEL(MatmulGemv)(// Threadgroup reduction (only when tg_simd_cols > 1)if (tg_simd_cols > 1) {- const int computed_output_rows_per_tg =- int(tg_simd_rows) * int(sg_thread_rows) * int(thread_out_rows);+ const uint computed_output_rows_per_tg =+ uint(tg_simd_rows) * uint(sg_thread_rows) * uint(thread_out_rows);if (thread_col_in_simdgroup == 0) {threadgroup float* threadgroup_partial_accumulations =threadgroup_memory +simdgroup_column_index *- (computed_output_rows_per_tg + int(thread_out_rows)) ++ (computed_output_rows_per_tg + uint(thread_out_rows)) +output_block_row_offset;METAL_PRAGMA_UNROLLfor (uint output_row_offset = 0; output_row_offset < thread_out_rows;@@ -280,7 +280,7 @@ KERNEL(MatmulGemv)(output_row_offset++) {accumulated_values[output_row_offset] += base_partial[reduction_simdgroup_col *- (computed_output_rows_per_tg + int(thread_out_rows)) ++ (computed_output_rows_per_tg + uint(thread_out_rows)) +output_row_offset];}}@@ -294,18 +294,16 @@ KERNEL(MatmulGemv)(METAL_PRAGMA_UNROLLfor (uint output_row_offset = 0; output_row_offset < thread_out_rows;output_row_offset++) {- if (apply_output_scale_and_accumulate) {- output_row_values[output_row_start + output_row_offset] =- static_cast<T>(output_scale) *- static_cast<T>(accumulated_values[output_row_offset]) +- static_cast<T>(output_accumulate_scale) *- output_source- [(output_row_start + output_row_offset) *- output_source_stride];- } else {- output_row_values[output_row_start + output_row_offset] =- static_cast<T>(accumulated_values[output_row_offset]);+ T accumulated_c = static_cast<T>(output_scale) *+ static_cast<T>(accumulated_values[output_row_offset]);+ if (is_accumulate) {+ accumulated_c +=+ output_row_values[output_row_start + output_row_offset];+ }+ if (is_bias) {+ accumulated_c += output_bias[output_row_start + output_row_offset];}+ output_row_values[output_row_start + output_row_offset] = accumulated_c;}}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rssimilarity index 53%rename from crates/uzu/src/backends/metal/kernel/matmul/gemv/specialization.rsrename to crates/uzu/src/backends/metal/kernel/matmul/gemv.rsindex 87fca97f1..4e83e73cd 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/specialization.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs@@ -1,15 +1,16 @@#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct Specialization {+pub struct GemvSpecialization {pub threadgroup_rows: u32,pub threadgroup_cols: u32,pub threads_per_simdgroup_row: u32,pub threads_per_simdgroup_col: u32,pub elements_per_thread_row: u32,pub elements_per_thread_col: u32,- pub apply_output_scale_and_accumulate: bool,+ pub is_accumulate: bool,+ pub is_bias: bool,}-impl Specialization {+impl GemvSpecialization {pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {use crate::DataType;match data_type {@@ -21,7 +22,8 @@ impl Specialization {threads_per_simdgroup_col: 32,elements_per_thread_row: 4,elements_per_thread_col: 4,- apply_output_scale_and_accumulate: false,+ is_accumulate: false,+ is_bias: false,},Self {threadgroup_rows: 4,@@ -30,7 +32,8 @@ impl Specialization {threads_per_simdgroup_col: 32,elements_per_thread_row: 4,elements_per_thread_col: 4,- apply_output_scale_and_accumulate: true,+ is_accumulate: false,+ is_bias: true,},Self {threadgroup_rows: 8,@@ -39,7 +42,8 @@ impl Specialization {threads_per_simdgroup_col: 32,elements_per_thread_row: 4,elements_per_thread_col: 4,- apply_output_scale_and_accumulate: false,+ is_accumulate: false,+ is_bias: false,},Self {threadgroup_rows: 8,@@ -48,7 +52,8 @@ impl Specialization {threads_per_simdgroup_col: 32,elements_per_thread_row: 4,elements_per_thread_col: 4,- apply_output_scale_and_accumulate: true,+ is_accumulate: false,+ is_bias: true,},],DataType::F16 => &[Self {@@ -58,7 +63,8 @@ impl Specialization {threads_per_simdgroup_col: 32,elements_per_thread_row: 4,elements_per_thread_col: 4,- apply_output_scale_and_accumulate: false,+ is_accumulate: false,+ is_bias: false,}],DataType::F32 => &[Self {threadgroup_rows: 8,@@ -67,7 +73,8 @@ impl Specialization {threads_per_simdgroup_col: 32,elements_per_thread_row: 4,elements_per_thread_col: 4,- apply_output_scale_and_accumulate: false,+ is_accumulate: false,+ is_bias: false,}],_ => &[],}@@ -78,75 +85,45 @@ impl Specialization {}pub fn select(- transpose_matrix: bool,- input_dimension: i32,- output_dimension: i32,- apply_output_scale_and_accumulate: bool,+ input_dimension: u32,+ output_dimension: u32,+ is_accumulate: bool,+ is_bias: bool,) -> Self {let (threadgroup_rows, threadgroup_cols);let (threads_per_simdgroup_row, threads_per_simdgroup_col);let (elements_per_thread_row, elements_per_thread_col);- if transpose_matrix {- let mut simdgroup_thread_rows = 8;- let mut simdgroup_thread_cols = 4;- if input_dimension >= 8192 && output_dimension >= 2048 {- simdgroup_thread_rows = 4;- simdgroup_thread_cols = 8;- }+ let threadgroup_simd_rows;+ let mut simdgroup_thread_rows = 1;+ let mut simdgroup_thread_cols = 32;+ let mut threadgroup_simd_cols = 1;- let threadgroup_simd_cols = if output_dimension >= 2048 {- 16- } else if output_dimension >= 512 {- 4- } else {- 2- };-- let thread_output_cols = if output_dimension < 4 {- 1- } else {- 4- };-- threadgroup_rows = 1;- threadgroup_cols = threadgroup_simd_cols;- threads_per_simdgroup_row = simdgroup_thread_rows;- threads_per_simdgroup_col = simdgroup_thread_cols;- elements_per_thread_row = 4;- elements_per_thread_col = thread_output_cols;+ if input_dimension <= 64 {+ threadgroup_simd_rows = 1;+ simdgroup_thread_rows = 8;+ simdgroup_thread_cols = 4;+ } else if input_dimension >= 16 * output_dimension {+ threadgroup_simd_rows = 1;+ threadgroup_simd_cols = 8;+ } else if output_dimension >= 4096 {+ threadgroup_simd_rows = 8;} else {- let threadgroup_simd_rows;- let mut simdgroup_thread_rows = 1;- let mut simdgroup_thread_cols = 32;- let mut threadgroup_simd_cols = 1;-- if input_dimension <= 64 {- threadgroup_simd_rows = 1;- simdgroup_thread_rows = 8;- simdgroup_thread_cols = 4;- } else if input_dimension >= 16 * output_dimension {- threadgroup_simd_rows = 1;- threadgroup_simd_cols = 8;- } else if output_dimension >= 4096 {- threadgroup_simd_rows = 8;- } else {- threadgroup_simd_rows = 4;- }+ threadgroup_simd_rows = 4;+ }- let thread_output_rows = if output_dimension < 4 {- 1- } else {- 4- };+ let thread_output_rows = if output_dimension < 4 {+ 1+ } else {+ 4+ };- threadgroup_rows = threadgroup_simd_rows;- threadgroup_cols = threadgroup_simd_cols;- threads_per_simdgroup_row = simdgroup_thread_rows;- threads_per_simdgroup_col = simdgroup_thread_cols;- elements_per_thread_row = thread_output_rows;- elements_per_thread_col = 4;- }+ threadgroup_rows = threadgroup_simd_rows;+ threadgroup_cols = threadgroup_simd_cols;+ threads_per_simdgroup_row = simdgroup_thread_rows;+ threads_per_simdgroup_col = simdgroup_thread_cols;+ elements_per_thread_row = thread_output_rows;+ elements_per_thread_col = 4;Self {threadgroup_rows,@@ -155,7 +132,8 @@ impl Specialization {threads_per_simdgroup_col,elements_per_thread_row,elements_per_thread_col,- apply_output_scale_and_accumulate,+ is_accumulate,+ is_bias,}}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rsdeleted file mode 100644index 1d2748972..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs+++ /dev/null@@ -1,3 +0,0 @@-mod specialization;--pub use specialization::Specialization;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs b/crates/uzu/src/backends/metal/kernel/matmul/matmul.rsdeleted file mode 100644index ae1d4d400..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs+++ /dev/null@@ -1,339 +0,0 @@-use std::{- collections::{HashMap, hash_map::Entry},- sync::OnceLock,-};--use super::{- super::dsl::{MatmulGemmMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel},- gemm, gemv,-};-use crate::{- DataType,- backends::{- common::{- Encoder,- gpu_types::GemmParams,- kernel::{- TensorAddBiasKernel,- matmul::{MatmulArguments, MatmulError, MatmulKernel},- },- },- metal::{Metal, context::MetalContext},- },-};--pub struct MatmulMetalKernel {- data_type: DataType,- gemv_kernels: HashMap<gemv::Specialization, MatmulGemvMetalKernel>,- gemm_kernels: HashMap<gemm::GemmSpecialization, MatmulGemmMetalKernel>,- bias_add: Option<TensorAddBiasMetalKernel>,-}--const DEFAULT_GEMV_MAX_BATCH: i32 = 8;--static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();--fn max_gemv_batch_threshold() -> i32 {- *GEMV_MAX_BATCH.get_or_init(|| {- std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)- })-}--impl MatmulMetalKernel {- fn is_gemv_eligible(arguments: &MatmulArguments<Metal>) -> bool {- let m = arguments.batch;- let n = arguments.output_dim;- let max_batch = max_gemv_batch_threshold();-- if !arguments.transpose_b {- return false;- }-- if n == 1 {- m == 1- } else {- m <= max_batch- }- }-- fn get_or_create_gemv(- &mut self,- context: &MetalContext,- specialization: gemv::Specialization,- ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {- match self.gemv_kernels.entry(specialization) {- Entry::Occupied(entry) => Ok(entry.into_mut()),- Entry::Vacant(entry) => {- let kernel = MatmulGemvMetalKernel::new(- context,- self.data_type,- specialization.threadgroup_rows,- specialization.threadgroup_cols,- specialization.threads_per_simdgroup_row,- specialization.threads_per_simdgroup_col,- specialization.elements_per_thread_row,- specialization.elements_per_thread_col,- specialization.apply_output_scale_and_accumulate,- )- .map_err(MatmulError::BackendError)?;- Ok(entry.insert(kernel))- },- }- }-- fn get_or_create_gemm(- &mut self,- context: &MetalContext,- specialization: gemm::GemmSpecialization,- ) -> Result<&MatmulGemmMetalKernel, MatmulError<Metal>> {- match self.gemm_kernels.entry(specialization) {- Entry::Occupied(entry) => Ok(entry.into_mut()),- Entry::Vacant(entry) => {- let kernel = MatmulGemmMetalKernel::new(- context,- self.data_type,- specialization.block_rows as u32,- specialization.block_cols as u32,- specialization.block_depth as u32,- specialization.simdgroups_per_row as u32,- specialization.simdgroups_per_column as u32,- specialization.align_m,- specialization.align_n,- specialization.align_k,- )- .map_err(MatmulError::BackendError)?;- Ok(entry.insert(kernel))- },- }- }-- fn encode_gemv(- &mut self,- context: &MetalContext,- encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<Metal>,- ) -> Result<(), MatmulError<Metal>> {- let m = arguments.batch;- let n = arguments.output_dim;-- let matrix_is_rhs = n != 1;- let has_bias = arguments.bias.is_some();- let apply_output_scale_and_accumulate = has_bias;-- let effective_output_dimension = if matrix_is_rhs {- n- } else {- m- };-- let transpose_matrix = if matrix_is_rhs {- !arguments.transpose_b- } else {- false- };-- let specialization = gemv::Specialization::select(- transpose_matrix,- arguments.input_dim,- effective_output_dimension,- apply_output_scale_and_accumulate,- );-- let (alpha, beta, bias_stride) = if apply_output_scale_and_accumulate {- (1.0f32, 1.0f32, 1i32)- } else {- (1.0f32, 0.0f32, 0i32)- };-- let matrix_leading_dim = if matrix_is_rhs {- arguments.leading_dimension_b- } else {- arguments.leading_dimension_a- };-- let (matrix, matrix_offset) = if matrix_is_rhs {- (arguments.b, 0usize)- } else {- (arguments.a, arguments.a_offset as usize)- };- let (input_vector, input_vector_offset) = if matrix_is_rhs {- (arguments.a, arguments.a_offset as usize)- } else {- (arguments.b, 0usize)- };-- let output_source = if apply_output_scale_and_accumulate {- arguments.bias- } else {- None- };-- let kernel = self.get_or_create_gemv(context, specialization)?;-- kernel.encode(- (matrix, matrix_offset),- (input_vector, input_vector_offset),- output_source.map(|buffer| (buffer, 0usize)),- &mut *arguments.d,- arguments.input_dim,- effective_output_dimension,- matrix_leading_dim,- alpha,- beta,- bias_stride,- arguments.batch,- specialization.output_rows_per_threadgroup() as i32,- encoder,- );-- let bias_is_fused = apply_output_scale_and_accumulate && has_bias;- if !bias_is_fused {- self.encode_bias_add(- context,- arguments.bias,- arguments.d,- arguments.batch as usize,- arguments.output_dim as usize,- encoder,- )?;- }-- Ok(())- }-- fn encode_gemm(- &mut self,- context: &MetalContext,- encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<Metal>,- ) -> Result<(), MatmulError<Metal>> {- let m = arguments.batch;- let n = arguments.output_dim;- let k = arguments.input_dim;-- let specialization = gemm::GemmSpecialization::select(context, self.data_type, &arguments);-- let threadgroups_per_row = (n + specialization.block_cols - 1) / specialization.block_cols;- let threadgroups_per_column = (m + specialization.block_rows - 1) / specialization.block_rows;- let swizzle_log = specialization.swizzle_log2;-- let swizzle_stride = 1_i32 << swizzle_log;- let tm_swizzled = (threadgroups_per_column + swizzle_stride - 1) / swizzle_stride;- let tn_swizzled = threadgroups_per_row * swizzle_stride;-- let params = GemmParams {- M: m,- N: n,- K: k,- leading_dimension_a: arguments.leading_dimension_a,- leading_dimension_b: arguments.leading_dimension_b,- leading_dimension_d: arguments.leading_dimension_d,- threadgroups_per_row,- threadgroups_per_column,- swizzle_log,- aligned_inner_iterations: k / specialization.block_depth,- };-- let group_count_x =- u32::try_from(tn_swizzled).map_err(|_| MatmulError::<Metal>::ThreadgroupOverflow(tn_swizzled as usize))?;- let group_count_y =- u32::try_from(tm_swizzled).map_err(|_| MatmulError::<Metal>::ThreadgroupOverflow(tm_swizzled as usize))?;-- let kernel = self.get_or_create_gemm(context, specialization)?;-- kernel.encode(- (arguments.a, arguments.a_offset as usize),- arguments.b,- &mut *arguments.d,- std::slice::from_ref(¶ms),- group_count_x,- group_count_y,- encoder,- );-- self.encode_bias_add(- context,- arguments.bias,- arguments.d,- arguments.batch as usize,- arguments.output_dim as usize,- encoder,- )?;-- Ok(())- }-- fn encode_bias_add(- &mut self,- context: &MetalContext,- bias: Option<&<Metal as crate::backends::common::Backend>::Buffer>,- output: &mut <Metal as crate::backends::common::Backend>::Buffer,- batch: usize,- output_dim: usize,- encoder: &mut Encoder<Metal>,- ) -> Result<(), MatmulError<Metal>> {- let Some(bias) = bias else {- return Ok(());- };- let total_length = batch * output_dim;- if total_length == 0 {- return Ok(());- }- if self.bias_add.is_none() {- self.bias_add =- Some(TensorAddBiasMetalKernel::new(context, self.data_type, true).map_err(MatmulError::BackendError)?);- }- let bias_add = self.bias_add.as_ref().unwrap();- bias_add.encode(- None::<&<Metal as crate::backends::common::Backend>::Buffer>,- bias,- output,- output_dim as u32,- total_length as u32,- encoder,- );- Ok(())- }-}--impl MatmulKernel for MatmulMetalKernel {- type Backend = Metal;-- fn new(- context: &MetalContext,- data_type: DataType,- ) -> Result<Self, MatmulError<Metal>> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::UnsupportedDataType(data_type));- }-- let mut kernel = Self {- data_type,- gemv_kernels: HashMap::new(),- gemm_kernels: HashMap::new(),- bias_add: None,- };-- for &config in gemv::Specialization::precompile_configs(data_type) {- kernel.get_or_create_gemv(context, config)?;- }- for &config in gemm::GemmSpecialization::precompile_configs(data_type) {- kernel.get_or_create_gemm(context, config)?;- }-- Ok(kernel)- }-- fn encode(- &mut self,- context: &MetalContext,- arguments: MatmulArguments<Metal>,- encoder: &mut Encoder<Metal>,- ) {- if Self::is_gemv_eligible(&arguments) {- self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV kernel");- } else {- self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM kernel");- }- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex 27c0a5c14..9ebb64163 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -1,5 +1,228 @@-pub mod gemm;-pub mod gemv;-mod matmul;+use std::{+ collections::{HashMap, hash_map::Entry},+ sync::OnceLock,+};-pub use matmul::MatmulMetalKernel;+use crate::{+ DataType,+ backends::{+ common::{+ Encoder,+ gpu_types::GemmParams,+ kernel::{+ TensorAddBiasKernel,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},+ },+ },+ metal::{+ Metal,+ context::MetalContext,+ kernel::{MatmulGemmMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel},+ },+ },+};++mod gemm;+mod gemv;++pub struct MatmulMetalKernel {+ data_type: DataType,+ gemv_kernels: HashMap<gemv::GemvSpecialization, MatmulGemvMetalKernel>,+ gemm_kernels: HashMap<gemm::GemmSpecialization, MatmulGemmMetalKernel>,+ bias_add: TensorAddBiasMetalKernel,+}++const DEFAULT_GEMV_MAX_BATCH: u32 = 8;++static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();++fn max_gemv_batch_threshold() -> u32 {+ *GEMV_MAX_BATCH.get_or_init(|| {+ std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)+ })+}++impl MatmulMetalKernel {+ fn get_or_create_gemv(+ &mut self,+ context: &MetalContext,+ specialization: gemv::GemvSpecialization,+ ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {+ match self.gemv_kernels.entry(specialization) {+ Entry::Occupied(entry) => Ok(entry.into_mut()),+ Entry::Vacant(entry) => {+ let kernel = MatmulGemvMetalKernel::new(+ context,+ self.data_type,+ specialization.threadgroup_rows,+ specialization.threadgroup_cols,+ specialization.threads_per_simdgroup_row,+ specialization.threads_per_simdgroup_col,+ specialization.elements_per_thread_row,+ specialization.elements_per_thread_col,+ specialization.is_accumulate,+ specialization.is_bias,+ )+ .map_err(MatmulError::BackendError)?;+ Ok(entry.insert(kernel))+ },+ }+ }++ fn get_or_create_gemm(+ &mut self,+ context: &MetalContext,+ specialization: gemm::GemmSpecialization,+ ) -> Result<&MatmulGemmMetalKernel, MatmulError<Metal>> {+ match self.gemm_kernels.entry(specialization) {+ Entry::Occupied(entry) => Ok(entry.into_mut()),+ Entry::Vacant(entry) => {+ let kernel = MatmulGemmMetalKernel::new(+ context,+ self.data_type,+ specialization.block_rows as u32,+ specialization.block_cols as u32,+ specialization.block_depth as u32,+ specialization.simdgroups_per_row as u32,+ specialization.simdgroups_per_column as u32,+ specialization.align_mn,+ specialization.align_k,+ specialization.is_accumulate,+ )+ .map_err(MatmulError::BackendError)?;+ Ok(entry.insert(kernel))+ },+ }+ }++ fn encode_gemv(+ &mut self,+ context: &MetalContext,+ encoder: &mut Encoder<Metal>,+ arguments: MatmulArguments<Metal>,+ ) -> Result<(), MatmulError<Metal>> {+ let (is_accumulate, output_bias) = match arguments.c {+ MatmulArgumentC::Accumulate => (true, None),+ MatmulArgumentC::Bias(bias) => (false, Some(bias)),+ MatmulArgumentC::None => (false, None),+ };++ let specialization = gemv::GemvSpecialization::select(+ arguments.input_dim,+ arguments.output_dim,+ is_accumulate,+ output_bias.is_some(),+ );++ self.get_or_create_gemv(context, specialization)?.encode(+ arguments.b,+ (arguments.a, arguments.a_offset as usize),+ output_bias,+ arguments.d,+ arguments.input_dim,+ arguments.output_dim,+ arguments.input_dim,+ arguments.ab_scale,+ arguments.batch_dim,+ specialization.output_rows_per_threadgroup(),+ encoder,+ );++ Ok(())+ }++ fn encode_gemm(+ &mut self,+ context: &MetalContext,+ encoder: &mut Encoder<Metal>,+ arguments: MatmulArguments<Metal>,+ ) -> Result<(), MatmulError<Metal>> {+ let specialization = gemm::GemmSpecialization::select(context, self.data_type, &arguments);++ let threadgroups_per_row = arguments.output_dim.div_ceil(specialization.block_cols);+ let threadgroups_per_column = arguments.batch_dim.div_ceil(specialization.block_rows);++ let params = GemmParams {+ M: arguments.batch_dim,+ N: arguments.output_dim,+ K: arguments.input_dim,+ leading_dimension_a: arguments.input_dim,+ leading_dimension_b: arguments.input_dim,+ leading_dimension_d: arguments.output_dim,+ threadgroups_per_row,+ threadgroups_per_column,+ swizzle_log: 0,+ aligned_inner_iterations: arguments.input_dim / specialization.block_depth,+ };++ let kernel = self.get_or_create_gemm(context, specialization)?;++ kernel.encode(+ (arguments.a, arguments.a_offset as usize),+ arguments.b,+ &mut *arguments.d,+ std::slice::from_ref(¶ms),+ threadgroups_per_row,+ threadgroups_per_column,+ arguments.ab_scale,+ encoder,+ );++ if let MatmulArgumentC::Bias(bias) = arguments.c {+ self.bias_add.encode(+ None::<&<Metal as crate::backends::common::Backend>::Buffer>,+ bias,+ arguments.d,+ arguments.output_dim,+ arguments.batch_dim * arguments.output_dim,+ encoder,+ );+ }++ Ok(())+ }+}++impl MatmulKernel for MatmulMetalKernel {+ type Backend = Metal;++ fn new(+ context: &MetalContext,+ data_type: DataType,+ ) -> Result<Self, MatmulError<Metal>> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MatmulError::UnsupportedDataType(data_type));+ }++ let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true).map_err(MatmulError::BackendError)?;++ let mut kernel = Self {+ data_type,+ gemv_kernels: HashMap::new(),+ gemm_kernels: HashMap::new(),+ bias_add,+ };++ for &config in gemv::GemvSpecialization::precompile_configs(data_type) {+ kernel.get_or_create_gemv(context, config)?;+ }+ for &config in gemm::GemmSpecialization::precompile_configs(data_type) {+ kernel.get_or_create_gemm(context, config)?;+ }++ Ok(kernel)+ }++ fn encode(+ &mut self,+ context: &MetalContext,+ arguments: MatmulArguments<Metal>,+ encoder: &mut Encoder<Metal>,+ ) {+ if arguments.batch_dim <= max_gemv_batch_threshold() {+ self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV kernel");+ } else {+ self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM kernel");+ }+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex dedaa9318..916078500 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -1,12 +1,11 @@pub mod matmul;-use crate::backends::common::kernel::matmul::MatmulKernels;+use crate::backends::common::kernel::ManualKernels;++include!(concat!(env!("OUT_DIR"), "/dsl.rs"));-pub mod dsl {- include!(concat!(env!("OUT_DIR"), "/dsl.rs"));-}pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));-impl MatmulKernels for dsl::MetalKernels {+impl ManualKernels for MetalKernels {type MatmulKernel = matmul::MatmulMetalKernel;}diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsindex 4c21fc68c..f2ab1afda 100644--- a/crates/uzu/src/encodable_block/embedding.rs+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -10,8 +10,8 @@ use crate::{backends::common::{Backend, Encoder, Kernels,kernel::{- FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,- matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},+ FullPrecisionEmbeddingLookupKernel, ManualKernels, QuantizedEmbeddingLookupKernel,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,QuantizedMatmulKernelEncodable, QuantizedMatmulType,@@ -48,7 +48,7 @@ enum TiedEmbeddingType<B: Backend> {FullPrecision {weights: B::Buffer,lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,- readout: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,+ readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,},Quantized {weights: B::Buffer,@@ -75,7 +75,7 @@ enum UntiedEmbeddingLookupType<B: Backend> {enum UntiedEmbeddingReadoutType<B: Backend> {FullPrecision {weights: B::Buffer,- readout: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,+ readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,},Quantized {weights: B::Buffer,@@ -213,7 +213,7 @@ impl<B: Backend> Embedding<B> {let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;- let readout = RefCell::new(<B::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)?);+ let readout = RefCell::new(<B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)?);EmbeddingTying::Tied {ty: TiedEmbeddingType::FullPrecision {@@ -239,7 +239,7 @@ impl<B: Backend> Embedding<B> {let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;- let readout = RefCell::new(<B::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)?);+ let readout = RefCell::new(<B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)?);EmbeddingTying::Untied {input_ty: UntiedEmbeddingLookupType::FullPrecision {@@ -615,15 +615,12 @@ impl<B: Backend> Embedding<B> {a: input,a_offset: input_offset as u64,b: weights,+ ab_scale: 1.0,+ c: MatmulArgumentC::None,d: output,- bias: None,- batch: batch_size as i32,- input_dim: input_dim as i32,- output_dim: output_dim as i32,- leading_dimension_a: input_dim as i32,- leading_dimension_b: input_dim as i32,- leading_dimension_d: output_dim as i32,- transpose_b: true,+ batch_dim: batch_size as u32,+ input_dim: input_dim as u32,+ output_dim: output_dim as u32,},encoder,);diff --git a/crates/uzu/src/encodable_block/linear/full_precision.rs b/crates/uzu/src/encodable_block/linear/full_precision.rsindex 64000ebba..9111d4a14 100644--- a/crates/uzu/src/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs@@ -11,7 +11,10 @@ use crate::{DataType,backends::common::{Backend, Encoder,- kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},+ kernel::{+ ManualKernels,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},+ },},forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},@@ -49,7 +52,7 @@ pub enum FullPrecisionLinearError<B: Backend> {}pub struct FullPrecisionLinear<B: Backend> {- kernel: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,+ kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,bias_buffer: Option<Rc<RefCell<B::Buffer>>>,weights_buffer: Rc<RefCell<B::Buffer>>,input_dim: usize,@@ -111,7 +114,7 @@ impl<B: Backend> FullPrecisionLinear<B> {Err(_) => None,};- let kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(context, precision)?;+ let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, precision)?;Ok(Self {kernel: RefCell::new(kernel),@@ -142,15 +145,15 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {a: input_array.buffer().borrow().deref(),a_offset: 0,b: self.weights_buffer.borrow().deref(),+ ab_scale: 1.0,+ c: match bias_borrow.as_deref() {+ Some(b) => MatmulArgumentC::Bias(b),+ None => MatmulArgumentC::None,+ },d: output_array.buffer().borrow_mut().deref_mut(),- bias: bias_borrow.as_deref(),- batch: batch_size as i32,- input_dim: self.input_dim as i32,- output_dim: self.output_dim as i32,- leading_dimension_a: self.input_dim as i32,- leading_dimension_b: self.input_dim as i32,- leading_dimension_d: self.output_dim as i32,- transpose_b: true,+ batch_dim: batch_size as u32,+ input_dim: self.input_dim as u32,+ output_dim: self.output_dim as u32,},encoder,);diff --git a/crates/uzu/src/session/tts_session.rs b/crates/uzu/src/session/tts_session.rsindex 040cefdd3..fd6c5518e 100644--- a/crates/uzu/src/session/tts_session.rs+++ b/crates/uzu/src/session/tts_session.rs@@ -33,10 +33,10 @@ use crate::{backends::common::{Backend, Context as BackendContext, Encoder, Kernels,kernel::{- EmbeddingRowsSumKernel, TensorAddScaleKernel, TensorCopyKernel, TokenCopySampledKernel,+ EmbeddingRowsSumKernel, ManualKernels, TensorAddScaleKernel, TensorCopyKernel, TokenCopySampledKernel,TokenCopyToResultsKernel,kv_cache_update::KVCacheUpdate,- matmul::{MatmulArguments, MatmulKernel, MatmulKernels},+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},},},config::{InnerModelConfig, ModelMetadata, TtsMessageProcessorConfig},diff --git a/crates/uzu/src/session/tts_session/fishaudio.rs b/crates/uzu/src/session/tts_session/fishaudio.rsindex 9ad5efdb2..681c0b15a 100644--- a/crates/uzu/src/session/tts_session/fishaudio.rs+++ b/crates/uzu/src/session/tts_session/fishaudio.rs@@ -34,7 +34,7 @@ pub(super) struct FishAudioTextDecoderRuntime<B: Backend> {struct FishAudioSemanticBridge<B: Backend> {embedding_rows_sum: <B::Kernels as Kernels>::EmbeddingRowsSumKernel,- projection: <B::Kernels as MatmulKernels>::MatmulKernel,+ projection: <B::Kernels as ManualKernels>::MatmulKernel,tensor_copy: <B::Kernels as Kernels>::TensorCopyKernel,codebook_embeddings: Array<B>,codebook_token_indices: Array<B>,@@ -104,7 +104,7 @@ impl<B: Backend> FishAudioSemanticBridge<B> {let embedding_rows_sum = <B::Kernels as Kernels>::EmbeddingRowsSumKernel::new(context, data_type).map_err(unable_to_create_context)?;- let projection = <<B::Kernels as MatmulKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)+ let projection = <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type).map_err(unable_to_create_context)?;let tensor_copy =<B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type).map_err(unable_to_create_context)?;@@ -540,15 +540,12 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {a: &hidden_buffer,a_offset: hidden.offset() as u64,b: &weights_buffer,+ ab_scale: 1.0,+ c: MatmulArgumentC::None,d: &mut output_buffer,- bias: None,- batch: 1,- input_dim: slow_model_dim as i32,- output_dim: fast_model_dim as i32,- leading_dimension_a: slow_model_dim as i32,- leading_dimension_b: fast_model_dim as i32,- leading_dimension_d: fast_model_dim as i32,- transpose_b: false,+ batch_dim: 1,+ input_dim: slow_model_dim as u32,+ output_dim: fast_model_dim as u32,},encoder,);diff --git a/crates/uzu/tests/performance/matmul/bench.rs b/crates/uzu/tests/performance/matmul/bench.rsindex 9236b0a68..ec8481f44 100644--- a/crates/uzu/tests/performance/matmul/bench.rs+++ b/crates/uzu/tests/performance/matmul/bench.rs@@ -7,7 +7,10 @@ use uzu::{backends::{common::{Backend, Encoder,- kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},+ kernel::{+ ManualKernels,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ },},metal::Metal,},@@ -34,7 +37,7 @@ fn fill_buffer_random(fn encode_and_run(context: &Ctx,- kernel: &mut <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel,+ kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,shape: &TestShape,a_buffer: &Buf,b_buffer: &Buf,@@ -48,15 +51,12 @@ fn encode_and_run(a: a_buffer,a_offset: 0,b: b_buffer,+ ab_scale: 1.0,+ c: MatmulArgumentC::None,d: d_buffer,- bias: None,- batch: shape.batch as i32,- input_dim: shape.input_dim as i32,- output_dim: shape.output_dim as i32,- leading_dimension_a: shape.input_dim as i32,- leading_dimension_b: shape.input_dim as i32,- leading_dimension_d: shape.output_dim as i32,- transpose_b: true,+ batch_dim: shape.batch as u32,+ input_dim: shape.input_dim as u32,+ output_dim: shape.output_dim as u32,},&mut encoder,);@@ -71,7 +71,7 @@ fn run_benchmark(data_type: DataType,shape: &TestShape,) -> Result<f64, BenchError> {- let mut kernel = <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)+ let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).map_err(|e| BenchError::Kernel(e.to_string()))?;let elem_size = data_type.size_in_bytes();diff --git a/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemv_test.rsindex 4fdfc6002..67604f30b 100644--- a/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs@@ -10,7 +10,10 @@ use uzu::{backends::{common::{Backend, Context, Encoder,- kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},+ kernel::{+ ManualKernels,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ },},cpu::Cpu,},@@ -49,9 +52,9 @@ fn get_test_data<T: ArrayElement + Float>(fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let m = input.m as i32;- let k = input.k as i32;- let n = input.n as i32;+ let m = input.m as u32;+ let k = input.k as u32;+ let n = input.n as u32;let a_array = context.create_array_from(&[input.m, input.k], &input.a, "");let b_array = context.create_array_from(&[input.n, input.k], &input.b, "");@@ -64,7 +67,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let d_buf = d_array.buffer();let mut d_ref = d_buf.borrow_mut();- let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type())+ let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type()).expect("Failed to create MatmulKernel");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -74,15 +77,12 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {a: a_ref.deref(),a_offset: 0,b: b_ref.deref(),+ ab_scale: 1.0,+ c: MatmulArgumentC::None,d: d_ref.deref_mut(),- bias: None,- batch: m,+ batch_dim: m,input_dim: k,output_dim: n,- leading_dimension_a: k,- leading_dimension_b: k,- leading_dimension_d: n,- transpose_b: true,},&mut encoder,);diff --git a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rsindex a0772b176..842642aa7 100644--- a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs@@ -11,7 +11,10 @@ use uzu::{backends::{common::{Backend, Context, Encoder,- kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},+ kernel::{+ ManualKernels,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ },},metal::Metal,},@@ -24,7 +27,6 @@ fn run_metal_matmul(m: usize,k: usize,n: usize,- transpose_b: bool,) -> Vec<bf16> {let a_buf = ctx.device@@ -39,14 +41,8 @@ fn run_metal_matmul(.new_buffer(m * n * core::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let leading_dim_b = if transpose_b {- k- } else {- n- };-let mut kernel =- <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel::new(ctx, DataType::BF16).expect("kernel");+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(ctx, DataType::BF16).expect("kernel");let mut encoder = Encoder::new(ctx).unwrap();kernel.encode(@@ -55,15 +51,12 @@ fn run_metal_matmul(a: &a_buf,a_offset: 0,b: &b_buf,+ ab_scale: 1.0,+ c: MatmulArgumentC::None,d: &mut d_buf,- bias: None,- batch: m as i32,- input_dim: k as i32,- output_dim: n as i32,- leading_dimension_a: k as i32,- leading_dimension_b: leading_dim_b as i32,- leading_dimension_d: n as i32,- transpose_b,+ batch_dim: m as u32,+ input_dim: k as u32,+ output_dim: n as u32,},&mut encoder,);@@ -81,20 +74,14 @@ fn run_ndarray_matmul(m: usize,k: usize,n: usize,- transpose_b: bool,) -> Vec<bf16> {let a_f32: Vec<f32> = a_data.iter().map(|x| x.to_f32()).collect();let b_f32: Vec<f32> = b_data.iter().map(|x| x.to_f32()).collect();let a_arr = Array2::from_shape_vec((m, k), a_f32).expect("A shape");+ let b_arr = Array2::from_shape_vec((n, k), b_f32).expect("B shape");- let result = if transpose_b {- let b_arr = Array2::from_shape_vec((n, k), b_f32).expect("B shape");- a_arr.dot(&b_arr.t())- } else {- let b_arr = Array2::from_shape_vec((k, n), b_f32).expect("B shape");- a_arr.dot(&b_arr)- };+ let result = a_arr.dot(&b_arr.t());result.iter().map(|&x| bf16::from_f32(x)).collect()}@@ -103,14 +90,9 @@ fn generate_test_data(m: usize,k: usize,n: usize,- transpose_b: bool,) -> (Vec<bf16>, Vec<bf16>) {let a: Vec<bf16> = (0..(m * k)).map(|i| bf16::from_f32(((i % 13) as f32) * 0.01 - 0.06)).collect();- let b_size = if transpose_b {- n * k- } else {- k * n- };+ let b_size = n * k;let b: Vec<bf16> = (0..b_size).map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.15)).collect();(a, b)}@@ -119,7 +101,6 @@ struct TestCase {m: usize,k: usize,n: usize,- transpose_b: bool,tolerance: f32,}@@ -213,24 +194,23 @@ fn test_cases(full_case_matrix: bool) -> Vec<TestCase> {(&QUICK_BATCH_SIZES, &QUICK_MODEL_SHAPES)};- let mut cases: Vec<(usize, usize, usize, bool)> =- model_shapes.iter().flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n, true))).collect();+ let mut cases: Vec<(usize, usize, usize)> =+ model_shapes.iter().flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n))).collect();// Edge case: small matrixfor &m in batch_sizes {- cases.push((m, 128, 128, true));+ cases.push((m, 128, 128));}cases.into_iter()- .map(|(m, k, n, transpose_b)| {+ .map(|(m, k, n)| {let tolerance = base_tolerance * (k as f32 / 1024.0).sqrt();let tolerance = tolerance * (1.0 + (m as f32).log2() * 0.02);TestCase {m,k,n,- transpose_b,tolerance,}})@@ -293,15 +273,11 @@ fn matmul_correctness_comprehensive() {let mut failed = Vec::new();for case in &cases {- let (a, b) = generate_test_data(case.m, case.k, case.n, case.transpose_b);- let metal_result = run_metal_matmul(&ctx, &a, &b, case.m, case.k, case.n, case.transpose_b);- let reference = run_ndarray_matmul(&a, &b, case.m, case.k, case.n, case.transpose_b);+ let (a, b) = generate_test_data(case.m, case.k, case.n);+ let metal_result = run_metal_matmul(&ctx, &a, &b, case.m, case.k, case.n);+ let reference = run_ndarray_matmul(&a, &b, case.m, case.k, case.n);- let trans_str = if case.transpose_b {- "T"- } else {- "N"- };+ let trans_str = "T";match compare_results(&metal_result, &reference, case.tolerance) {Ok(()) => {@@ -323,11 +299,7 @@ fn matmul_correctness_comprehensive() {if !failed.is_empty() {eprintln!("\nFailed tests:");for (case, max_diff, idx, count) in &failed {- let trans_str = if case.transpose_b {- "T"- } else {- "N"- };+ let trans_str = "T";eprintln!(" m={} k={} n={} B={}: max_diff={:.6} at idx {}, {} exceed tol {}",case.m, case.k, case.n, trans_str, max_diff, idx, count, case.tolerancediff --git a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rsindex e2a476fdf..7b1966953 100644--- a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs@@ -10,7 +10,10 @@ use uzu::{backends::{common::{Backend, Context, Encoder,- kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},+ kernel::{+ ManualKernels,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ },},cpu::Cpu,},@@ -49,9 +52,9 @@ fn get_test_data<T: ArrayElement + Float>(fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let m = input.m as i32;- let k = input.k as i32;- let n = input.n as i32;+ let m = input.m as u32;+ let k = input.k as u32;+ let n = input.n as u32;let a_array = context.create_array_from(&[input.m, input.k], &input.a, "");let b_array = context.create_array_from(&[input.n, input.k], &input.b, "");@@ -64,7 +67,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let d_buf = d_array.buffer();let mut d_ref = d_buf.borrow_mut();- let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type())+ let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type()).expect("Failed to create MatmulKernel");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -74,15 +77,12 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {a: a_ref.deref(),a_offset: 0,b: b_ref.deref(),+ ab_scale: 1.0,+ c: MatmulArgumentC::None,d: d_ref.deref_mut(),- bias: None,- batch: m,+ batch_dim: m,input_dim: k,output_dim: n,- leading_dimension_a: k,- leading_dimension_b: k,- leading_dimension_d: n,- transpose_b: true,},&mut encoder,);diff --git a/deny.toml b/deny.tomlindex dab26902d..7a48cd900 100644--- a/deny.toml+++ b/deny.toml@@ -15,6 +15,7 @@ allow = ["ISC","Zlib","BSL-1.0",+ "CC0-1.0","CDLA-Permissive-2.0","Unicode-3.0"]