A high-performance inference engine for AI models
f32 matmul support
eugenef427ec93a812
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 563af413d..3cffd4945 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -27,7 +27,7 @@ template <GemmWeightPrologueKind WEIGHT_PROLOGUE,uint BITS,uint GROUP_SIZE>-VARIANTS(T, half, bfloat)+VARIANTS(T, half, bfloat, float)VARIANTS(GEMM_TILING,GemmTiling::T8x32x32_1x1,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex d56741462..155d38f64 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -103,7 +103,7 @@ impl GemmKernel {encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {let path = if encoder.context().device.supports_mxu()- && matches!(self.data_type, DataType::F16 | DataType::BF16)+ && matches!(self.data_type, DataType::F16 | DataType::BF16 | DataType::F32)&& matches!(arguments.b, MatmulB::FullPrecision { .. }){GemmDispatchPath::Mxu@@ -126,8 +126,8 @@ impl GemmKernel {"GemmDispatchPath::Mxu requested on hardware without MXU support",);assert!(- matches!(self.data_type, DataType::F16 | DataType::BF16),- "GemmDispatchPath::Mxu requires F16 or BF16 data type, got {:?}",+ matches!(self.data_type, DataType::F16 | DataType::BF16 | DataType::F32),+ "GemmDispatchPath::Mxu requires F16, BF16, or F32 data type, got {:?}",self.data_type,);assert!(diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 0c4fd79b2..43e2b80ff 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -132,12 +132,13 @@ fn simdgroup_tiling_set(data_type: DataType) -> &'static [GemmTiling] {match data_type {DataType::BF16 => &[GemmTiling::T64x32x32_2x2, GemmTiling::T64x64x16_2x2],DataType::F16 => &[GemmTiling::T64x64x16_2x2, GemmTiling::T64x32x32_2x2],+ DataType::F32 => &[GemmTiling::T64x32x32_2x2, GemmTiling::T64x64x16_2x2],_ => &[],}}fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {- if !matches!(data_type, DataType::F16 | DataType::BF16) {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {return &[];}&[GemmTiling::T64x64x32_2x2, GemmTiling::T32x64x32_2x2, GemmTiling::T64x32x32_4x1, GemmTiling::T128x128x32_4x4]diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metalindex 660b21222..91362ea91 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -14,7 +14,7 @@ using namespace metal;#define GEMV_MAX_THREADGROUP_MEMORY 320template <typename T>-VARIANTS(T, half, bfloat)+VARIANTS(T, half, bfloat, float)KERNEL(MatmulGemv)(const device T* matrix,const device T* input_vector,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rsindex 343189ea4..bd985238f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs@@ -14,6 +14,28 @@ impl GemvSpecialization {pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {use crate::DataType;match data_type {+ DataType::F32 => &[+ Self {+ threadgroup_rows: 4,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ is_accumulate: false,+ is_bias: false,+ },+ Self {+ threadgroup_rows: 4,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ is_accumulate: false,+ is_bias: true,+ },+ ],DataType::BF16 => &[Self {threadgroup_rows: 4,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex 04e1db85d..b94707ccb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -142,7 +142,7 @@ impl MatmulKernel for MatmulMetalKernel {context: &MetalContext,data_type: DataType,) -> Result<Self, MetalError> {- if !matches!(data_type, DataType::F16 | DataType::BF16) {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rsindex 97ece1a8b..eb03ae323 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs@@ -79,6 +79,11 @@ fn matches_cpu_reference_f16() {run_matrix::<f16>(|shape| Case::new(shape), 0.5);}+#[test]+fn matches_cpu_reference_f32() {+ run_matrix::<f32>(|shape| Case::new(shape), 0.01);+}+#[test]fn b_transpose_false_bf16() {run_matrix::<bf16>(diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rsindex d71526524..948e06985 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs@@ -135,3 +135,19 @@ fn gemv_bf16() {test::<bf16>(m, k, n, 0.1);}++#[rstest]+#[case::m1(1, 128, 64)]+#[case::batched(4, 128, 64)]+#[case::max_batch(8, 128, 64)]+#[case::unaligned_k(1, 33, 64)]+#[case::unaligned_n(1, 128, 11)]+#[case::large(1, 4096, 2048)]+#[case::small_n(1, 128, 3)]+fn gemv_f32(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+) {+ test::<f32>(m, k, n, 0.01);+}