Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

f32 matmul support

eugenef427ec93a812
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 563af413d..3cffd4945 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -27,7 +27,7 @@ template <
GemmWeightPrologueKind WEIGHT_PROLOGUE,
uint BITS,
uint GROUP_SIZE>
-VARIANTS(T, half, bfloat)
+VARIANTS(T, half, bfloat, float)
VARIANTS(
GEMM_TILING,
GemmTiling::T8x32x32_1x1,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index d56741462..155d38f64 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -103,7 +103,7 @@ impl GemmKernel {
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
let path = if encoder.context().device.supports_mxu()
- && matches!(self.data_type, DataType::F16 | DataType::BF16)
+ && matches!(self.data_type, DataType::F16 | DataType::BF16 | DataType::F32)
&& matches!(arguments.b, MatmulB::FullPrecision { .. })
{
GemmDispatchPath::Mxu
@@ -126,8 +126,8 @@ impl GemmKernel {
"GemmDispatchPath::Mxu requested on hardware without MXU support",
);
assert!(
- matches!(self.data_type, DataType::F16 | DataType::BF16),
- "GemmDispatchPath::Mxu requires F16 or BF16 data type, got {:?}",
+ matches!(self.data_type, DataType::F16 | DataType::BF16 | DataType::F32),
+ "GemmDispatchPath::Mxu requires F16, BF16, or F32 data type, got {:?}",
self.data_type,
);
assert!(
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 0c4fd79b2..43e2b80ff 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -132,12 +132,13 @@ fn simdgroup_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
match data_type {
DataType::BF16 => &[GemmTiling::T64x32x32_2x2, GemmTiling::T64x64x16_2x2],
DataType::F16 => &[GemmTiling::T64x64x16_2x2, GemmTiling::T64x32x32_2x2],
+ DataType::F32 => &[GemmTiling::T64x32x32_2x2, GemmTiling::T64x64x16_2x2],
_ => &[],
}
}
fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
- if !matches!(data_type, DataType::F16 | DataType::BF16) {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
return &[];
}
&[GemmTiling::T64x64x32_2x2, GemmTiling::T32x64x32_2x2, GemmTiling::T64x32x32_4x1, GemmTiling::T128x128x32_4x4]
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 660b21222..91362ea91 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -14,7 +14,7 @@ using namespace metal;
#define GEMV_MAX_THREADGROUP_MEMORY 320
template <typename T>
-VARIANTS(T, half, bfloat)
+VARIANTS(T, half, bfloat, float)
KERNEL(MatmulGemv)(
const device T* matrix,
const device T* input_vector,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
index 343189ea4..bd985238f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
@@ -14,6 +14,28 @@ impl GemvSpecialization {
pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {
use crate::DataType;
match data_type {
+ DataType::F32 => &[
+ Self {
+ threadgroup_rows: 4,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ is_accumulate: false,
+ is_bias: false,
+ },
+ Self {
+ threadgroup_rows: 4,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ is_accumulate: false,
+ is_bias: true,
+ },
+ ],
DataType::BF16 => &[
Self {
threadgroup_rows: 4,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 04e1db85d..b94707ccb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -142,7 +142,7 @@ impl MatmulKernel for MatmulMetalKernel {
context: &MetalContext,
data_type: DataType,
) -> Result<Self, MetalError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16) {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());
}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
index 97ece1a8b..eb03ae323 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
@@ -79,6 +79,11 @@ fn matches_cpu_reference_f16() {
run_matrix::<f16>(|shape| Case::new(shape), 0.5);
}
+#[test]
+fn matches_cpu_reference_f32() {
+ run_matrix::<f32>(|shape| Case::new(shape), 0.01);
+}
+
#[test]
fn b_transpose_false_bf16() {
run_matrix::<bf16>(
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
index d71526524..948e06985 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
@@ -135,3 +135,19 @@ fn gemv_bf16(
) {
test::<bf16>(m, k, n, 0.1);
}
+
+#[rstest]
+#[case::m1(1, 128, 64)]
+#[case::batched(4, 128, 64)]
+#[case::max_batch(8, 128, 64)]
+#[case::unaligned_k(1, 33, 64)]
+#[case::unaligned_n(1, 128, 11)]
+#[case::large(1, 4096, 2048)]
+#[case::small_n(1, 128, 3)]
+fn gemv_f32(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+) {
+ test::<f32>(m, k, n, 0.01);
+}