Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

small matmul cleanup and new api for lora (#296)

uuuvn028fb04b988d
diff --git a/Cargo.lock b/Cargo.lock
index cc1ddc0cc..6b09e1746 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -15,6 +15,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
+ "const-random",
"getrandom 0.3.4",
"once_cell",
"serde",
@@ -658,6 +659,26 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "const-random"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
+dependencies = [
+ "const-random-macro",
+]
+
+[[package]]
+name = "const-random-macro"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
+dependencies = [
+ "getrandom 0.2.17",
+ "once_cell",
+ "tiny-keccak",
+]
+
[[package]]
name = "constant_time_eq"
version = "0.4.2"
@@ -2531,6 +2552,9 @@ name = "once_cell"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+dependencies = [
+ "portable-atomic",
+]
[[package]]
name = "once_cell_polyfill"
@@ -3169,6 +3193,34 @@ dependencies = [
"web-sys",
]
+[[package]]
+name = "rhai"
+version = "1.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f9ef5dabe4c0b43d8f1187dc6beb67b53fe607fff7e30c5eb7f71b814b8c2c1"
+dependencies = [
+ "ahash",
+ "bitflags",
+ "num-traits",
+ "once_cell",
+ "rhai_codegen",
+ "smallvec",
+ "smartstring",
+ "thin-vec",
+ "web-time",
+]
+
+[[package]]
+name = "rhai_codegen"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4322a2a4e8cf30771dd9f27f7f37ca9ac8fe812dddd811096a98483080dabe6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
[[package]]
name = "ring"
version = "0.17.14"
@@ -3612,6 +3664,17 @@ version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
+[[package]]
+name = "smartstring"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
+dependencies = [
+ "autocfg",
+ "static_assertions",
+ "version_check",
+]
+
[[package]]
name = "socket2"
version = "0.5.10"
@@ -3813,6 +3876,12 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "thin-vec"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "144f754d318415ac792f9d69fc87abbbfc043ce2ef041c60f16ad828f638717d"
+
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -3893,6 +3962,15 @@ dependencies = [
"time-core",
]
+[[package]]
+name = "tiny-keccak"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
+dependencies = [
+ "crunchy",
+]
+
[[package]]
name = "tinystr"
version = "0.8.2"
@@ -4409,6 +4487,7 @@ dependencies = [
"rand_distr",
"rangemap",
"regex",
+ "rhai",
"schemars",
"serde",
"serde_json",
diff --git a/Cargo.toml b/Cargo.toml
index f9eec995e..4894e1bce 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -58,6 +58,7 @@ quote = "1.0"
proc-macro2 = "1.0"
syn = "2.0"
prettyplease = "0.2"
+rhai = "1.24.0"
mach2 = "0.6"
cbindgen = "0.29"
ctrlc = "3.5"
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 873f2595d..cfe806582 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -71,12 +71,14 @@ cbindgen.workspace = true
# metal on macos requires clang-ast for dsl build
[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.build-dependencies]
clang-ast = { workspace = true, optional = true }
+rhai = { workspace = true, optional = true }
[features]
default = ["metal"]
metal = [
"dep:clang-ast",
+ "dep:rhai",
"dep:objc2",
"dep:objc2-foundation",
"dep:bytesize",
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rs
index 8bbc95f8b..c8e789d6a 100644
--- a/crates/uzu/build/metal/ast.rs
+++ b/crates/uzu/build/metal/ast.rs
@@ -327,6 +327,7 @@ pub struct MetalKernelInfo {
pub name: Box<str>,
pub arguments: Box<[MetalArgument]>,
pub variants: Option<Box<[MetalTemplateParameter]>>,
+ pub constraints: Box<[Box<str>]>,
}
impl MetalKernelInfo {
@@ -543,6 +544,18 @@ impl MetalKernelInfo {
None
};
+ let constraints: Box<[_]> = annotations
+ .iter()
+ .filter(|(k, _)| k.as_ref() == "dsl.constraint")
+ .map(|(_, v)| {
+ let [constraint_expr] = v.as_ref() else {
+ bail!("malformed dsl.constraint annotation");
+ };
+
+ Ok(constraint_expr.clone())
+ })
+ .collect::<anyhow::Result<_>>()?;
+
let arguments = arg_nodes
.into_iter()
.map(|an| MetalArgument::from_ast_node_and_source(an, source))
@@ -553,6 +566,7 @@ impl MetalKernelInfo {
name,
arguments,
variants,
+ constraints,
}))
}
}
diff --git a/crates/uzu/build/metal/wrapper.rs b/crates/uzu/build/metal/wrapper.rs
index 6e63c71df..32e781924 100644
--- a/crates/uzu/build/metal/wrapper.rs
+++ b/crates/uzu/build/metal/wrapper.rs
@@ -82,6 +82,7 @@ fn kernel_wrappers(
Vec::new()
};
+ let engine = rhai::Engine::new();
for type_variant in if let Some(variants) = &kernel.variants {
variants
.iter()
@@ -100,6 +101,25 @@ fn kernel_wrappers(
} else {
vec![None]
} {
+ if let Some(type_variant) = &type_variant {
+ let mut scope = rhai::Scope::with_capacity(type_variant.len());
+ for (type_variant_name, type_variant_value) in type_variant {
+ scope.push(
+ type_variant_name.clone(),
+ engine
+ .eval_expression::<rhai::Dynamic>(type_variant_value.as_ref())
+ .unwrap_or_else(|_| type_variant_value.clone().into()),
+ );
+ }
+ if !kernel
+ .constraints
+ .iter()
+ .all(|constraint_expr| engine.eval_expression_with_scope(&mut scope, constraint_expr.as_ref()).unwrap())
+ {
+ continue;
+ }
+ }
+
let (wrapper_name, underlying_name) = if let Some(type_variant) = &type_variant {
(
static_mangle(kernel.name.as_ref(), type_variant.iter().map(|(_k, v)| v.as_str())),
diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rs
index e0f18747a..5bcd36c47 100644
--- a/crates/uzu/src/backends/common/backend.rs
+++ b/crates/uzu/src/backends/common/backend.rs
@@ -1,13 +1,13 @@
use std::{error::Error, fmt::Debug};
-use super::{Buffer, CommandBuffer, Context, Event, Kernels, kernel::matmul::MatmulKernels};
+use super::{Buffer, CommandBuffer, Context, Event, Kernels, kernel::ManualKernels};
pub trait Backend: Debug + Clone + 'static {
type Context: Context<Backend = Self>;
type CommandBuffer: CommandBuffer<Backend = Self>;
type Buffer: Buffer<Backend = Self>;
type Event: Event<Backend = Self>;
- type Kernels: Kernels<Backend = Self> + MatmulKernels;
+ type Kernels: Kernels<Backend = Self> + ManualKernels;
type Error: Error + Debug;
const MIN_ALLOCATION_ALIGNMENT: usize;
diff --git a/crates/uzu/src/backends/common/gpu_types/matmul.rs b/crates/uzu/src/backends/common/gpu_types/matmul.rs
index 96713f917..a9c496065 100644
--- a/crates/uzu/src/backends/common/gpu_types/matmul.rs
+++ b/crates/uzu/src/backends/common/gpu_types/matmul.rs
@@ -3,14 +3,14 @@
#[repr(C)]
#[derive(Debug, Default, Copy, Clone)]
pub struct GemmParams {
- pub M: i32,
- pub N: i32,
- pub K: i32,
- pub leading_dimension_a: i32,
- pub leading_dimension_b: i32,
- pub leading_dimension_d: i32,
- pub threadgroups_per_column: i32,
- pub threadgroups_per_row: i32,
- pub swizzle_log: i32,
- pub aligned_inner_iterations: i32,
+ pub M: u32,
+ pub N: u32,
+ pub K: u32,
+ pub leading_dimension_a: u32,
+ pub leading_dimension_b: u32,
+ pub leading_dimension_d: u32,
+ pub threadgroups_per_column: u32,
+ pub threadgroups_per_row: u32,
+ pub swizzle_log: u32,
+ pub aligned_inner_iterations: u32,
}
diff --git a/crates/uzu/src/backends/common/kernel/matmul.rs b/crates/uzu/src/backends/common/kernel/matmul.rs
new file mode 100644
index 000000000..ce5b3df03
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul.rs
@@ -0,0 +1,61 @@
+use thiserror::Error;
+
+use crate::{
+ DataType,
+ backends::common::{Backend, Encoder, kernel::ManualKernels},
+};
+
+#[derive(Debug, Error)]
+pub enum MatmulError<B: Backend> {
+ #[error("Unsupported data type: {0:?}")]
+ UnsupportedDataType(DataType),
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+}
+
+#[derive(Debug)]
+pub enum MatmulArgumentC<'a, B: Backend> {
+ None,
+ /// Accumulate: [M, N]
+ Accumulate,
+ /// Bias: [N] (broadcasted across M/batch)
+ Bias(&'a B::Buffer),
+}
+
+// D = ab_scale * (A @ B.T) + C
+#[derive(Debug)]
+pub struct MatmulArguments<'a, B: Backend> {
+ /// A: [M, K]
+ pub a: &'a B::Buffer,
+ pub a_offset: u64,
+ /// B: [N, K]
+ pub b: &'a B::Buffer,
+ /// AB scale: also known as alpha
+ pub ab_scale: f32,
+ /// C: behavior depends on enum variant
+ pub c: MatmulArgumentC<'a, B>,
+ /// D: [M, N]
+ pub d: &'a mut B::Buffer,
+ /// M dimension: usually batch/number of tokens (rows of A, rows of C)
+ pub batch_dim: u32,
+ /// K dimension: usually input_dim/reduction dimension (cols of A, rows of B)
+ pub input_dim: u32,
+ /// N dimension: usually output_dim (cols of B, cols of C)
+ pub output_dim: u32,
+}
+
+pub trait MatmulKernel: Sized {
+ type Backend: Backend<Kernels: ManualKernels<MatmulKernel = Self>>;
+
+ fn new(
+ context: &<Self::Backend as Backend>::Context,
+ data_type: DataType,
+ ) -> Result<Self, MatmulError<Self::Backend>>;
+
+ fn encode(
+ &mut self,
+ context: &<Self::Backend as Backend>::Context,
+ arguments: MatmulArguments<Self::Backend>,
+ encoder: &mut Encoder<Self::Backend>,
+ );
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
deleted file mode 100644
index a432f4e0b..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-use crate::backends::common::Backend;
-
-#[derive(Debug)]
-pub struct MatmulArguments<'a, B: Backend> {
- pub a: &'a B::Buffer,
- /// Byte offset into `a` (used for slicing the batch dimension).
- pub a_offset: u64,
- pub b: &'a B::Buffer,
- pub d: &'a mut B::Buffer,
- pub bias: Option<&'a B::Buffer>,
- /// M dimension - batch/number of tokens (rows of A, rows of D)
- pub batch: i32,
- /// K dimension - input_dim/reduction dimension (cols of A, rows of B)
- pub input_dim: i32,
- /// N dimension - output_dim (cols of B, cols of D)
- pub output_dim: i32,
- pub leading_dimension_a: i32,
- pub leading_dimension_b: i32,
- pub leading_dimension_d: i32,
- pub transpose_b: bool,
-}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
deleted file mode 100644
index 78a99f7d8..000000000
--- a/crates/uzu/src/backends/common/kernel/matmul/mod.rs
+++ /dev/null
@@ -1,40 +0,0 @@
-mod matmul_arguments;
-
-pub use matmul_arguments::MatmulArguments;
-use thiserror::Error;
-
-use super::Kernels;
-use crate::{
- DataType,
- backends::common::{Backend, Encoder},
-};
-
-pub trait MatmulKernels: Kernels {
- type MatmulKernel: MatmulKernel<Backend = Self::Backend>;
-}
-
-pub trait MatmulKernel: Sized {
- type Backend: Backend<Kernels: MatmulKernels<MatmulKernel = Self>>;
-
- fn new(
- context: &<Self::Backend as Backend>::Context,
- data_type: DataType,
- ) -> Result<Self, MatmulError<Self::Backend>>;
-
- fn encode(
- &mut self,
- context: &<Self::Backend as Backend>::Context,
- arguments: MatmulArguments<Self::Backend>,
- encoder: &mut Encoder<Self::Backend>,
- );
-}
-
-#[derive(Debug, Error)]
-pub enum MatmulError<B: Backend> {
- #[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(DataType),
- #[error("Threadgroup dimension overflows u32: {0}")]
- ThreadgroupOverflow(usize),
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
-}
diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rs
index b5a284288..a2659f323 100644
--- a/crates/uzu/src/backends/common/kernel/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/mod.rs
@@ -2,6 +2,10 @@
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
+pub trait ManualKernels: Kernels {
+ type MatmulKernel: matmul::MatmulKernel<Backend = Self::Backend>;
+}
+
pub mod attention;
pub mod kv_cache_update;
pub mod matmul;
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
index 8c71e6dbf..eb9fe033b 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
@@ -9,6 +9,7 @@ use crate::{
unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(
a: SendPtr<T>,
b: SendPtr<T>,
+ ab_scale: f32,
d: SendPtrMut<T>,
row: usize,
n: usize,
@@ -16,6 +17,7 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(
leading_dimension_a: usize,
leading_dimension_b: usize,
leading_dimension_d: usize,
+ is_accumulate: bool,
) {
unsafe {
for col in 0..n {
@@ -25,7 +27,12 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(
let b_val = *b.0.add(col * leading_dimension_b + i);
acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();
}
- *d.0.add(row * leading_dimension_d + col) = T::from(acc).unwrap();
+ let d_element = d.0.add(row * leading_dimension_d + col);
+ acc *= ab_scale;
+ if is_accumulate {
+ acc += (*d_element).to_f32().unwrap();
+ }
+ *d_element = T::from(acc).unwrap();
}
}
}
@@ -33,6 +40,7 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(
pub fn matmul_gemm_impl<T: ArrayElement + Float>(
a: *const T,
b: *const T,
+ ab_scale: f32,
d: *mut T,
m: usize,
n: usize,
@@ -40,6 +48,7 @@ pub fn matmul_gemm_impl<T: ArrayElement + Float>(
leading_dimension_a: usize,
leading_dimension_b: usize,
leading_dimension_d: usize,
+ is_accumulate: bool,
) {
let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
let a = SendPtr(a);
@@ -66,6 +75,7 @@ pub fn matmul_gemm_impl<T: ArrayElement + Float>(
matmul_gemm_compute_row(
a,
b,
+ ab_scale,
d,
row,
n,
@@ -73,6 +83,7 @@ pub fn matmul_gemm_impl<T: ArrayElement + Float>(
leading_dimension_a,
leading_dimension_b,
leading_dimension_d,
+ is_accumulate,
);
}
});
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 3991c4f8a..99213a5b7 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -5,7 +5,7 @@ use crate::{
backends::{
common::{
Encoder,
- kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},
+ kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
},
cpu::{Cpu, context::CpuContext},
},
@@ -37,25 +37,32 @@ impl MatmulKernel for MatmulCpuKernel {
encoder: &mut Encoder<Cpu>,
) {
let command_buffer = encoder.as_command_buffer_mut();
- let m = arguments.batch as usize;
+ let m = arguments.batch_dim as usize;
let n = arguments.output_dim as usize;
let k = arguments.input_dim as usize;
- let lda = arguments.leading_dimension_a as usize;
- let ldb = arguments.leading_dimension_b as usize;
- let ldd = arguments.leading_dimension_d as usize;
+ let lda = k;
+ let ldb = k;
+ let ldd = n;
let a_offset = arguments.a_offset as usize;
let data_type = self.data_type;
let a_ptr = unsafe { &*arguments.a.get() }.as_ptr().wrapping_byte_add(a_offset);
let b_ptr = unsafe { &*arguments.b.get() }.as_ptr();
+ let ab_scale = arguments.ab_scale;
let d_ptr = arguments.d.get_mut().as_mut_ptr();
- let bias_ptr = arguments.bias.map(|b| unsafe { &*b.get() }.as_ptr());
+
+ let (is_accumulate, bias_ptr) = match arguments.c {
+ MatmulArgumentC::Accumulate => (true, None),
+ MatmulArgumentC::Bias(bias) => (false, Some(unsafe { &*bias.get() }.as_ptr())),
+ MatmulArgumentC::None => (false, None),
+ };
command_buffer.push_command(move || match data_type {
DataType::F32 => {
super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f32>(
a_ptr as *const f32,
b_ptr as *const f32,
+ ab_scale,
d_ptr as *mut f32,
m,
n,
@@ -63,6 +70,7 @@ impl MatmulKernel for MatmulCpuKernel {
lda,
ldb,
ldd,
+ is_accumulate,
);
if let Some(bias) = bias_ptr {
apply_bias::<f32>(d_ptr as *mut f32, bias as *const f32, m, n);
@@ -72,6 +80,7 @@ impl MatmulKernel for MatmulCpuKernel {
super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f16>(
a_ptr as *const f16,
b_ptr as *const f16,
+ ab_scale,
d_ptr as *mut f16,
m,
n,
@@ -79,6 +88,7 @@ impl MatmulKernel for MatmulCpuKernel {
lda,
ldb,
ldd,
+ is_accumulate,
);
if let Some(bias) = bias_ptr {
apply_bias::<f16>(d_ptr as *mut f16, bias as *const f16, m, n);
@@ -88,6 +98,7 @@ impl MatmulKernel for MatmulCpuKernel {
super::gemm::shaders::steel_gemm::matmul_gemm_impl::<bf16>(
a_ptr as *const bf16,
b_ptr as *const bf16,
+ ab_scale,
d_ptr as *mut bf16,
m,
n,
@@ -95,6 +106,7 @@ impl MatmulKernel for MatmulCpuKernel {
lda,
ldb,
ldd,
+ is_accumulate,
);
if let Some(bias) = bias_ptr {
apply_bias::<bf16>(d_ptr as *mut bf16, bias as *const bf16, m, n);
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
index 23d8c2f63..bbdde8210 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/mod.rs
@@ -4,8 +4,8 @@ mod matmul;
pub use matmul::MatmulCpuKernel;
-use crate::backends::{common::kernel::matmul::MatmulKernels, cpu::kernel::CpuKernels};
+use crate::backends::{common::kernel::ManualKernels, cpu::kernel::CpuKernels};
-impl MatmulKernels for CpuKernels {
+impl ManualKernels for CpuKernels {
type MatmulKernel = MatmulCpuKernel;
}
diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rs
index bd242efb9..03fa26a94 100644
--- a/crates/uzu/src/backends/metal/backend.rs
+++ b/crates/uzu/src/backends/metal/backend.rs
@@ -1,7 +1,7 @@
use metal::{MTLBuffer, MTLEvent};
use objc2::{rc::Retained, runtime::ProtocolObject};
-use super::{command_buffer::MetalCommandBuffer, context::MetalContext, error::MetalError, kernel::dsl::MetalKernels};
+use super::{command_buffer::MetalCommandBuffer, context::MetalContext, error::MetalError, kernel::MetalKernels};
use crate::backends::common::Backend;
#[derive(Debug, Clone)]
diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/generated/matmul.h
index 86d45c6ec..c3fdf827e 100644
--- a/crates/uzu/src/backends/metal/generated/matmul.h
+++ b/crates/uzu/src/backends/metal/generated/matmul.h
@@ -15,16 +15,16 @@ namespace matmul {
#endif
typedef struct {
- int32_t M;
- int32_t N;
- int32_t K;
- int32_t leading_dimension_a;
- int32_t leading_dimension_b;
- int32_t leading_dimension_d;
- int32_t threadgroups_per_column;
- int32_t threadgroups_per_row;
- int32_t swizzle_log;
- int32_t aligned_inner_iterations;
+ uint32_t M;
+ uint32_t N;
+ uint32_t K;
+ uint32_t leading_dimension_a;
+ uint32_t leading_dimension_b;
+ uint32_t leading_dimension_d;
+ uint32_t threadgroups_per_column;
+ uint32_t threadgroups_per_row;
+ uint32_t swizzle_log;
+ uint32_t aligned_inner_iterations;
} GemmParams;
#ifdef __METAL_VERSION__
diff --git a/crates/uzu/src/backends/metal/kernel/common/dsl.h b/crates/uzu/src/backends/metal/kernel/common/dsl.h
index bae47519e..3bda99637 100644
--- a/crates/uzu/src/backends/metal/kernel/common/dsl.h
+++ b/crates/uzu/src/backends/metal/kernel/common/dsl.h
@@ -20,6 +20,7 @@ using namespace metal;
#define VARIANTS(TYPENAME, ...) \
DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)
+#define CONSTRAINT(C) DSL_META("dsl.constraint", #C)
#define PUBLIC DSL_META("dsl.public")
#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
index 4e4401177..c441e1f58 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
@@ -68,7 +68,8 @@ struct TransformScaleAccumulate {
METAL_FUNC OutT apply(InT x, OutT c) const {
return static_cast<OutT>(
- x * static_cast<InT>(alpha) + (static_cast<OutT>(beta) * c)
+ x * static_cast<InT>(alpha) +
+ (beta != 0.0f ? (static_cast<OutT>(beta) * c) : static_cast<OutT>(0.0))
);
}
};
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
index 8cee3d832..88130e664 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
@@ -150,6 +150,8 @@ struct ThreadgroupGemm {
const device T* b,
device U* d,
const constant GemmParams* params,
+ const constant float& ab_scale,
+ const bool is_accumulate,
threadgroup T* a_shared,
threadgroup T* b_shared,
uint simd_lane_id [[thread_index_in_simdgroup]],
@@ -199,6 +201,11 @@ struct ThreadgroupGemm {
thread ThreadgroupTileType threadgroup_tile(simd_group_id, simd_lane_id);
+ TransformScaleAccumulate<AccumType, AccumType> epilogue(
+ ab_scale,
+ is_accumulate ? 1.0f : 0.0f
+ );
+
int gemm_k_iterations = params->aligned_inner_iterations;
///////////////////////////////////////////////////////////////////////////
@@ -237,14 +244,18 @@ struct ThreadgroupGemm {
threadgroup_tile.multiply_accumulate(a_shared, b_shared);
}
+ threadgroup_tile
+ .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
threadgroup_tile.store_result(d, params->leading_dimension_d);
return;
}
///////////////////////////////////////////////////////////////////////////
// MN unaligned loop
else {
- short threadgroup_block_rows = min(BLOCK_ROWS, params->M - output_row);
- short threadgroup_block_cols = min(BLOCK_COLS, params->N - output_col);
+ short threadgroup_block_rows =
+ min(BLOCK_ROWS, ((int)params->M) - output_row);
+ short threadgroup_block_cols =
+ min(BLOCK_COLS, ((int)params->N) - output_col);
short leftover_block_depth =
params->K - params->aligned_inner_iterations * BLOCK_DEPTH;
@@ -262,6 +273,8 @@ struct ThreadgroupGemm {
leftover_block_depth
);
+ threadgroup_tile
+ .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
threadgroup_tile.store_result(d, params->leading_dimension_d);
return;
@@ -278,6 +291,13 @@ struct ThreadgroupGemm {
leftover_block_depth
);
+ threadgroup_tile.apply_epilogue_safe(
+ d,
+ params->leading_dimension_d,
+ 1,
+ short2(threadgroup_block_cols, threadgroup_block_rows),
+ epilogue
+ );
threadgroup_tile.store_result_safe(
d,
params->leading_dimension_d,
@@ -298,6 +318,13 @@ struct ThreadgroupGemm {
leftover_block_depth
);
+ threadgroup_tile.apply_epilogue_safe(
+ d,
+ params->leading_dimension_d,
+ 1,
+ short2(threadgroup_block_cols, threadgroup_block_rows),
+ epilogue
+ );
threadgroup_tile.store_result_safe(
d,
params->leading_dimension_d,
@@ -318,6 +345,13 @@ struct ThreadgroupGemm {
leftover_block_depth
);
+ threadgroup_tile.apply_epilogue_safe(
+ d,
+ params->leading_dimension_d,
+ 1,
+ short2(threadgroup_block_cols, threadgroup_block_rows),
+ epilogue
+ );
threadgroup_tile.store_result_safe(
d,
params->leading_dimension_d,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal
new file mode 100644
index 000000000..9f6d548fb
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal
@@ -0,0 +1,65 @@
+#include "../common/utils.h"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+
+#include "common/gemm.h"
+
+using namespace uzu::matmul;
+
+#define GEMM_MAX_THREADGROUP_A 2560
+#define GEMM_MAX_THREADGROUP_B 1536
+
+template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint BLOCK_DEPTH, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN, bool MN_ALIGNED, bool K_ALIGNED>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(BLOCK_ROWS, 32, 64)
+VARIANTS(BLOCK_COLS, 32, 64)
+VARIANTS(BLOCK_DEPTH, 8, 16, 32)
+VARIANTS(SIMDGROUPS_PER_ROW, 1, 2, 4)
+VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2)
+VARIANTS(MN_ALIGNED, false, true)
+VARIANTS(K_ALIGNED, false, true)
+CONSTRAINT(max(BLOCK_ROWS, BLOCK_COLS) <= 32 * SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN)
+KERNEL(MatmulGemm)(
+ const device T* a,
+ const device T* b,
+ device T* d,
+ const constant uzu::matmul::GemmParams* params,
+ const constant uint& group_count_x,
+ const constant uint& group_count_y,
+ const constant float& ab_scale,
+ const bool is_accumulate SPECIALIZE,
+ threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],
+ threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],
+ const uint group_x GROUPS(group_count_x),
+ const uint group_y GROUPS(group_count_y),
+ const uint thread_x THREADS(32),
+ const uint thread_y THREADS(2),
+ const uint thread_z THREADS(2),
+ const ThreadContext thread_context
+) {
+ ThreadgroupGemm<
+ T,
+ T,
+ BLOCK_ROWS,
+ BLOCK_COLS,
+ BLOCK_DEPTH,
+ SIMDGROUPS_PER_ROW,
+ SIMDGROUPS_PER_COLUMN,
+ false, // transpose_a
+ true, // transpose_b
+ MN_ALIGNED,
+ K_ALIGNED,
+ float>::
+ run(a,
+ b,
+ d,
+ params,
+ ab_scale,
+ is_accumulate,
+ a_shared,
+ b_shared,
+ thread_context.simdgroup_index,
+ thread_context.threadgroup_index,
+ uint2(group_x, group_y),
+ uint3(thread_x, thread_y, thread_z));
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm.rs
similarity index 57%
rename from crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
rename to crates/uzu/src/backends/metal/kernel/matmul/gemm.rs
index 36e9db557..1d31ba2cd 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm.rs
@@ -1,22 +1,24 @@
use crate::{
DataType,
backends::{
- common::{Context, kernel::matmul::MatmulArguments},
+ common::{
+ Context,
+ kernel::matmul::{MatmulArgumentC, MatmulArguments},
+ },
metal::{Metal, context::MetalContext},
},
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct GemmSpecialization {
- pub block_rows: i32,
- pub block_cols: i32,
- pub block_depth: i32,
- pub simdgroups_per_row: u64,
- pub simdgroups_per_column: u64,
- pub swizzle_log2: i32,
- pub align_m: bool,
- pub align_n: bool,
+ pub block_rows: u32,
+ pub block_cols: u32,
+ pub block_depth: u32,
+ pub simdgroups_per_row: u32,
+ pub simdgroups_per_column: u32,
+ pub align_mn: bool,
pub align_k: bool,
+ pub is_accumulate: bool,
}
impl GemmSpecialization {
@@ -29,10 +31,9 @@ impl GemmSpecialization {
block_depth: 32,
simdgroups_per_row: 2,
simdgroups_per_column: 2,
- swizzle_log2: 0,
- align_m: false,
- align_n: true,
+ align_mn: false,
align_k: true,
+ is_accumulate: false,
},
Self {
block_rows: 64,
@@ -40,10 +41,9 @@ impl GemmSpecialization {
block_depth: 32,
simdgroups_per_row: 2,
simdgroups_per_column: 2,
- swizzle_log2: 0,
- align_m: true,
- align_n: true,
+ align_mn: true,
align_k: true,
+ is_accumulate: false,
},
Self {
block_rows: 64,
@@ -51,10 +51,9 @@ impl GemmSpecialization {
block_depth: 16,
simdgroups_per_row: 2,
simdgroups_per_column: 2,
- swizzle_log2: 0,
- align_m: false,
- align_n: true,
+ align_mn: false,
align_k: true,
+ is_accumulate: false,
},
Self {
block_rows: 64,
@@ -62,21 +61,9 @@ impl GemmSpecialization {
block_depth: 16,
simdgroups_per_row: 2,
simdgroups_per_column: 2,
- swizzle_log2: 0,
- align_m: true,
- align_n: false,
- align_k: true,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- block_depth: 16,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- swizzle_log2: 0,
- align_m: true,
- align_n: true,
+ align_mn: true,
align_k: true,
+ is_accumulate: false,
},
Self {
block_rows: 64,
@@ -84,10 +71,9 @@ impl GemmSpecialization {
block_depth: 16,
simdgroups_per_row: 1,
simdgroups_per_column: 2,
- swizzle_log2: 0,
- align_m: true,
- align_n: true,
+ align_mn: true,
align_k: true,
+ is_accumulate: false,
},
],
DataType::F16 => &[
@@ -97,10 +83,9 @@ impl GemmSpecialization {
block_depth: 16,
simdgroups_per_row: 2,
simdgroups_per_column: 2,
- swizzle_log2: 0,
- align_m: true,
- align_n: true,
+ align_mn: true,
align_k: true,
+ is_accumulate: false,
},
Self {
block_rows: 64,
@@ -108,10 +93,9 @@ impl GemmSpecialization {
block_depth: 16,
simdgroups_per_row: 2,
simdgroups_per_column: 2,
- swizzle_log2: 0,
- align_m: false,
- align_n: true,
+ align_mn: false,
align_k: true,
+ is_accumulate: false,
},
],
DataType::F32 => &[
@@ -121,10 +105,9 @@ impl GemmSpecialization {
block_depth: 16,
simdgroups_per_row: 2,
simdgroups_per_column: 2,
- swizzle_log2: 0,
- align_m: false,
- align_n: true,
+ align_mn: false,
align_k: true,
+ is_accumulate: false,
},
Self {
block_rows: 32,
@@ -132,10 +115,9 @@ impl GemmSpecialization {
block_depth: 16,
simdgroups_per_row: 2,
simdgroups_per_column: 2,
- swizzle_log2: 0,
- align_m: true,
- align_n: true,
+ align_mn: true,
align_k: true,
+ is_accumulate: false,
},
],
_ => &[],
@@ -147,51 +129,36 @@ impl GemmSpecialization {
data_type: DataType,
arguments: &MatmulArguments<Metal>,
) -> Self {
- let overall_work_elements = (arguments.batch as i64) * (arguments.output_dim as i64);
+ let overall_work_elements = arguments.batch_dim * arguments.output_dim;
let is_float32 = matches!(data_type, DataType::F32);
let prefer_half_or_tf32 = !is_float32 || MetalContext::tf32_enabled();
- let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column, swizzle_log2) =
- if context.is_high_performance() && overall_work_elements >= (1_i64 << 20) {
+ let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column) =
+ if context.is_high_performance() && overall_work_elements >= (1 << 20) {
if prefer_half_or_tf32 {
- if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {
- (64, 64, 16, 2, 2, 0)
- } else if arguments.transpose_b {
- (64, 32, 32, 2, 2, 0)
+ if 2 * std::cmp::max(arguments.batch_dim, arguments.output_dim) > arguments.input_dim {
+ (64, 64, 16, 2, 2)
} else {
- (32, 64, 16, 2, 2, 0)
+ (64, 32, 32, 2, 2)
}
- } else if arguments.transpose_b {
- (32, 64, 16, 2, 2, 0)
} else {
- (64, 32, 32, 2, 2, 0)
+ (32, 64, 16, 2, 2)
}
} else if prefer_half_or_tf32 {
- if arguments.transpose_b {
- (64, 32, 32, 2, 2, 0)
- } else {
- (64, 64, 16, 2, 2, 0)
- }
- } else if arguments.transpose_b {
- (32, 64, 16, 2, 2, 0)
+ (64, 32, 32, 2, 2)
} else {
- (64, 32, 32, 2, 2, 0)
+ (32, 64, 16, 2, 2)
};
- let m = arguments.batch;
- let n = arguments.output_dim;
- let k = arguments.input_dim;
-
Self {
block_rows,
block_cols,
block_depth,
simdgroups_per_row,
simdgroups_per_column,
- swizzle_log2,
- align_m: (m % block_rows) == 0,
- align_n: (n % block_cols) == 0,
- align_k: (k % block_depth) == 0,
+ align_mn: (arguments.batch_dim % block_rows) == 0 && (arguments.output_dim % block_cols) == 0,
+ align_k: (arguments.input_dim % block_depth) == 0,
+ is_accumulate: matches!(arguments.c, MatmulArgumentC::Accumulate),
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
deleted file mode 100644
index 14f9855ce..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-mod specialization;
-
-pub use specialization::GemmSpecialization;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metal
deleted file mode 100644
index d020bced4..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/gemm.metal
+++ /dev/null
@@ -1,82 +0,0 @@
-// clang-format off
-#include "../../../common/utils.h"
-#include "../../../common/dsl.h"
-#include "../../../common/thread_context.h"
-
-#include "../../common/gemm.h"
-
-using namespace uzu::matmul;
-
-template <typename T, int BLOCK_ROWS, int BLOCK_COLS, int BLOCK_DEPTH, int SIMDGROUPS_PER_ROW, int SIMDGROUPS_PER_COLUMN, bool MN_aligned, bool K_aligned>
-METAL_FUNC void gemm_dispatch(
- const device T* a,
- const device T* b,
- device T* d,
- const constant GemmParams* params,
- threadgroup T* a_shared,
- threadgroup T* b_shared,
- uint simd_lane_id,
- uint simd_group_id,
- uint2 tid,
- uint3 lid
-) {
- using ThreadgroupGemmType = ThreadgroupGemm<T, T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true, MN_aligned, K_aligned, float>;
- ThreadgroupGemmType::run(a, b, d, params, a_shared, b_shared, simd_lane_id, simd_group_id, tid, lid);
-}
-
-#define GEMM_DISPATCH(T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN) \
- if (block_rows == BLOCK_ROWS && block_cols == BLOCK_COLS && block_depth == BLOCK_DEPTH && \
- simdgroups_per_row == SIMDGROUPS_PER_ROW && simdgroups_per_column == SIMDGROUPS_PER_COLUMN) { \
- if (align_m && align_n) { \
- if (align_k) \
- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, true>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
- else \
- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, false>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
- } else { \
- if (align_k) \
- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
- else \
- gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, false>(a, b, d, params, a_shared, b_shared, thread_context.simdgroup_index, thread_context.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
- } \
- return; \
- }
-
-#define GEMM_MAX_THREADGROUP_A 2560
-#define GEMM_MAX_THREADGROUP_B 1536
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-KERNEL(MatmulGemm)(
- const device T* a,
- const device T* b,
- device T* d,
- const constant uzu::matmul::GemmParams* params,
- const constant uint& group_count_x,
- const constant uint& group_count_y,
- threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],
- threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],
- const uint block_rows SPECIALIZE,
- const uint block_cols SPECIALIZE,
- const uint block_depth SPECIALIZE,
- const uint simdgroups_per_row SPECIALIZE,
- const uint simdgroups_per_column SPECIALIZE,
- const bool align_m SPECIALIZE,
- const bool align_n SPECIALIZE,
- const bool align_k SPECIALIZE,
- const uint group_x GROUPS(group_count_x),
- const uint group_y GROUPS(group_count_y),
- const uint thread_x THREADS(32),
- const uint thread_y THREADS(2),
- const uint thread_z THREADS(2),
- const ThreadContext thread_context
-) {
- GEMM_DISPATCH(T, 64, 64, 16, 2, 2)
- GEMM_DISPATCH(T, 64, 64, 16, 1, 2)
- GEMM_DISPATCH(T, 64, 32, 32, 2, 2)
- GEMM_DISPATCH(T, 32, 64, 16, 1, 2)
- GEMM_DISPATCH(T, 32, 64, 16, 2, 2)
- GEMM_DISPATCH(T, 32, 32, 16, 2, 2)
- GEMM_DISPATCH(T, 64, 32, 8, 4, 1)
-}
-
-// clang-format on
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
similarity index 71%
rename from crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
rename to crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
index 169715ce6..3187bee36 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
@@ -1,6 +1,6 @@
-#include "../../../common/utils.h"
-#include "../../../common/dsl.h"
-#include "../../../common/thread_context.h"
+#include "../common/utils.h"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
#include <metal_simdgroup>
@@ -16,16 +16,14 @@ VARIANTS(T, float, half, bfloat)
KERNEL(MatmulGemv)(
const device T* matrix,
const device T* input_vector,
- const device T* output_source OPTIONAL(apply_output_scale_and_accumulate),
+ const device T* output_bias OPTIONAL(is_bias),
device T* output_vector,
- const constant int& input_dimension,
- const constant int& output_dimension,
- const constant int& matrix_leading_dimension,
+ const constant uint& input_dimension,
+ const constant uint& output_dimension,
+ const constant uint& matrix_leading_dimension,
const constant float& output_scale,
- const constant float& output_accumulate_scale,
- const constant int& output_source_stride,
- const constant int& batch_rows,
- const constant int& output_rows_per_threadgroup,
+ const constant uint& batch_rows,
+ const constant uint& output_rows_per_threadgroup,
threadgroup float threadgroup_memory[GEMV_MAX_THREADGROUP_MEMORY],
const uint tg_simd_rows SPECIALIZE,
const uint tg_simd_cols SPECIALIZE,
@@ -33,7 +31,8 @@ KERNEL(MatmulGemv)(
const uint sg_thread_cols SPECIALIZE,
const uint thread_out_rows SPECIALIZE,
const uint thread_out_cols SPECIALIZE,
- const bool apply_output_scale_and_accumulate SPECIALIZE,
+ const bool is_accumulate SPECIALIZE,
+ const bool is_bias SPECIALIZE,
const uint threadgroup_index_x GROUPS((output_dimension + output_rows_per_threadgroup - 1) / output_rows_per_threadgroup),
const uint threadgroup_index_y GROUPS(batch_rows),
const uint thread_index_x THREADS(32),
@@ -51,11 +50,11 @@ KERNEL(MatmulGemv)(
}
// Derived constants from SPECIALIZE params
- const int threads_per_threadgroup_col = tg_simd_cols * sg_thread_cols;
- const int input_columns_per_threadgroup =
+ const uint threads_per_threadgroup_col = tg_simd_cols * sg_thread_cols;
+ const uint input_columns_per_threadgroup =
threads_per_threadgroup_col * thread_out_cols;
- const int batch_row = static_cast<int>(threadgroup_index_y);
+ const uint batch_row = static_cast<uint>(threadgroup_index_y);
if (batch_row >= batch_rows) {
return;
}
@@ -65,40 +64,40 @@ KERNEL(MatmulGemv)(
thread T matrix_values[4];
thread float vector_coefficients[4];
- const int thread_row_in_simdgroup =
+ const uint thread_row_in_simdgroup =
sg_thread_cols != 32
- ? int(thread_context.simdgroup_index) / int(sg_thread_cols)
+ ? uint(thread_context.simdgroup_index) / uint(sg_thread_cols)
: 0;
const int thread_col_in_simdgroup =
sg_thread_cols != 32
- ? int(thread_context.simdgroup_index) % int(sg_thread_cols)
- : int(thread_context.simdgroup_index);
+ ? uint(thread_context.simdgroup_index) % uint(sg_thread_cols)
+ : uint(thread_context.simdgroup_index);
const int simdgroup_column_index =
- tg_simd_cols != 1 ? int(thread_context.threadgroup_index % tg_simd_cols)
+ tg_simd_cols != 1 ? uint(thread_context.threadgroup_index % tg_simd_cols)
: 0;
- const int simdgroup_row_thread_base =
+ const uint simdgroup_row_thread_base =
tg_simd_cols != 1
- ? int(sg_thread_rows) *
- int(thread_context.threadgroup_index / tg_simd_cols)
- : int(sg_thread_rows) * int(thread_context.threadgroup_index);
- const int simdgroup_col_thread_base =
+ ? uint(sg_thread_rows) *
+ uint(thread_context.threadgroup_index / tg_simd_cols)
+ : uint(sg_thread_rows) * uint(thread_context.threadgroup_index);
+ const uint simdgroup_col_thread_base =
tg_simd_cols != 1
- ? int(sg_thread_cols) *
- int(thread_context.threadgroup_index % tg_simd_cols)
+ ? uint(sg_thread_cols) *
+ uint(thread_context.threadgroup_index % tg_simd_cols)
: 0;
- int output_block_row_offset =
+ uint output_block_row_offset =
(simdgroup_row_thread_base + thread_row_in_simdgroup) *
- int(thread_out_rows);
- int input_block_col_offset =
+ uint(thread_out_rows);
+ uint input_block_col_offset =
(simdgroup_col_thread_base + thread_col_in_simdgroup) *
- int(thread_out_cols);
+ uint(thread_out_cols);
// Block position
- int output_row_start =
- int(threadgroup_index_x) * int(output_rows_per_threadgroup) +
+ uint output_row_start =
+ uint(threadgroup_index_x) * uint(output_rows_per_threadgroup) +
output_block_row_offset;
// Exit simdgroup if rows out of bound
@@ -106,23 +105,24 @@ KERNEL(MatmulGemv)(
return;
// Adjust tail simdgroup to ensure in bound reads
- output_row_start = output_row_start + int(thread_out_rows) <= output_dimension
- ? output_row_start
- : output_dimension - int(thread_out_rows);
+ output_row_start =
+ output_row_start + uint(thread_out_rows) <= output_dimension
+ ? output_row_start
+ : output_dimension - uint(thread_out_rows);
const device T* thread_matrix =
matrix + output_row_start * matrix_leading_dimension;
- const uniform<int> input_block_stride =
- make_uniform(int(input_columns_per_threadgroup));
- const uniform<int> input_vector_length = make_uniform(input_dimension);
- const uniform<int> full_input_blocks =
+ const uniform<uint> input_block_stride =
+ make_uniform(uint(input_columns_per_threadgroup));
+ const uniform<uint> input_vector_length = make_uniform(input_dimension);
+ const uniform<uint> full_input_blocks =
input_vector_length / input_block_stride;
- const uniform<int> remaining_input_columns =
+ const uniform<uint> remaining_input_columns =
input_vector_length - input_block_stride * full_input_blocks;
// Loop over input_vector in blocks of input_columns_per_threadgroup
- for (int input_block_index = 0; input_block_index < full_input_blocks;
+ for (uint input_block_index = 0; input_block_index < full_input_blocks;
++input_block_index) {
// Load vector coefficients (unchecked)
{
@@ -138,7 +138,7 @@ KERNEL(MatmulGemv)(
}
// Per thread work loop
- int matrix_row_offset = 0;
+ uint matrix_row_offset = 0;
METAL_PRAGMA_UNROLL
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
output_row_offset++) {
@@ -170,7 +170,7 @@ KERNEL(MatmulGemv)(
{
const device T* input_vector_row =
input_vector + batch_row * input_dimension;
- if (input_block_col_offset + int(thread_out_cols) <=
+ if (input_block_col_offset + uint(thread_out_cols) <=
input_vector_length) {
METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
@@ -184,7 +184,7 @@ KERNEL(MatmulGemv)(
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
input_col_offset++) {
vector_coefficients[input_col_offset] =
- input_block_col_offset + int(input_col_offset) <
+ input_block_col_offset + uint(input_col_offset) <
input_vector_length
? static_cast<float>(
input_vector_row
@@ -200,7 +200,7 @@ KERNEL(MatmulGemv)(
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
output_row_offset++) {
// Load matrix row (checked)
- if (input_block_col_offset + int(thread_out_cols) <=
+ if (input_block_col_offset + uint(thread_out_cols) <=
input_vector_length) {
METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
@@ -214,7 +214,7 @@ KERNEL(MatmulGemv)(
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
input_col_offset++) {
matrix_values[input_col_offset] =
- input_block_col_offset + int(input_col_offset) <
+ input_block_col_offset + uint(input_col_offset) <
input_vector_length
? thread_matrix
[output_row_offset * matrix_leading_dimension +
@@ -251,14 +251,14 @@ KERNEL(MatmulGemv)(
// Threadgroup reduction (only when tg_simd_cols > 1)
if (tg_simd_cols > 1) {
- const int computed_output_rows_per_tg =
- int(tg_simd_rows) * int(sg_thread_rows) * int(thread_out_rows);
+ const uint computed_output_rows_per_tg =
+ uint(tg_simd_rows) * uint(sg_thread_rows) * uint(thread_out_rows);
if (thread_col_in_simdgroup == 0) {
threadgroup float* threadgroup_partial_accumulations =
threadgroup_memory +
simdgroup_column_index *
- (computed_output_rows_per_tg + int(thread_out_rows)) +
+ (computed_output_rows_per_tg + uint(thread_out_rows)) +
output_block_row_offset;
METAL_PRAGMA_UNROLL
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
@@ -280,7 +280,7 @@ KERNEL(MatmulGemv)(
output_row_offset++) {
accumulated_values[output_row_offset] += base_partial
[reduction_simdgroup_col *
- (computed_output_rows_per_tg + int(thread_out_rows)) +
+ (computed_output_rows_per_tg + uint(thread_out_rows)) +
output_row_offset];
}
}
@@ -294,18 +294,16 @@ KERNEL(MatmulGemv)(
METAL_PRAGMA_UNROLL
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
output_row_offset++) {
- if (apply_output_scale_and_accumulate) {
- output_row_values[output_row_start + output_row_offset] =
- static_cast<T>(output_scale) *
- static_cast<T>(accumulated_values[output_row_offset]) +
- static_cast<T>(output_accumulate_scale) *
- output_source
- [(output_row_start + output_row_offset) *
- output_source_stride];
- } else {
- output_row_values[output_row_start + output_row_offset] =
- static_cast<T>(accumulated_values[output_row_offset]);
+ T accumulated_c = static_cast<T>(output_scale) *
+ static_cast<T>(accumulated_values[output_row_offset]);
+ if (is_accumulate) {
+ accumulated_c +=
+ output_row_values[output_row_start + output_row_offset];
+ }
+ if (is_bias) {
+ accumulated_c += output_bias[output_row_start + output_row_offset];
}
+ output_row_values[output_row_start + output_row_offset] = accumulated_c;
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/specialization.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
similarity index 53%
rename from crates/uzu/src/backends/metal/kernel/matmul/gemv/specialization.rs
rename to crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
index 87fca97f1..4e83e73cd 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/specialization.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
@@ -1,15 +1,16 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct Specialization {
+pub struct GemvSpecialization {
pub threadgroup_rows: u32,
pub threadgroup_cols: u32,
pub threads_per_simdgroup_row: u32,
pub threads_per_simdgroup_col: u32,
pub elements_per_thread_row: u32,
pub elements_per_thread_col: u32,
- pub apply_output_scale_and_accumulate: bool,
+ pub is_accumulate: bool,
+ pub is_bias: bool,
}
-impl Specialization {
+impl GemvSpecialization {
pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {
use crate::DataType;
match data_type {
@@ -21,7 +22,8 @@ impl Specialization {
threads_per_simdgroup_col: 32,
elements_per_thread_row: 4,
elements_per_thread_col: 4,
- apply_output_scale_and_accumulate: false,
+ is_accumulate: false,
+ is_bias: false,
},
Self {
threadgroup_rows: 4,
@@ -30,7 +32,8 @@ impl Specialization {
threads_per_simdgroup_col: 32,
elements_per_thread_row: 4,
elements_per_thread_col: 4,
- apply_output_scale_and_accumulate: true,
+ is_accumulate: false,
+ is_bias: true,
},
Self {
threadgroup_rows: 8,
@@ -39,7 +42,8 @@ impl Specialization {
threads_per_simdgroup_col: 32,
elements_per_thread_row: 4,
elements_per_thread_col: 4,
- apply_output_scale_and_accumulate: false,
+ is_accumulate: false,
+ is_bias: false,
},
Self {
threadgroup_rows: 8,
@@ -48,7 +52,8 @@ impl Specialization {
threads_per_simdgroup_col: 32,
elements_per_thread_row: 4,
elements_per_thread_col: 4,
- apply_output_scale_and_accumulate: true,
+ is_accumulate: false,
+ is_bias: true,
},
],
DataType::F16 => &[Self {
@@ -58,7 +63,8 @@ impl Specialization {
threads_per_simdgroup_col: 32,
elements_per_thread_row: 4,
elements_per_thread_col: 4,
- apply_output_scale_and_accumulate: false,
+ is_accumulate: false,
+ is_bias: false,
}],
DataType::F32 => &[Self {
threadgroup_rows: 8,
@@ -67,7 +73,8 @@ impl Specialization {
threads_per_simdgroup_col: 32,
elements_per_thread_row: 4,
elements_per_thread_col: 4,
- apply_output_scale_and_accumulate: false,
+ is_accumulate: false,
+ is_bias: false,
}],
_ => &[],
}
@@ -78,75 +85,45 @@ impl Specialization {
}
pub fn select(
- transpose_matrix: bool,
- input_dimension: i32,
- output_dimension: i32,
- apply_output_scale_and_accumulate: bool,
+ input_dimension: u32,
+ output_dimension: u32,
+ is_accumulate: bool,
+ is_bias: bool,
) -> Self {
let (threadgroup_rows, threadgroup_cols);
let (threads_per_simdgroup_row, threads_per_simdgroup_col);
let (elements_per_thread_row, elements_per_thread_col);
- if transpose_matrix {
- let mut simdgroup_thread_rows = 8;
- let mut simdgroup_thread_cols = 4;
- if input_dimension >= 8192 && output_dimension >= 2048 {
- simdgroup_thread_rows = 4;
- simdgroup_thread_cols = 8;
- }
+ let threadgroup_simd_rows;
+ let mut simdgroup_thread_rows = 1;
+ let mut simdgroup_thread_cols = 32;
+ let mut threadgroup_simd_cols = 1;
- let threadgroup_simd_cols = if output_dimension >= 2048 {
- 16
- } else if output_dimension >= 512 {
- 4
- } else {
- 2
- };
-
- let thread_output_cols = if output_dimension < 4 {
- 1
- } else {
- 4
- };
-
- threadgroup_rows = 1;
- threadgroup_cols = threadgroup_simd_cols;
- threads_per_simdgroup_row = simdgroup_thread_rows;
- threads_per_simdgroup_col = simdgroup_thread_cols;
- elements_per_thread_row = 4;
- elements_per_thread_col = thread_output_cols;
+ if input_dimension <= 64 {
+ threadgroup_simd_rows = 1;
+ simdgroup_thread_rows = 8;
+ simdgroup_thread_cols = 4;
+ } else if input_dimension >= 16 * output_dimension {
+ threadgroup_simd_rows = 1;
+ threadgroup_simd_cols = 8;
+ } else if output_dimension >= 4096 {
+ threadgroup_simd_rows = 8;
} else {
- let threadgroup_simd_rows;
- let mut simdgroup_thread_rows = 1;
- let mut simdgroup_thread_cols = 32;
- let mut threadgroup_simd_cols = 1;
-
- if input_dimension <= 64 {
- threadgroup_simd_rows = 1;
- simdgroup_thread_rows = 8;
- simdgroup_thread_cols = 4;
- } else if input_dimension >= 16 * output_dimension {
- threadgroup_simd_rows = 1;
- threadgroup_simd_cols = 8;
- } else if output_dimension >= 4096 {
- threadgroup_simd_rows = 8;
- } else {
- threadgroup_simd_rows = 4;
- }
+ threadgroup_simd_rows = 4;
+ }
- let thread_output_rows = if output_dimension < 4 {
- 1
- } else {
- 4
- };
+ let thread_output_rows = if output_dimension < 4 {
+ 1
+ } else {
+ 4
+ };
- threadgroup_rows = threadgroup_simd_rows;
- threadgroup_cols = threadgroup_simd_cols;
- threads_per_simdgroup_row = simdgroup_thread_rows;
- threads_per_simdgroup_col = simdgroup_thread_cols;
- elements_per_thread_row = thread_output_rows;
- elements_per_thread_col = 4;
- }
+ threadgroup_rows = threadgroup_simd_rows;
+ threadgroup_cols = threadgroup_simd_cols;
+ threads_per_simdgroup_row = simdgroup_thread_rows;
+ threads_per_simdgroup_col = simdgroup_thread_cols;
+ elements_per_thread_row = thread_output_rows;
+ elements_per_thread_col = 4;
Self {
threadgroup_rows,
@@ -155,7 +132,8 @@ impl Specialization {
threads_per_simdgroup_col,
elements_per_thread_row,
elements_per_thread_col,
- apply_output_scale_and_accumulate,
+ is_accumulate,
+ is_bias,
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
deleted file mode 100644
index 1d2748972..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-mod specialization;
-
-pub use specialization::Specialization;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs b/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs
deleted file mode 100644
index ae1d4d400..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs
+++ /dev/null
@@ -1,339 +0,0 @@
-use std::{
- collections::{HashMap, hash_map::Entry},
- sync::OnceLock,
-};
-
-use super::{
- super::dsl::{MatmulGemmMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel},
- gemm, gemv,
-};
-use crate::{
- DataType,
- backends::{
- common::{
- Encoder,
- gpu_types::GemmParams,
- kernel::{
- TensorAddBiasKernel,
- matmul::{MatmulArguments, MatmulError, MatmulKernel},
- },
- },
- metal::{Metal, context::MetalContext},
- },
-};
-
-pub struct MatmulMetalKernel {
- data_type: DataType,
- gemv_kernels: HashMap<gemv::Specialization, MatmulGemvMetalKernel>,
- gemm_kernels: HashMap<gemm::GemmSpecialization, MatmulGemmMetalKernel>,
- bias_add: Option<TensorAddBiasMetalKernel>,
-}
-
-const DEFAULT_GEMV_MAX_BATCH: i32 = 8;
-
-static GEMV_MAX_BATCH: OnceLock<i32> = OnceLock::new();
-
-fn max_gemv_batch_threshold() -> i32 {
- *GEMV_MAX_BATCH.get_or_init(|| {
- std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)
- })
-}
-
-impl MatmulMetalKernel {
- fn is_gemv_eligible(arguments: &MatmulArguments<Metal>) -> bool {
- let m = arguments.batch;
- let n = arguments.output_dim;
- let max_batch = max_gemv_batch_threshold();
-
- if !arguments.transpose_b {
- return false;
- }
-
- if n == 1 {
- m == 1
- } else {
- m <= max_batch
- }
- }
-
- fn get_or_create_gemv(
- &mut self,
- context: &MetalContext,
- specialization: gemv::Specialization,
- ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {
- match self.gemv_kernels.entry(specialization) {
- Entry::Occupied(entry) => Ok(entry.into_mut()),
- Entry::Vacant(entry) => {
- let kernel = MatmulGemvMetalKernel::new(
- context,
- self.data_type,
- specialization.threadgroup_rows,
- specialization.threadgroup_cols,
- specialization.threads_per_simdgroup_row,
- specialization.threads_per_simdgroup_col,
- specialization.elements_per_thread_row,
- specialization.elements_per_thread_col,
- specialization.apply_output_scale_and_accumulate,
- )
- .map_err(MatmulError::BackendError)?;
- Ok(entry.insert(kernel))
- },
- }
- }
-
- fn get_or_create_gemm(
- &mut self,
- context: &MetalContext,
- specialization: gemm::GemmSpecialization,
- ) -> Result<&MatmulGemmMetalKernel, MatmulError<Metal>> {
- match self.gemm_kernels.entry(specialization) {
- Entry::Occupied(entry) => Ok(entry.into_mut()),
- Entry::Vacant(entry) => {
- let kernel = MatmulGemmMetalKernel::new(
- context,
- self.data_type,
- specialization.block_rows as u32,
- specialization.block_cols as u32,
- specialization.block_depth as u32,
- specialization.simdgroups_per_row as u32,
- specialization.simdgroups_per_column as u32,
- specialization.align_m,
- specialization.align_n,
- specialization.align_k,
- )
- .map_err(MatmulError::BackendError)?;
- Ok(entry.insert(kernel))
- },
- }
- }
-
- fn encode_gemv(
- &mut self,
- context: &MetalContext,
- encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<Metal>,
- ) -> Result<(), MatmulError<Metal>> {
- let m = arguments.batch;
- let n = arguments.output_dim;
-
- let matrix_is_rhs = n != 1;
- let has_bias = arguments.bias.is_some();
- let apply_output_scale_and_accumulate = has_bias;
-
- let effective_output_dimension = if matrix_is_rhs {
- n
- } else {
- m
- };
-
- let transpose_matrix = if matrix_is_rhs {
- !arguments.transpose_b
- } else {
- false
- };
-
- let specialization = gemv::Specialization::select(
- transpose_matrix,
- arguments.input_dim,
- effective_output_dimension,
- apply_output_scale_and_accumulate,
- );
-
- let (alpha, beta, bias_stride) = if apply_output_scale_and_accumulate {
- (1.0f32, 1.0f32, 1i32)
- } else {
- (1.0f32, 0.0f32, 0i32)
- };
-
- let matrix_leading_dim = if matrix_is_rhs {
- arguments.leading_dimension_b
- } else {
- arguments.leading_dimension_a
- };
-
- let (matrix, matrix_offset) = if matrix_is_rhs {
- (arguments.b, 0usize)
- } else {
- (arguments.a, arguments.a_offset as usize)
- };
- let (input_vector, input_vector_offset) = if matrix_is_rhs {
- (arguments.a, arguments.a_offset as usize)
- } else {
- (arguments.b, 0usize)
- };
-
- let output_source = if apply_output_scale_and_accumulate {
- arguments.bias
- } else {
- None
- };
-
- let kernel = self.get_or_create_gemv(context, specialization)?;
-
- kernel.encode(
- (matrix, matrix_offset),
- (input_vector, input_vector_offset),
- output_source.map(|buffer| (buffer, 0usize)),
- &mut *arguments.d,
- arguments.input_dim,
- effective_output_dimension,
- matrix_leading_dim,
- alpha,
- beta,
- bias_stride,
- arguments.batch,
- specialization.output_rows_per_threadgroup() as i32,
- encoder,
- );
-
- let bias_is_fused = apply_output_scale_and_accumulate && has_bias;
- if !bias_is_fused {
- self.encode_bias_add(
- context,
- arguments.bias,
- arguments.d,
- arguments.batch as usize,
- arguments.output_dim as usize,
- encoder,
- )?;
- }
-
- Ok(())
- }
-
- fn encode_gemm(
- &mut self,
- context: &MetalContext,
- encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<Metal>,
- ) -> Result<(), MatmulError<Metal>> {
- let m = arguments.batch;
- let n = arguments.output_dim;
- let k = arguments.input_dim;
-
- let specialization = gemm::GemmSpecialization::select(context, self.data_type, &arguments);
-
- let threadgroups_per_row = (n + specialization.block_cols - 1) / specialization.block_cols;
- let threadgroups_per_column = (m + specialization.block_rows - 1) / specialization.block_rows;
- let swizzle_log = specialization.swizzle_log2;
-
- let swizzle_stride = 1_i32 << swizzle_log;
- let tm_swizzled = (threadgroups_per_column + swizzle_stride - 1) / swizzle_stride;
- let tn_swizzled = threadgroups_per_row * swizzle_stride;
-
- let params = GemmParams {
- M: m,
- N: n,
- K: k,
- leading_dimension_a: arguments.leading_dimension_a,
- leading_dimension_b: arguments.leading_dimension_b,
- leading_dimension_d: arguments.leading_dimension_d,
- threadgroups_per_row,
- threadgroups_per_column,
- swizzle_log,
- aligned_inner_iterations: k / specialization.block_depth,
- };
-
- let group_count_x =
- u32::try_from(tn_swizzled).map_err(|_| MatmulError::<Metal>::ThreadgroupOverflow(tn_swizzled as usize))?;
- let group_count_y =
- u32::try_from(tm_swizzled).map_err(|_| MatmulError::<Metal>::ThreadgroupOverflow(tm_swizzled as usize))?;
-
- let kernel = self.get_or_create_gemm(context, specialization)?;
-
- kernel.encode(
- (arguments.a, arguments.a_offset as usize),
- arguments.b,
- &mut *arguments.d,
- std::slice::from_ref(&params),
- group_count_x,
- group_count_y,
- encoder,
- );
-
- self.encode_bias_add(
- context,
- arguments.bias,
- arguments.d,
- arguments.batch as usize,
- arguments.output_dim as usize,
- encoder,
- )?;
-
- Ok(())
- }
-
- fn encode_bias_add(
- &mut self,
- context: &MetalContext,
- bias: Option<&<Metal as crate::backends::common::Backend>::Buffer>,
- output: &mut <Metal as crate::backends::common::Backend>::Buffer,
- batch: usize,
- output_dim: usize,
- encoder: &mut Encoder<Metal>,
- ) -> Result<(), MatmulError<Metal>> {
- let Some(bias) = bias else {
- return Ok(());
- };
- let total_length = batch * output_dim;
- if total_length == 0 {
- return Ok(());
- }
- if self.bias_add.is_none() {
- self.bias_add =
- Some(TensorAddBiasMetalKernel::new(context, self.data_type, true).map_err(MatmulError::BackendError)?);
- }
- let bias_add = self.bias_add.as_ref().unwrap();
- bias_add.encode(
- None::<&<Metal as crate::backends::common::Backend>::Buffer>,
- bias,
- output,
- output_dim as u32,
- total_length as u32,
- encoder,
- );
- Ok(())
- }
-}
-
-impl MatmulKernel for MatmulMetalKernel {
- type Backend = Metal;
-
- fn new(
- context: &MetalContext,
- data_type: DataType,
- ) -> Result<Self, MatmulError<Metal>> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::UnsupportedDataType(data_type));
- }
-
- let mut kernel = Self {
- data_type,
- gemv_kernels: HashMap::new(),
- gemm_kernels: HashMap::new(),
- bias_add: None,
- };
-
- for &config in gemv::Specialization::precompile_configs(data_type) {
- kernel.get_or_create_gemv(context, config)?;
- }
- for &config in gemm::GemmSpecialization::precompile_configs(data_type) {
- kernel.get_or_create_gemm(context, config)?;
- }
-
- Ok(kernel)
- }
-
- fn encode(
- &mut self,
- context: &MetalContext,
- arguments: MatmulArguments<Metal>,
- encoder: &mut Encoder<Metal>,
- ) {
- if Self::is_gemv_eligible(&arguments) {
- self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV kernel");
- } else {
- self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM kernel");
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index 27c0a5c14..9ebb64163 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -1,5 +1,228 @@
-pub mod gemm;
-pub mod gemv;
-mod matmul;
+use std::{
+ collections::{HashMap, hash_map::Entry},
+ sync::OnceLock,
+};
-pub use matmul::MatmulMetalKernel;
+use crate::{
+ DataType,
+ backends::{
+ common::{
+ Encoder,
+ gpu_types::GemmParams,
+ kernel::{
+ TensorAddBiasKernel,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
+ },
+ },
+ metal::{
+ Metal,
+ context::MetalContext,
+ kernel::{MatmulGemmMetalKernel, MatmulGemvMetalKernel, TensorAddBiasMetalKernel},
+ },
+ },
+};
+
+mod gemm;
+mod gemv;
+
+pub struct MatmulMetalKernel {
+ data_type: DataType,
+ gemv_kernels: HashMap<gemv::GemvSpecialization, MatmulGemvMetalKernel>,
+ gemm_kernels: HashMap<gemm::GemmSpecialization, MatmulGemmMetalKernel>,
+ bias_add: TensorAddBiasMetalKernel,
+}
+
+const DEFAULT_GEMV_MAX_BATCH: u32 = 8;
+
+static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();
+
+fn max_gemv_batch_threshold() -> u32 {
+ *GEMV_MAX_BATCH.get_or_init(|| {
+ std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)
+ })
+}
+
+impl MatmulMetalKernel {
+ fn get_or_create_gemv(
+ &mut self,
+ context: &MetalContext,
+ specialization: gemv::GemvSpecialization,
+ ) -> Result<&MatmulGemvMetalKernel, MatmulError<Metal>> {
+ match self.gemv_kernels.entry(specialization) {
+ Entry::Occupied(entry) => Ok(entry.into_mut()),
+ Entry::Vacant(entry) => {
+ let kernel = MatmulGemvMetalKernel::new(
+ context,
+ self.data_type,
+ specialization.threadgroup_rows,
+ specialization.threadgroup_cols,
+ specialization.threads_per_simdgroup_row,
+ specialization.threads_per_simdgroup_col,
+ specialization.elements_per_thread_row,
+ specialization.elements_per_thread_col,
+ specialization.is_accumulate,
+ specialization.is_bias,
+ )
+ .map_err(MatmulError::BackendError)?;
+ Ok(entry.insert(kernel))
+ },
+ }
+ }
+
+ fn get_or_create_gemm(
+ &mut self,
+ context: &MetalContext,
+ specialization: gemm::GemmSpecialization,
+ ) -> Result<&MatmulGemmMetalKernel, MatmulError<Metal>> {
+ match self.gemm_kernels.entry(specialization) {
+ Entry::Occupied(entry) => Ok(entry.into_mut()),
+ Entry::Vacant(entry) => {
+ let kernel = MatmulGemmMetalKernel::new(
+ context,
+ self.data_type,
+ specialization.block_rows as u32,
+ specialization.block_cols as u32,
+ specialization.block_depth as u32,
+ specialization.simdgroups_per_row as u32,
+ specialization.simdgroups_per_column as u32,
+ specialization.align_mn,
+ specialization.align_k,
+ specialization.is_accumulate,
+ )
+ .map_err(MatmulError::BackendError)?;
+ Ok(entry.insert(kernel))
+ },
+ }
+ }
+
+ fn encode_gemv(
+ &mut self,
+ context: &MetalContext,
+ encoder: &mut Encoder<Metal>,
+ arguments: MatmulArguments<Metal>,
+ ) -> Result<(), MatmulError<Metal>> {
+ let (is_accumulate, output_bias) = match arguments.c {
+ MatmulArgumentC::Accumulate => (true, None),
+ MatmulArgumentC::Bias(bias) => (false, Some(bias)),
+ MatmulArgumentC::None => (false, None),
+ };
+
+ let specialization = gemv::GemvSpecialization::select(
+ arguments.input_dim,
+ arguments.output_dim,
+ is_accumulate,
+ output_bias.is_some(),
+ );
+
+ self.get_or_create_gemv(context, specialization)?.encode(
+ arguments.b,
+ (arguments.a, arguments.a_offset as usize),
+ output_bias,
+ arguments.d,
+ arguments.input_dim,
+ arguments.output_dim,
+ arguments.input_dim,
+ arguments.ab_scale,
+ arguments.batch_dim,
+ specialization.output_rows_per_threadgroup(),
+ encoder,
+ );
+
+ Ok(())
+ }
+
+ fn encode_gemm(
+ &mut self,
+ context: &MetalContext,
+ encoder: &mut Encoder<Metal>,
+ arguments: MatmulArguments<Metal>,
+ ) -> Result<(), MatmulError<Metal>> {
+ let specialization = gemm::GemmSpecialization::select(context, self.data_type, &arguments);
+
+ let threadgroups_per_row = arguments.output_dim.div_ceil(specialization.block_cols);
+ let threadgroups_per_column = arguments.batch_dim.div_ceil(specialization.block_rows);
+
+ let params = GemmParams {
+ M: arguments.batch_dim,
+ N: arguments.output_dim,
+ K: arguments.input_dim,
+ leading_dimension_a: arguments.input_dim,
+ leading_dimension_b: arguments.input_dim,
+ leading_dimension_d: arguments.output_dim,
+ threadgroups_per_row,
+ threadgroups_per_column,
+ swizzle_log: 0,
+ aligned_inner_iterations: arguments.input_dim / specialization.block_depth,
+ };
+
+ let kernel = self.get_or_create_gemm(context, specialization)?;
+
+ kernel.encode(
+ (arguments.a, arguments.a_offset as usize),
+ arguments.b,
+ &mut *arguments.d,
+ std::slice::from_ref(&params),
+ threadgroups_per_row,
+ threadgroups_per_column,
+ arguments.ab_scale,
+ encoder,
+ );
+
+ if let MatmulArgumentC::Bias(bias) = arguments.c {
+ self.bias_add.encode(
+ None::<&<Metal as crate::backends::common::Backend>::Buffer>,
+ bias,
+ arguments.d,
+ arguments.output_dim,
+ arguments.batch_dim * arguments.output_dim,
+ encoder,
+ );
+ }
+
+ Ok(())
+ }
+}
+
+impl MatmulKernel for MatmulMetalKernel {
+ type Backend = Metal;
+
+ fn new(
+ context: &MetalContext,
+ data_type: DataType,
+ ) -> Result<Self, MatmulError<Metal>> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MatmulError::UnsupportedDataType(data_type));
+ }
+
+ let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true).map_err(MatmulError::BackendError)?;
+
+ let mut kernel = Self {
+ data_type,
+ gemv_kernels: HashMap::new(),
+ gemm_kernels: HashMap::new(),
+ bias_add,
+ };
+
+ for &config in gemv::GemvSpecialization::precompile_configs(data_type) {
+ kernel.get_or_create_gemv(context, config)?;
+ }
+ for &config in gemm::GemmSpecialization::precompile_configs(data_type) {
+ kernel.get_or_create_gemm(context, config)?;
+ }
+
+ Ok(kernel)
+ }
+
+ fn encode(
+ &mut self,
+ context: &MetalContext,
+ arguments: MatmulArguments<Metal>,
+ encoder: &mut Encoder<Metal>,
+ ) {
+ if arguments.batch_dim <= max_gemv_batch_threshold() {
+ self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV kernel");
+ } else {
+ self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM kernel");
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index dedaa9318..916078500 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -1,12 +1,11 @@
pub mod matmul;
-use crate::backends::common::kernel::matmul::MatmulKernels;
+use crate::backends::common::kernel::ManualKernels;
+
+include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
-pub mod dsl {
- include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
-}
pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
-impl MatmulKernels for dsl::MetalKernels {
+impl ManualKernels for MetalKernels {
type MatmulKernel = matmul::MatmulMetalKernel;
}
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
index 4c21fc68c..f2ab1afda 100644
--- a/crates/uzu/src/encodable_block/embedding.rs
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -10,8 +10,8 @@ use crate::{
backends::common::{
Backend, Encoder, Kernels,
kernel::{
- FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,
- matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},
+ FullPrecisionEmbeddingLookupKernel, ManualKernels, QuantizedEmbeddingLookupKernel,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
quant_matmul::{
QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,
QuantizedMatmulKernelEncodable, QuantizedMatmulType,
@@ -48,7 +48,7 @@ enum TiedEmbeddingType<B: Backend> {
FullPrecision {
weights: B::Buffer,
lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,
- readout: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,
+ readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
},
Quantized {
weights: B::Buffer,
@@ -75,7 +75,7 @@ enum UntiedEmbeddingLookupType<B: Backend> {
enum UntiedEmbeddingReadoutType<B: Backend> {
FullPrecision {
weights: B::Buffer,
- readout: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,
+ readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
},
Quantized {
weights: B::Buffer,
@@ -213,7 +213,7 @@ impl<B: Backend> Embedding<B> {
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
.map_err(EmbeddingError::BackendError)?;
- let readout = RefCell::new(<B::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)?);
+ let readout = RefCell::new(<B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)?);
EmbeddingTying::Tied {
ty: TiedEmbeddingType::FullPrecision {
@@ -239,7 +239,7 @@ impl<B: Backend> Embedding<B> {
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
.map_err(EmbeddingError::BackendError)?;
- let readout = RefCell::new(<B::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)?);
+ let readout = RefCell::new(<B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)?);
EmbeddingTying::Untied {
input_ty: UntiedEmbeddingLookupType::FullPrecision {
@@ -615,15 +615,12 @@ impl<B: Backend> Embedding<B> {
a: input,
a_offset: input_offset as u64,
b: weights,
+ ab_scale: 1.0,
+ c: MatmulArgumentC::None,
d: output,
- bias: None,
- batch: batch_size as i32,
- input_dim: input_dim as i32,
- output_dim: output_dim as i32,
- leading_dimension_a: input_dim as i32,
- leading_dimension_b: input_dim as i32,
- leading_dimension_d: output_dim as i32,
- transpose_b: true,
+ batch_dim: batch_size as u32,
+ input_dim: input_dim as u32,
+ output_dim: output_dim as u32,
},
encoder,
);
diff --git a/crates/uzu/src/encodable_block/linear/full_precision.rs b/crates/uzu/src/encodable_block/linear/full_precision.rs
index 64000ebba..9111d4a14 100644
--- a/crates/uzu/src/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs
@@ -11,7 +11,10 @@ use crate::{
DataType,
backends::common::{
Backend, Encoder,
- kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
+ },
},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
@@ -49,7 +52,7 @@ pub enum FullPrecisionLinearError<B: Backend> {
}
pub struct FullPrecisionLinear<B: Backend> {
- kernel: RefCell<<B::Kernels as MatmulKernels>::MatmulKernel>,
+ kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
bias_buffer: Option<Rc<RefCell<B::Buffer>>>,
weights_buffer: Rc<RefCell<B::Buffer>>,
input_dim: usize,
@@ -111,7 +114,7 @@ impl<B: Backend> FullPrecisionLinear<B> {
Err(_) => None,
};
- let kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(context, precision)?;
+ let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, precision)?;
Ok(Self {
kernel: RefCell::new(kernel),
@@ -142,15 +145,15 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
a: input_array.buffer().borrow().deref(),
a_offset: 0,
b: self.weights_buffer.borrow().deref(),
+ ab_scale: 1.0,
+ c: match bias_borrow.as_deref() {
+ Some(b) => MatmulArgumentC::Bias(b),
+ None => MatmulArgumentC::None,
+ },
d: output_array.buffer().borrow_mut().deref_mut(),
- bias: bias_borrow.as_deref(),
- batch: batch_size as i32,
- input_dim: self.input_dim as i32,
- output_dim: self.output_dim as i32,
- leading_dimension_a: self.input_dim as i32,
- leading_dimension_b: self.input_dim as i32,
- leading_dimension_d: self.output_dim as i32,
- transpose_b: true,
+ batch_dim: batch_size as u32,
+ input_dim: self.input_dim as u32,
+ output_dim: self.output_dim as u32,
},
encoder,
);
diff --git a/crates/uzu/src/session/tts_session.rs b/crates/uzu/src/session/tts_session.rs
index 040cefdd3..fd6c5518e 100644
--- a/crates/uzu/src/session/tts_session.rs
+++ b/crates/uzu/src/session/tts_session.rs
@@ -33,10 +33,10 @@ use crate::{
backends::common::{
Backend, Context as BackendContext, Encoder, Kernels,
kernel::{
- EmbeddingRowsSumKernel, TensorAddScaleKernel, TensorCopyKernel, TokenCopySampledKernel,
+ EmbeddingRowsSumKernel, ManualKernels, TensorAddScaleKernel, TensorCopyKernel, TokenCopySampledKernel,
TokenCopyToResultsKernel,
kv_cache_update::KVCacheUpdate,
- matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
},
},
config::{InnerModelConfig, ModelMetadata, TtsMessageProcessorConfig},
diff --git a/crates/uzu/src/session/tts_session/fishaudio.rs b/crates/uzu/src/session/tts_session/fishaudio.rs
index 9ad5efdb2..681c0b15a 100644
--- a/crates/uzu/src/session/tts_session/fishaudio.rs
+++ b/crates/uzu/src/session/tts_session/fishaudio.rs
@@ -34,7 +34,7 @@ pub(super) struct FishAudioTextDecoderRuntime<B: Backend> {
struct FishAudioSemanticBridge<B: Backend> {
embedding_rows_sum: <B::Kernels as Kernels>::EmbeddingRowsSumKernel,
- projection: <B::Kernels as MatmulKernels>::MatmulKernel,
+ projection: <B::Kernels as ManualKernels>::MatmulKernel,
tensor_copy: <B::Kernels as Kernels>::TensorCopyKernel,
codebook_embeddings: Array<B>,
codebook_token_indices: Array<B>,
@@ -104,7 +104,7 @@ impl<B: Backend> FishAudioSemanticBridge<B> {
let embedding_rows_sum = <B::Kernels as Kernels>::EmbeddingRowsSumKernel::new(context, data_type)
.map_err(unable_to_create_context)?;
- let projection = <<B::Kernels as MatmulKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)
+ let projection = <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)
.map_err(unable_to_create_context)?;
let tensor_copy =
<B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type).map_err(unable_to_create_context)?;
@@ -540,15 +540,12 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
a: &hidden_buffer,
a_offset: hidden.offset() as u64,
b: &weights_buffer,
+ ab_scale: 1.0,
+ c: MatmulArgumentC::None,
d: &mut output_buffer,
- bias: None,
- batch: 1,
- input_dim: slow_model_dim as i32,
- output_dim: fast_model_dim as i32,
- leading_dimension_a: slow_model_dim as i32,
- leading_dimension_b: fast_model_dim as i32,
- leading_dimension_d: fast_model_dim as i32,
- transpose_b: false,
+ batch_dim: 1,
+ input_dim: slow_model_dim as u32,
+ output_dim: fast_model_dim as u32,
},
encoder,
);
diff --git a/crates/uzu/tests/performance/matmul/bench.rs b/crates/uzu/tests/performance/matmul/bench.rs
index 9236b0a68..ec8481f44 100644
--- a/crates/uzu/tests/performance/matmul/bench.rs
+++ b/crates/uzu/tests/performance/matmul/bench.rs
@@ -7,7 +7,10 @@ use uzu::{
backends::{
common::{
Backend, Encoder,
- kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ },
},
metal::Metal,
},
@@ -34,7 +37,7 @@ fn fill_buffer_random(
fn encode_and_run(
context: &Ctx,
- kernel: &mut <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel,
+ kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,
shape: &TestShape,
a_buffer: &Buf,
b_buffer: &Buf,
@@ -48,15 +51,12 @@ fn encode_and_run(
a: a_buffer,
a_offset: 0,
b: b_buffer,
+ ab_scale: 1.0,
+ c: MatmulArgumentC::None,
d: d_buffer,
- bias: None,
- batch: shape.batch as i32,
- input_dim: shape.input_dim as i32,
- output_dim: shape.output_dim as i32,
- leading_dimension_a: shape.input_dim as i32,
- leading_dimension_b: shape.input_dim as i32,
- leading_dimension_d: shape.output_dim as i32,
- transpose_b: true,
+ batch_dim: shape.batch as u32,
+ input_dim: shape.input_dim as u32,
+ output_dim: shape.output_dim as u32,
},
&mut encoder,
);
@@ -71,7 +71,7 @@ fn run_benchmark(
data_type: DataType,
shape: &TestShape,
) -> Result<f64, BenchError> {
- let mut kernel = <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel::new(context, data_type)
+ let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
.map_err(|e| BenchError::Kernel(e.to_string()))?;
let elem_size = data_type.size_in_bytes();
diff --git a/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs
index 4fdfc6002..67604f30b 100644
--- a/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs
+++ b/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs
@@ -10,7 +10,10 @@ use uzu::{
backends::{
common::{
Backend, Context, Encoder,
- kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ },
},
cpu::Cpu,
},
@@ -49,9 +52,9 @@ fn get_test_data<T: ArrayElement + Float>(
fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let m = input.m as i32;
- let k = input.k as i32;
- let n = input.n as i32;
+ let m = input.m as u32;
+ let k = input.k as u32;
+ let n = input.n as u32;
let a_array = context.create_array_from(&[input.m, input.k], &input.a, "");
let b_array = context.create_array_from(&[input.n, input.k], &input.b, "");
@@ -64,7 +67,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let d_buf = d_array.buffer();
let mut d_ref = d_buf.borrow_mut();
- let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type())
+ let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
.expect("Failed to create MatmulKernel");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -74,15 +77,12 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
a: a_ref.deref(),
a_offset: 0,
b: b_ref.deref(),
+ ab_scale: 1.0,
+ c: MatmulArgumentC::None,
d: d_ref.deref_mut(),
- bias: None,
- batch: m,
+ batch_dim: m,
input_dim: k,
output_dim: n,
- leading_dimension_a: k,
- leading_dimension_b: k,
- leading_dimension_d: n,
- transpose_b: true,
},
&mut encoder,
);
diff --git a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs
index a0772b176..842642aa7 100644
--- a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs
+++ b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs
@@ -11,7 +11,10 @@ use uzu::{
backends::{
common::{
Backend, Context, Encoder,
- kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ },
},
metal::Metal,
},
@@ -24,7 +27,6 @@ fn run_metal_matmul(
m: usize,
k: usize,
n: usize,
- transpose_b: bool,
) -> Vec<bf16> {
let a_buf = ctx
.device
@@ -39,14 +41,8 @@ fn run_metal_matmul(
.new_buffer(m * n * core::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let leading_dim_b = if transpose_b {
- k
- } else {
- n
- };
-
let mut kernel =
- <<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel::new(ctx, DataType::BF16).expect("kernel");
+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(ctx, DataType::BF16).expect("kernel");
let mut encoder = Encoder::new(ctx).unwrap();
kernel.encode(
@@ -55,15 +51,12 @@ fn run_metal_matmul(
a: &a_buf,
a_offset: 0,
b: &b_buf,
+ ab_scale: 1.0,
+ c: MatmulArgumentC::None,
d: &mut d_buf,
- bias: None,
- batch: m as i32,
- input_dim: k as i32,
- output_dim: n as i32,
- leading_dimension_a: k as i32,
- leading_dimension_b: leading_dim_b as i32,
- leading_dimension_d: n as i32,
- transpose_b,
+ batch_dim: m as u32,
+ input_dim: k as u32,
+ output_dim: n as u32,
},
&mut encoder,
);
@@ -81,20 +74,14 @@ fn run_ndarray_matmul(
m: usize,
k: usize,
n: usize,
- transpose_b: bool,
) -> Vec<bf16> {
let a_f32: Vec<f32> = a_data.iter().map(|x| x.to_f32()).collect();
let b_f32: Vec<f32> = b_data.iter().map(|x| x.to_f32()).collect();
let a_arr = Array2::from_shape_vec((m, k), a_f32).expect("A shape");
+ let b_arr = Array2::from_shape_vec((n, k), b_f32).expect("B shape");
- let result = if transpose_b {
- let b_arr = Array2::from_shape_vec((n, k), b_f32).expect("B shape");
- a_arr.dot(&b_arr.t())
- } else {
- let b_arr = Array2::from_shape_vec((k, n), b_f32).expect("B shape");
- a_arr.dot(&b_arr)
- };
+ let result = a_arr.dot(&b_arr.t());
result.iter().map(|&x| bf16::from_f32(x)).collect()
}
@@ -103,14 +90,9 @@ fn generate_test_data(
m: usize,
k: usize,
n: usize,
- transpose_b: bool,
) -> (Vec<bf16>, Vec<bf16>) {
let a: Vec<bf16> = (0..(m * k)).map(|i| bf16::from_f32(((i % 13) as f32) * 0.01 - 0.06)).collect();
- let b_size = if transpose_b {
- n * k
- } else {
- k * n
- };
+ let b_size = n * k;
let b: Vec<bf16> = (0..b_size).map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.15)).collect();
(a, b)
}
@@ -119,7 +101,6 @@ struct TestCase {
m: usize,
k: usize,
n: usize,
- transpose_b: bool,
tolerance: f32,
}
@@ -213,24 +194,23 @@ fn test_cases(full_case_matrix: bool) -> Vec<TestCase> {
(&QUICK_BATCH_SIZES, &QUICK_MODEL_SHAPES)
};
- let mut cases: Vec<(usize, usize, usize, bool)> =
- model_shapes.iter().flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n, true))).collect();
+ let mut cases: Vec<(usize, usize, usize)> =
+ model_shapes.iter().flat_map(|&(k, n)| batch_sizes.iter().map(move |&m| (m, k, n))).collect();
// Edge case: small matrix
for &m in batch_sizes {
- cases.push((m, 128, 128, true));
+ cases.push((m, 128, 128));
}
cases
.into_iter()
- .map(|(m, k, n, transpose_b)| {
+ .map(|(m, k, n)| {
let tolerance = base_tolerance * (k as f32 / 1024.0).sqrt();
let tolerance = tolerance * (1.0 + (m as f32).log2() * 0.02);
TestCase {
m,
k,
n,
- transpose_b,
tolerance,
}
})
@@ -293,15 +273,11 @@ fn matmul_correctness_comprehensive() {
let mut failed = Vec::new();
for case in &cases {
- let (a, b) = generate_test_data(case.m, case.k, case.n, case.transpose_b);
- let metal_result = run_metal_matmul(&ctx, &a, &b, case.m, case.k, case.n, case.transpose_b);
- let reference = run_ndarray_matmul(&a, &b, case.m, case.k, case.n, case.transpose_b);
+ let (a, b) = generate_test_data(case.m, case.k, case.n);
+ let metal_result = run_metal_matmul(&ctx, &a, &b, case.m, case.k, case.n);
+ let reference = run_ndarray_matmul(&a, &b, case.m, case.k, case.n);
- let trans_str = if case.transpose_b {
- "T"
- } else {
- "N"
- };
+ let trans_str = "T";
match compare_results(&metal_result, &reference, case.tolerance) {
Ok(()) => {
@@ -323,11 +299,7 @@ fn matmul_correctness_comprehensive() {
if !failed.is_empty() {
eprintln!("\nFailed tests:");
for (case, max_diff, idx, count) in &failed {
- let trans_str = if case.transpose_b {
- "T"
- } else {
- "N"
- };
+ let trans_str = "T";
eprintln!(
" m={} k={} n={} B={}: max_diff={:.6} at idx {}, {} exceed tol {}",
case.m, case.k, case.n, trans_str, max_diff, idx, count, case.tolerance
diff --git a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs
index e2a476fdf..7b1966953 100644
--- a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs
+++ b/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs
@@ -10,7 +10,10 @@ use uzu::{
backends::{
common::{
Backend, Context, Encoder,
- kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
+ kernel::{
+ ManualKernels,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ },
},
cpu::Cpu,
},
@@ -49,9 +52,9 @@ fn get_test_data<T: ArrayElement + Float>(
fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let m = input.m as i32;
- let k = input.k as i32;
- let n = input.n as i32;
+ let m = input.m as u32;
+ let k = input.k as u32;
+ let n = input.n as u32;
let a_array = context.create_array_from(&[input.m, input.k], &input.a, "");
let b_array = context.create_array_from(&[input.n, input.k], &input.b, "");
@@ -64,7 +67,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let d_buf = d_array.buffer();
let mut d_ref = d_buf.borrow_mut();
- let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type())
+ let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
.expect("Failed to create MatmulKernel");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -74,15 +77,12 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
a: a_ref.deref(),
a_offset: 0,
b: b_ref.deref(),
+ ab_scale: 1.0,
+ c: MatmulArgumentC::None,
d: d_ref.deref_mut(),
- bias: None,
- batch: m,
+ batch_dim: m,
input_dim: k,
output_dim: n,
- leading_dimension_a: k,
- leading_dimension_b: k,
- leading_dimension_d: n,
- transpose_b: true,
},
&mut encoder,
);
diff --git a/deny.toml b/deny.toml
index dab26902d..7a48cd900 100644
--- a/deny.toml
+++ b/deny.toml
@@ -15,6 +15,7 @@ allow = [
"ISC",
"Zlib",
"BSL-1.0",
+ "CC0-1.0",
"CDLA-Permissive-2.0",
"Unicode-3.0"
]