A high-performance inference engine for AI models
optimizations
eugened7e5dbf542c6
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rsindex 4e3220a24..f56caf374 100644--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs@@ -7,7 +7,8 @@ use crate::{gpu_types::QuantizationMode,kernel::{QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,- QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, QuantizedMatmulQvmKernel,+ QuantizedMatmulQmmTransposedWideKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel,+ QuantizedMatmulQvmKernel,},},};@@ -81,6 +82,7 @@ enum EncodableVariant<B: Backend> {Qmm(<B::Kernels as Kernels>::QuantizedMatmulQmmKernel),QmmTransposed(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel),QmmTransposed64x64(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel),+ QmmTransposedWide(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel),}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -103,6 +105,7 @@ enum MatrixMatrixFamily {QmmTransposedAlignedN,QmmTransposedUnalignedN,QmmTransposed64x64,+ QmmTransposedWide,}#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -268,6 +271,20 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {encoder,);},+ EncodableVariant::QmmTransposedWide(kernel) => {+ kernel.encode(+ arguments.b_buffer,+ arguments.scales_buffer,+ zero_points,+ biases,+ a_with_offset,+ arguments.output_buffer,+ k,+ n,+ m,+ encoder,+ );+ },}Ok(())@@ -408,6 +425,17 @@ fn create_matrix_matrix_kernel<B: Backend>().map_err(QuantizedMatmulError::BackendError)?,),+ MatrixMatrixFamily::QmmTransposedWide => EncodableVariant::QmmTransposedWide(+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(+ context,+ data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )+ .map_err(QuantizedMatmulError::BackendError)?,+ ),};Ok(kernel)@@ -446,12 +474,16 @@ fn select_matrix_matrix_family() -> MatrixMatrixFamily {if configuration.weights_transposed {let aligned_n = configuration.output_dim % 32 == 0;- let use_64x64 = aligned_n+ let aligned_n_64 = configuration.output_dim % 64 == 0;+ let use_64x64 = aligned_n_64&& configuration.data_type == DataType::BF16&& matches!(configuration.group_size, 64 | 128)&& matches!(bits, 4 | 8);+ let use_wide = aligned_n_64 && configuration.data_type == DataType::BF16 && matches!(bits, 4 | 8) && !use_64x64;if use_64x64 {MatrixMatrixFamily::QmmTransposed64x64+ } else if use_wide {+ MatrixMatrixFamily::QmmTransposedWide} else if aligned_n {MatrixMatrixFamily::QmmTransposedAlignedN} else {@@ -508,6 +540,7 @@ fn kernel_key_name(key: KernelKey) -> &'static str {"matrix_matrix_qmm_transposed_unaligned_n"},KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposed64x64) => "matrix_matrix_qmm_transposed_64x64",+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedWide) => "matrix_matrix_qmm_transposed_wide",}}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rsindex dadd0367b..330ce71e9 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs@@ -1,6 +1,7 @@pub mod qmm;pub mod qmm_transposed;pub mod qmm_transposed_64x64;+pub mod qmm_transposed_wide;pub mod qmv;pub mod qmv_fast;pub mod qvm;diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rsnew file mode 100644index 000000000..f9cabc631--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs@@ -0,0 +1,33 @@+use dsl::kernel;+use half::bf16;+use num_traits::Float;++use crate::ArrayElement;++#[kernel(QuantizedMatmulQmmTransposedWide)]+#[variants(T, bf16)]+#[variants(GROUP_SIZE, 32, 64, 128)]+#[variants(BITS, 4, 8)]+pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(+ #[allow(unused)] w: *const u32,+ #[allow(unused)] scales: *const T,+ #[allow(unused)]+ #[optional(use_zero_points)]+ zero_points: Option<*const u8>,+ #[allow(unused)]+ #[optional(use_mlx_quant)]+ biases: Option<*const T>,+ #[allow(unused)] x: *const T,+ #[allow(unused)] y: *mut T,+ #[allow(unused)] k: i32,+ #[allow(unused)] n: i32,+ #[allow(unused)] m: i32,+ #[allow(unused)]+ #[specialize]+ use_zero_points: bool,+ #[allow(unused)]+ #[specialize]+ use_mlx_quant: bool,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metalnew file mode 100644index 000000000..6118a70fe--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal@@ -0,0 +1,76 @@+#include <metal_stdlib>+#include "../common/dsl.h"+#include "quant_matmul.metal"++// Wide QMM tile: BM=64, BK=32, BN=64.+// Processes 4x more output elements per threadgroup than the 32x32 variant.+// BK=32 satisfies the group_size >= BK constraint for all group sizes (32, 64,+// 128).+template <typename T, int GROUP_SIZE, int BITS>+VARIANTS(T, bfloat)+VARIANTS(GROUP_SIZE, 32, 64, 128)+VARIANTS(BITS, 4, 8)+PUBLIC KERNEL(QuantizedMatmulQmmTransposedWide)(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points OPTIONAL(use_zero_points),+ const device T* biases OPTIONAL(use_mlx_quant),+ const device T* x,+ device T* y,+ const constant int& k,+ const constant int& n,+ const constant int& m,+ threadgroup T Xs[64 * (32 + 16 / sizeof(T))],+ threadgroup T Ws[64 * (32 + 16 / sizeof(T))],+ const bool use_zero_points SPECIALIZE,+ const bool use_mlx_quant SPECIALIZE,+ const uint tgid_x GROUPS((n + 64 - 1) / 64),+ const uint tgid_y GROUPS((m + 64 - 1) / 64),+ const uint tgid_z GROUPS(1),+ const uint tid_x THREADS(32),+ const uint tid_y THREADS(2),+ const uint tid_z THREADS(2)+) {+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);+ const uint lid = tid_z * 64 + tid_y * 32 + tid_x;+ const uint simd_gid = tid_z * 2 + tid_y;+ const uint simd_lid = tid_x;++ if (use_mlx_quant) {+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, true>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ } else {+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, false>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metalindex 4b3120dee..c8d0a8b35 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal@@ -19,10 +19,10 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(const bool use_zero_points SPECIALIZE,const bool use_mlx_quant SPECIALIZE,const uint tgid_x GROUPS(m),- const uint tgid_y GROUPS((n + 8 - 1) / 8),+ const uint tgid_y GROUPS((n + 32 - 1) / 32),const uint tgid_z GROUPS(1),const uint tid_x THREADS(32),- const uint tid_y THREADS(2)+ const uint tid_y THREADS(8)) {const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);const uint simd_gid = tid_y;diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalindex 59657ecff..6da2bfedf 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -1372,7 +1372,7 @@ void qmv_fast_impl(uint simd_lid [[thread_index_in_simdgroup]]) {constexpr int packs_per_thread = bits == 2 ? 1 : 2;- constexpr int num_simdgroups = 2;+ constexpr int num_simdgroups = 8;constexpr int results_per_simdgroup = 4;constexpr int pack_factor = get_pack_factor<bits, 32>();constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metalindex 82a033113..7675efb0b 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal@@ -7,7 +7,7 @@ PUBLIC KERNEL(TensorAddSwap)(device T* skip_buffer,device T* main_buffer,constant uint& length,- const uint position AXIS(length, 32)+ const uint position AXIS(length, 256)) {T skip_value = skip_buffer[position];T main_value = main_buffer[position];diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metalindex 052a48efe..7c6a35f79 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal@@ -7,7 +7,7 @@ PUBLIC KERNEL(TensorCopy)(const device T* src_buffer,device T* dst_buffer,constant uint& length,- const uint position AXIS(length, 32)+ const uint position AXIS(length, 256)) {dst_buffer[position] = src_buffer[position];}\ No newline at end of file