Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

optimizations

eugened7e5dbf542c6
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
index 4e3220a24..f56caf374 100644
--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
@@ -7,7 +7,8 @@ use crate::{
gpu_types::QuantizationMode,
kernel::{
QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,
- QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, QuantizedMatmulQvmKernel,
+ QuantizedMatmulQmmTransposedWideKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel,
+ QuantizedMatmulQvmKernel,
},
},
};
@@ -81,6 +82,7 @@ enum EncodableVariant<B: Backend> {
Qmm(<B::Kernels as Kernels>::QuantizedMatmulQmmKernel),
QmmTransposed(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel),
QmmTransposed64x64(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel),
+ QmmTransposedWide(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -103,6 +105,7 @@ enum MatrixMatrixFamily {
QmmTransposedAlignedN,
QmmTransposedUnalignedN,
QmmTransposed64x64,
+ QmmTransposedWide,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -268,6 +271,20 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
encoder,
);
},
+ EncodableVariant::QmmTransposedWide(kernel) => {
+ kernel.encode(
+ arguments.b_buffer,
+ arguments.scales_buffer,
+ zero_points,
+ biases,
+ a_with_offset,
+ arguments.output_buffer,
+ k,
+ n,
+ m,
+ encoder,
+ );
+ },
}
Ok(())
@@ -408,6 +425,17 @@ fn create_matrix_matrix_kernel<B: Backend>(
)
.map_err(QuantizedMatmulError::BackendError)?,
),
+ MatrixMatrixFamily::QmmTransposedWide => EncodableVariant::QmmTransposedWide(
+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(
+ context,
+ data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )
+ .map_err(QuantizedMatmulError::BackendError)?,
+ ),
};
Ok(kernel)
@@ -446,12 +474,16 @@ fn select_matrix_matrix_family(
) -> MatrixMatrixFamily {
if configuration.weights_transposed {
let aligned_n = configuration.output_dim % 32 == 0;
- let use_64x64 = aligned_n
+ let aligned_n_64 = configuration.output_dim % 64 == 0;
+ let use_64x64 = aligned_n_64
&& configuration.data_type == DataType::BF16
&& matches!(configuration.group_size, 64 | 128)
&& matches!(bits, 4 | 8);
+ let use_wide = aligned_n_64 && configuration.data_type == DataType::BF16 && matches!(bits, 4 | 8) && !use_64x64;
if use_64x64 {
MatrixMatrixFamily::QmmTransposed64x64
+ } else if use_wide {
+ MatrixMatrixFamily::QmmTransposedWide
} else if aligned_n {
MatrixMatrixFamily::QmmTransposedAlignedN
} else {
@@ -508,6 +540,7 @@ fn kernel_key_name(key: KernelKey) -> &'static str {
"matrix_matrix_qmm_transposed_unaligned_n"
},
KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposed64x64) => "matrix_matrix_qmm_transposed_64x64",
+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedWide) => "matrix_matrix_qmm_transposed_wide",
}
}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
index dadd0367b..330ce71e9 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
@@ -1,6 +1,7 @@
pub mod qmm;
pub mod qmm_transposed;
pub mod qmm_transposed_64x64;
+pub mod qmm_transposed_wide;
pub mod qmv;
pub mod qmv_fast;
pub mod qvm;
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs
new file mode 100644
index 000000000..f9cabc631
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs
@@ -0,0 +1,33 @@
+use dsl::kernel;
+use half::bf16;
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(QuantizedMatmulQmmTransposedWide)]
+#[variants(T, bf16)]
+#[variants(GROUP_SIZE, 32, 64, 128)]
+#[variants(BITS, 4, 8)]
+pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
+ #[allow(unused)] w: *const u32,
+ #[allow(unused)] scales: *const T,
+ #[allow(unused)]
+ #[optional(use_zero_points)]
+ zero_points: Option<*const u8>,
+ #[allow(unused)]
+ #[optional(use_mlx_quant)]
+ biases: Option<*const T>,
+ #[allow(unused)] x: *const T,
+ #[allow(unused)] y: *mut T,
+ #[allow(unused)] k: i32,
+ #[allow(unused)] n: i32,
+ #[allow(unused)] m: i32,
+ #[allow(unused)]
+ #[specialize]
+ use_zero_points: bool,
+ #[allow(unused)]
+ #[specialize]
+ use_mlx_quant: bool,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal
new file mode 100644
index 000000000..6118a70fe
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal
@@ -0,0 +1,76 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+#include "quant_matmul.metal"
+
+// Wide QMM tile: BM=64, BK=32, BN=64.
+// Processes 4x more output elements per threadgroup than the 32x32 variant.
+// BK=32 satisfies the group_size >= BK constraint for all group sizes (32, 64,
+// 128).
+template <typename T, int GROUP_SIZE, int BITS>
+VARIANTS(T, bfloat)
+VARIANTS(GROUP_SIZE, 32, 64, 128)
+VARIANTS(BITS, 4, 8)
+PUBLIC KERNEL(QuantizedMatmulQmmTransposedWide)(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points OPTIONAL(use_zero_points),
+ const device T* biases OPTIONAL(use_mlx_quant),
+ const device T* x,
+ device T* y,
+ const constant int& k,
+ const constant int& n,
+ const constant int& m,
+ threadgroup T Xs[64 * (32 + 16 / sizeof(T))],
+ threadgroup T Ws[64 * (32 + 16 / sizeof(T))],
+ const bool use_zero_points SPECIALIZE,
+ const bool use_mlx_quant SPECIALIZE,
+ const uint tgid_x GROUPS((n + 64 - 1) / 64),
+ const uint tgid_y GROUPS((m + 64 - 1) / 64),
+ const uint tgid_z GROUPS(1),
+ const uint tid_x THREADS(32),
+ const uint tid_y THREADS(2),
+ const uint tid_z THREADS(2)
+) {
+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
+ const uint lid = tid_z * 64 + tid_y * 32 + tid_x;
+ const uint simd_gid = tid_z * 2 + tid_y;
+ const uint simd_lid = tid_x;
+
+ if (use_mlx_quant) {
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, true>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ } else {
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, false>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
index 4b3120dee..c8d0a8b35 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
@@ -19,10 +19,10 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
const bool use_zero_points SPECIALIZE,
const bool use_mlx_quant SPECIALIZE,
const uint tgid_x GROUPS(m),
- const uint tgid_y GROUPS((n + 8 - 1) / 8),
+ const uint tgid_y GROUPS((n + 32 - 1) / 32),
const uint tgid_z GROUPS(1),
const uint tid_x THREADS(32),
- const uint tid_y THREADS(2)
+ const uint tid_y THREADS(8)
) {
const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
const uint simd_gid = tid_y;
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
index 59657ecff..6da2bfedf 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -1372,7 +1372,7 @@ void qmv_fast_impl(
uint simd_lid [[thread_index_in_simdgroup]]
) {
constexpr int packs_per_thread = bits == 2 ? 1 : 2;
- constexpr int num_simdgroups = 2;
+ constexpr int num_simdgroups = 8;
constexpr int results_per_simdgroup = 4;
constexpr int pack_factor = get_pack_factor<bits, 32>();
constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
index 82a033113..7675efb0b 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
@@ -7,7 +7,7 @@ PUBLIC KERNEL(TensorAddSwap)(
device T* skip_buffer,
device T* main_buffer,
constant uint& length,
- const uint position AXIS(length, 32)
+ const uint position AXIS(length, 256)
) {
T skip_value = skip_buffer[position];
T main_value = main_buffer[position];
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
index 052a48efe..7c6a35f79 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
@@ -7,7 +7,7 @@ PUBLIC KERNEL(TensorCopy)(
const device T* src_buffer,
device T* dst_buffer,
constant uint& length,
- const uint position AXIS(length, 32)
+ const uint position AXIS(length, 256)
) {
dst_buffer[position] = src_buffer[position];
}
\ No newline at end of file