Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

New generic qmv + tuning (#741)

CCYehbd3c8983f5f1
diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rs
index 310f88b24..a81773318 100644
--- a/crates/uzu-engine/src/backends/metal/context.rs
+++ b/crates/uzu-engine/src/backends/metal/context.rs
@@ -51,10 +51,10 @@ pub struct MetalContext {
impl MetalContext {
pub fn supports_mxu(&self) -> bool {
- self.device.supports_mxu()
+ self.device_profile.supports_mxu()
}
- pub fn device_profile(&self) -> DeviceProfile {
+ pub(super) fn device_profile(&self) -> DeviceProfile {
self.device_profile
}
@@ -157,12 +157,7 @@ impl Context for MetalContext {
let command_queue4 = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueueMtl4)?;
let gpu_core_count = device.gpu_core_count();
- let device_profile = classify_device(
- gpu_core_count,
- device.supports_family(metal::MTLGPUFamily::Apple8),
- device.supports_family(metal::MTLGPUFamily::Apple9),
- device.supports_mxu(),
- );
+ let device_profile = classify_device(&device.name(), gpu_core_count, device.supports_mxu());
let page_size = MTLSparsePageSize::KB256;
let heap_capacity = Metal::ALLOCATION_GRANULARITY;
let sparse_pool = MetalSparseHeapPool::new(page_size, heap_capacity);
diff --git a/crates/uzu-engine/src/backends/metal/device_profile.rs b/crates/uzu-engine/src/backends/metal/device_profile.rs
index 074a4055c..4177bb767 100644
--- a/crates/uzu-engine/src/backends/metal/device_profile.rs
+++ b/crates/uzu-engine/src/backends/metal/device_profile.rs
@@ -1,76 +1,141 @@
-//TODO: remove after retune with gpu core counts
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DeviceSize {
- /// Below Max/Ultra class: base and Pro dies, plus Max parts binned under
- /// the core-count cutoff (a 24-core M1 Max lands here, a 32-core one does not).
- Small,
- /// Max/Ultra-class GPUs, currently classified by >= 30 GPU cores.
- Large,
+pub(super) enum DeviceIdentity {
+ M1,
+ M1Pro,
+ M1Max,
+ M1Ultra,
+ M2,
+ M2Pro,
+ M2Max,
+ M2Ultra,
+ M3,
+ M3Pro,
+ M3Max,
+ M4,
+ M4Pro,
+ M4Max,
+ M5,
+ M5Pro,
+ M5Max,
+}
+
+impl DeviceIdentity {
+ fn from_name(device_name: &str) -> Self {
+ let Some(chip) = device_name.strip_prefix("Apple M") else {
+ panic!("unsupported Metal device {device_name:?}; add its DeviceIdentity mapping");
+ };
+ match chip {
+ "1" => Self::M1,
+ "1 Pro" => Self::M1Pro,
+ "1 Max" => Self::M1Max,
+ "1 Ultra" => Self::M1Ultra,
+ "2" => Self::M2,
+ "2 Pro" => Self::M2Pro,
+ "2 Max" => Self::M2Max,
+ "2 Ultra" => Self::M2Ultra,
+ "3" => Self::M3,
+ "3 Pro" => Self::M3Pro,
+ "3 Max" => Self::M3Max,
+ "4" => Self::M4,
+ "4 Pro" => Self::M4Pro,
+ "4 Max" => Self::M4Max,
+ "5" => Self::M5,
+ "5 Pro" => Self::M5Pro,
+ "5 Max" => Self::M5Max,
+ _ => panic!("unsupported Metal device {device_name:?}; add its DeviceIdentity mapping"),
+ }
+ }
+
+ const fn gpu_family(self) -> GpuFamily {
+ match self {
+ Self::M1 | Self::M1Pro | Self::M1Max | Self::M1Ultra => GpuFamily::Legacy,
+ Self::M2 | Self::M2Pro | Self::M2Max | Self::M2Ultra => GpuFamily::Apple8,
+ Self::M3 | Self::M3Pro | Self::M3Max | Self::M4 | Self::M4Pro | Self::M4Max => GpuFamily::Apple9,
+ Self::M5 | Self::M5Pro | Self::M5Max => GpuFamily::M5Plus,
+ }
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DeviceGeneration {
- Legacy, // M1 (G13) and A14 or older
- Apple8, // M2, A15/16
- Apple9, // M3/4, A17 Pro, A18
- M5Plus, // M5 (Apple9 + MXU support)
+pub(super) enum GpuFamily {
+ Legacy,
+ Apple8,
+ Apple9,
+ M5Plus,
+}
+
+impl GpuFamily {
+ pub(super) fn contains(
+ self,
+ identity: DeviceIdentity,
+ ) -> bool {
+ identity.gpu_family() == self
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct DeviceProfile {
- gpu_core_count: u32,
- generation: DeviceGeneration,
+pub(super) enum DeviceSize {
+ Small,
+ Large,
}
-const LARGE_MIN_GPU_CORES: u32 = 30;
+impl DeviceSize {
+ fn from_gpu_core_count(gpu_core_count: u32) -> Self {
+ if gpu_core_count >= 30 {
+ Self::Large
+ } else {
+ Self::Small
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) struct DeviceProfile {
+ identity: DeviceIdentity,
+ size: DeviceSize,
+ supports_mxu: bool,
+}
impl DeviceProfile {
- pub const fn new(
- gpu_core_count: u32,
- generation: DeviceGeneration,
+ pub(super) const fn new(
+ identity: DeviceIdentity,
+ size: DeviceSize,
+ supports_mxu: bool,
) -> Self {
Self {
- gpu_core_count,
- generation,
+ identity,
+ size,
+ supports_mxu,
}
}
- pub const fn size(self) -> DeviceSize {
- if self.gpu_core_count >= LARGE_MIN_GPU_CORES {
- DeviceSize::Large
- } else {
- DeviceSize::Small
- }
+ pub(super) const fn identity(self) -> DeviceIdentity {
+ self.identity
+ }
+
+ pub(super) const fn gpu_family(self) -> GpuFamily {
+ self.identity.gpu_family()
}
- // TODO: retune based on gpu core counts
- pub const fn gpu_core_count(self) -> u32 {
- self.gpu_core_count
+ pub(super) const fn size(self) -> DeviceSize {
+ self.size
}
- pub const fn generation(self) -> DeviceGeneration {
- self.generation
+ pub(super) const fn supports_mxu(self) -> bool {
+ self.supports_mxu
}
}
pub(super) fn classify_device(
+ device_name: &str,
gpu_core_count: u32,
- supports_apple8_family: bool,
- supports_apple9_family: bool,
supports_mxu: bool,
) -> DeviceProfile {
- // MXU is probed first: M5 also reports Apple9, so a family check alone
- // cannot separate the two generations.
- let generation = if supports_mxu {
- DeviceGeneration::M5Plus
- } else if supports_apple9_family {
- DeviceGeneration::Apple9
- } else if supports_apple8_family {
- DeviceGeneration::Apple8
- } else {
- DeviceGeneration::Legacy
- };
- DeviceProfile::new(gpu_core_count, generation)
+ DeviceProfile::new(
+ DeviceIdentity::from_name(device_name),
+ DeviceSize::from_gpu_core_count(gpu_core_count),
+ supports_mxu,
+ )
}
#[cfg(test)]
diff --git a/crates/uzu-engine/src/backends/metal/kernel/common/integral_constant.h b/crates/uzu-engine/src/backends/metal/kernel/common/integral_constant.h
index fa36461fc..6fd581ec6 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/common/integral_constant.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/common/integral_constant.h
@@ -6,6 +6,8 @@
namespace uzu {
+#define UZU_ALWAYS_INLINE __attribute__((always_inline))
+
template <typename T, T v>
struct integral_constant {
static constexpr constant T value = v;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs
index 475fc2254..5ee51c0e5 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs
@@ -9,7 +9,7 @@ use crate::{
},
metal::{
Metal, MetalContext,
- device_profile::{DeviceGeneration, DeviceSize},
+ device_profile::{DeviceSize, GpuFamily},
error::MetalError,
kernel::MetalKernels,
},
@@ -76,7 +76,7 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {
let use_mxu = arguments.data_type == DataType::BF16 && context.supports_mxu();
let transposed_h0 = !use_mxu
&& context.device_profile().size() == DeviceSize::Small
- && matches!(context.device_profile().generation(), DeviceGeneration::Legacy | DeviceGeneration::Apple8);
+ && matches!(context.device_profile().gpu_family(), GpuFamily::Legacy | GpuFamily::Apple8);
Ok(Self {
arguments: *arguments,
prefix: <MetalKernels as Kernels>::BuildTreePrefixKernel::new(context)?,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/qdot.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/qdot.h
deleted file mode 100644
index a72149d1e..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/qdot.h
+++ /dev/null
@@ -1,139 +0,0 @@
-#pragma once
-
-#include <metal_stdlib>
-
-#include "../../common/defines.h"
-#include "quant_unpack.h"
-
-using namespace metal;
-
-namespace uzu {
-namespace gemm {
-
-// For packed unsigned weights, pre-scales each activation lane k by
-// 2^-(BITS*k). qdot then reads the matching nibble/byte in place (value =
-// q * 2^(BITS*k)); the factors cancel, so the dot product is unchanged. All
-// factors are powers of two, so this is bit-exact.
-template <typename T, typename U, int VALUES_PER_THREAD, int BITS>
-METAL_FUNC U load_vector(const device T* x, thread U* x_thread, const bool signed_codes) {
- using U4 = vec<U, 4>;
- const U4 inv = (BITS == 8 && signed_codes)
- ? U4(U(1))
- : U4(U(1), U(1) / U(1u << BITS), U(1) / U(1u << (2u * BITS)), U(1) / U(1u << (3u * BITS)));
- U sum = 0;
- thread U4* x_vec4 = reinterpret_cast<thread U4*>(x_thread);
- METAL_PRAGMA_UNROLL
- for (int i = 0; i < VALUES_PER_THREAD / 4; i++) {
- U4 v = U4(x[4 * i], x[4 * i + 1], x[4 * i + 2], x[4 * i + 3]);
- sum += v[0] + v[1] + v[2] + v[3];
- x_vec4[i] = v * inv;
- }
- return sum;
-}
-
-template <typename T, typename U, int VALUES_PER_THREAD>
-METAL_FUNC U load_vector_safe(const device T* x, thread U* x_thread, int N) {
- U sum = 0;
- METAL_PRAGMA_UNROLL
- for (int i = 0; i < VALUES_PER_THREAD; ++i) {
- x_thread[i] = 0;
- }
- for (int i = 0; i < N; ++i) {
- U v = x[i];
- sum += v;
- x_thread[i] = v;
- }
- return sum;
-}
-
-template <typename U, int VALUES_PER_THREAD, int BITS>
-METAL_FUNC U qdot(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum, const bool signed_codes) {
- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
-
- U accumulator = 0;
- if constexpr (BITS == 4) {
- using U4 = vec<U, 4>;
- const device ushort* weight_words = reinterpret_cast<const device ushort*>(w);
- const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);
- const ushort packed_mask = signed_codes ? 0x8888u : 0u;
- METAL_PRAGMA_UNROLL
- for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {
- // Mask each nibble in place (no shifts); value of lane k is n_k << (4*k),
- // i.e. n_k * 16^k, which is < 2^23 so the magic-number convert is valid.
- // The matching x lane was pre-divided by 16^k in load_vector.
- const uint4 lanes = uint4(ushort(weight_words[i] ^ packed_mask)) & uint4(0x000fu, 0x00f0u, 0x0f00u, 0xf000u);
- const U4 weight_vec4 = U4(as_type<float4>(lanes | uint4(0x4b000000u)) - float4(8388608.0f));
- accumulator += dot(x_vec4[i], weight_vec4);
- }
- } else if constexpr (BITS == 8) {
- using U4 = vec<U, 4>;
- const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);
- if (signed_codes) {
- const device char4* weight_vectors = reinterpret_cast<const device char4*>(w);
- METAL_PRAGMA_UNROLL
- for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {
- accumulator += dot(x_vec4[i], U4(weight_vectors[i]));
- }
- } else {
- const device uint* weight_words = reinterpret_cast<const device uint*>(w);
- METAL_PRAGMA_UNROLL
- for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {
- // Keep each byte in place. Lane k is b_k * 256^k, while the
- // matching activation lane was pre-divided by 256^k.
- const uint4 lanes = uint4(weight_words[i]) & uint4(0x000000ffu, 0x0000ff00u, 0x00ff0000u, 0xff000000u);
- accumulator += dot(x_vec4[i], U4(float4(lanes)));
- }
- }
- }
- const U adjusted_bias = (BITS == 8 && signed_codes) ? bias + scale * U(128) : bias;
- return scale * accumulator + sum * adjusted_bias;
-}
-
-template <typename U, int VALUES_PER_THREAD, int BITS>
-METAL_FUNC U
-qdot_safe(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum, int N, const bool signed_codes) {
- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
-
- U accumulator = 0;
- if constexpr (BITS == 4) {
- using U4 = vec<U, 4>;
- const device uint16_t* weight_words = reinterpret_cast<const device uint16_t*>(w);
- const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);
- const uint16_t packed_mask = signed_codes ? 0x8888u : 0u;
-
- int full_chunks = N / 4;
- for (int i = 0; i < full_chunks; i++) {
- uint16_t weight_word = weight_words[i] ^ packed_mask;
- U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(weight_word, weight_word >> 4, weight_word >> 8, weight_word >> 12));
- accumulator += dot(x_vec4[i], weight_vec4);
- }
-
- int remainder = N & 3;
- if (remainder > 0) {
- uint16_t weight_word = weight_words[full_chunks] ^ packed_mask;
- int base_index = 4 * full_chunks;
- accumulator += x_thread[base_index] * uint_to_fp<U>(weight_word & 0xf);
- if (remainder > 1)
- accumulator += x_thread[base_index + 1] * uint_to_fp<U>((weight_word >> 4) & 0xf);
- if (remainder > 2)
- accumulator += x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);
- }
- } else if constexpr (BITS == 8) {
- if (signed_codes) {
- const device int8_t* signed_weights = reinterpret_cast<const device int8_t*>(w);
- for (int i = 0; i < N; i++) {
- accumulator += x_thread[i] * U(signed_weights[i]);
- }
- } else {
- for (int i = 0; i < N; i++) {
- accumulator += x_thread[i] * U(w[i]);
- }
- }
- }
-
- const U adjusted_bias = (BITS == 8 && signed_codes) ? bias + scale * U(128) : bias;
- return scale * accumulator + sum * adjusted_bias;
-}
-
-} // namespace gemm
-} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
index 9b1677df9..ec8a95f03 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
@@ -61,16 +61,21 @@ METAL_FUNC constexpr Int zero_point_row_stride(Int groups_per_row) {
}
template <ushort BITS>
-METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint group_index) {
+METAL_FUNC uint decode_zero_point(uint8_t packed, uint group_index) {
static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 zero points supported");
if constexpr (BITS == 4) {
- const uint packed = uint(zero_points_row[group_index >> 1]);
- return (packed >> ((group_index & 1u) * 4u)) & 0x0Fu;
+ return (uint(packed) >> ((group_index & 1u) * 4u)) & 0x0Fu;
} else {
- return uint(zero_points_row[group_index]);
+ return uint(packed);
}
}
+template <ushort BITS>
+METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint group_index) {
+ const uint byte_index = BITS == 4 ? group_index >> 1 : group_index;
+ return decode_zero_point<BITS>(zero_points_row[byte_index], group_index);
+}
+
METAL_FUNC char4 unpack_signed_nibbles_to_int8(uint packed) {
uint spread = (packed | (packed << 8)) & 0x00FF00FFu;
spread = (spread | (spread << 4)) & 0x0F0F0F0Fu;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
index 1e5091984..2350fd612 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -10,13 +10,13 @@ pub(super) use selection::GemmProblem;
use crate::backends::common::gpu_types::gemm::GemmTiling;
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GemmEngine {
Simdgroup,
Mxu,
}
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GemmPlan {
pub(super) engine: GemmEngine,
pub(super) tiling: GemmTiling,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
index 6b88dab0c..fffdc926d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
@@ -6,7 +6,7 @@
use crate::backends::{
common::gpu_types::gemm::GemmTiling,
- metal::device_profile::{DeviceGeneration, DeviceProfile},
+ metal::device_profile::{DeviceProfile, GpuFamily},
};
pub(super) const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;
@@ -103,7 +103,7 @@ pub(super) fn simdgroup_fp_tile(
/// A partial trailing M block costs a second pass over the weights. Older GPUs are bound by that;
/// Apple9 and newer would rather keep the narrow tile's parallelism.
fn prefers_wide_partial_m_tile(profile: DeviceProfile) -> bool {
- matches!(profile.generation(), DeviceGeneration::Legacy | DeviceGeneration::Apple8)
+ matches!(profile.gpu_family(), GpuFamily::Legacy | GpuFamily::Apple8)
}
pub(super) fn simdgroup_quant_tile(
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index d36e2f12a..8c3676138 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -77,6 +77,39 @@ impl GemmProblem {
Ok(())
}
+ #[cfg(test)]
+ pub fn plan_is_legal(
+ self,
+ plan: GemmPlan,
+ ) -> bool {
+ use super::specialization::GemmSpecialization;
+ use crate::backends::common::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment};
+
+ if self.validate_engine(plan.engine).is_err()
+ || plan.engine == GemmEngine::Mxu && !plan.tiling.is_mxu_variant()
+ || plan.engine == GemmEngine::Simdgroup && plan.tiling.is_mxu_variant()
+ || plan.split_k == 0
+ || !self.shape.k.is_multiple_of(plan.split_k)
+ {
+ return false;
+ }
+ let alignment = GemmAlignment::new(
+ self.shape.m.is_multiple_of(plan.tiling.block_m()),
+ self.shape.n.is_multiple_of(plan.tiling.block_n()),
+ self.shape.k.is_multiple_of(plan.tiling.block_k()),
+ );
+ GemmSpecialization::from_plan(
+ plan,
+ self.shape,
+ self.weights_data_type,
+ self.shape.d_transform,
+ alignment,
+ GemmAPrologueKind::FullPrecision,
+ None,
+ )
+ .is_ok()
+ }
+
fn finish_plan(
&self,
engine: GemmEngine,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h
new file mode 100644
index 000000000..739a3697f
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../common/defines.h"
+
+namespace uzu {
+namespace gemm {
+
+template <typename AT_, typename BT_, typename DT_>
+struct GemvOperands {
+ using AT = AT_;
+ using BT = BT_;
+ using DT = DT_;
+ const device uint32_t* b;
+ const device BT* scales;
+ const device uint8_t* zero_points;
+ const device BT* biases;
+ const device AT* a;
+ device DT* d;
+ const device BT* output_bias;
+ const device int32_t* hadamard_factors;
+ const device uint* gather_indices;
+};
+
+struct GemvParams {
+ uint in_vec_size;
+ uint out_vec_size;
+ uint batch_size;
+ float ab_scale;
+ float soft_cap;
+ GemmDTransform output_transform;
+ bool gathered;
+ bool signed_codes;
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/b_source.h
deleted file mode 100644
index 0c66e9157..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/b_source.h
+++ /dev/null
@@ -1,74 +0,0 @@
-#pragma once
-
-#include "../../../generated/gemm.h"
-#include "full_precision_b_source.h"
-#include "quantized_b_source.h"
-
-namespace uzu {
-namespace gemm {
-
-template <
- typename BT,
- typename AT,
- typename U,
- GemmBPrologueKind B_PROLOGUE,
- uint GROUP_SIZE,
- uint BITS,
- uint K_SPLIT,
- uint RESULTS_PER_SIMDGROUP,
- bool INPUT_ALIGNED>
-struct BSource {
- static METAL_FUNC void accumulate(
- thread U (&result)[RESULTS_PER_SIMDGROUP],
- const device uint32_t* b,
- const device BT* scales,
- const device uint8_t* zero_points,
- const device BT* biases,
- const device AT* a,
- const device uint* gather_indices,
- bool gathered,
- uint in_vec_size,
- uint out_vec_size,
- uint out_row,
- uint batch_idx,
- uint simd_lane,
- uint k_slice,
- const bool signed_codes
- ) {
- if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
- FullPrecisionBSource<BT, AT, U, RESULTS_PER_SIMDGROUP, K_SPLIT, INPUT_ALIGNED>::accumulate(
- result,
- b,
- a,
- gather_indices,
- gathered,
- in_vec_size,
- out_vec_size,
- out_row,
- batch_idx,
- simd_lane,
- k_slice
- );
- } else {
- QuantizedBSource<BT, AT, U, B_PROLOGUE, GROUP_SIZE, BITS, RESULTS_PER_SIMDGROUP, INPUT_ALIGNED>::accumulate(
- result,
- b,
- scales,
- zero_points,
- biases,
- a,
- gather_indices,
- gathered,
- in_vec_size,
- out_vec_size,
- out_row,
- batch_idx,
- simd_lane,
- signed_codes
- );
- }
- }
-};
-
-} // namespace gemm
-} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
index f28e52208..0bc642579 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
@@ -2,89 +2,122 @@
#include "../../../common/soft_cap.h"
#include "../../../hadamard_transform/hadamard_transform.h"
+#include "arguments.h"
+#include "tile.h"
namespace uzu {
namespace gemm {
-template <typename BT, typename DT, typename U, uint RESULTS_PER_SIMDGROUP>
+template <typename Tile, typename AT, typename BT, typename DT, bool FULL_TILE>
struct Epilogue {
+ using U = float;
+
static METAL_FUNC void store(
- thread U (&result)[RESULTS_PER_SIMDGROUP],
- device DT* d,
- const device BT* output_bias,
- const device int32_t* hadamard_factors,
- threadgroup U* shared_results,
- float ab_scale,
- float soft_cap,
- GemmDTransform output_transform,
- uint out_row,
- uint out_vec_size,
- uint out_block_idx,
- uint simd_group,
- uint simd_lane,
- bool writer
+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],
+ const thread GemvOperands<AT, BT, DT>& ops,
+ const thread GemvParams& params,
+ const thread OutputTile<Tile, FULL_TILE>& tile,
+ threadgroup U* shared_results
) {
- const bool is_scale = output_transform.contains(GemmDTransform::SCALE);
- const bool is_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);
- const bool is_bias = output_transform.contains(GemmDTransform::BIAS);
- const bool is_soft_cap = output_transform.contains(GemmDTransform::SOFT_CAP);
- const bool use_hadamard = output_transform.contains(GemmDTransform::RHT);
-
- if (writer && simd_lane == 0) {
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- U value = result[row];
- if (is_scale) {
- value = static_cast<U>(ab_scale) * value;
- }
- const uint global_row = out_row + row;
- if (is_accumulate && global_row < out_vec_size) {
- value += static_cast<U>(d[row]);
- }
- if (is_bias && !use_hadamard && global_row < out_vec_size) {
- value += static_cast<U>(output_bias[global_row]);
- }
- if (is_soft_cap) {
- value = apply_soft_cap(value, soft_cap);
- }
- result[row] = value;
- }
+ const bool use_hadamard =
+ Tile::OUTPUT_ROWS >= METAL_SIMD_SIZE && params.output_transform.contains(GemmDTransform::RHT);
+ const bool output_writer = tile.writer && tile.reduction_lane == 0;
+ if (!use_hadamard && !output_writer) {
+ return;
}
- if (use_hadamard) {
- if (simd_lane == 0) {
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] = result[row];
- }
- }
+ if (!use_hadamard) {
+ write_results<false>(result, ops, params, tile, shared_results);
+ return;
+ }
+ if (output_writer) {
+ write_results<true>(result, ops, params, tile, shared_results);
+ }
+ if constexpr (Tile::OUTPUT_ROWS >= METAL_SIMD_SIZE) {
+ static_assert(Tile::K_SPLIT == 1, "RHT reuses shared results after Reduce");
threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (simd_group == 0) {
- uint global_out_idx = out_block_idx * 32 + simd_lane;
- if (global_out_idx < out_vec_size) {
- DT transformed = simdgroup_output_random_hadamard_transform(
- static_cast<ushort>(simd_lane),
- static_cast<DT>(shared_results[simd_lane]),
- hadamard_factors[global_out_idx]
+ constexpr uint OUTPUT_BLOCKS = Tile::OUTPUT_ROWS / METAL_SIMD_SIZE;
+ for (uint job = tile.simd_group; job < Tile::INPUT_ROWS * OUTPUT_BLOCKS; job += Tile::NUM_SIMDGROUPS) {
+ const uint input_index = job / OUTPUT_BLOCKS;
+ const uint output_block = job % OUTPUT_BLOCKS;
+ const uint input_row = tile.input_row + input_index;
+ const uint global_row = tile.tile_row + output_block * METAL_SIMD_SIZE + tile.simd_lane;
+ if ((FULL_TILE || input_row < params.batch_size) && (FULL_TILE || global_row < params.out_vec_size)) {
+ DT value = simdgroup_output_random_hadamard_transform(
+ static_cast<ushort>(tile.simd_lane),
+ static_cast<DT>(
+ shared_results[input_index * Tile::OUTPUT_ROWS + output_block * METAL_SIMD_SIZE + tile.simd_lane]
+ ),
+ ops.hadamard_factors[global_row]
);
- if (is_bias) {
- transformed += static_cast<DT>(output_bias[global_out_idx]);
+ if (params.output_transform.contains(GemmDTransform::BIAS)) {
+ value += static_cast<DT>(ops.output_bias[global_row]);
}
- d[simd_lane] = transformed;
+ ops.d[input_row * params.out_vec_size + global_row] = value;
}
}
- } else {
- if (writer && simd_lane == 0) {
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- if (out_row + row < out_vec_size) {
- d[row] = static_cast<DT>(result[row]);
- }
+ }
+ }
+
+private:
+ template <bool STAGE_HADAMARD>
+ static METAL_FUNC void write_results(
+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],
+ const thread GemvOperands<AT, BT, DT>& ops,
+ const thread GemvParams& params,
+ const thread OutputTile<Tile, FULL_TILE>& tile,
+ threadgroup U* shared_results
+ ) {
+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {
+ constexpr uint I = decltype(input_index)::value;
+ const uint input_row = tile.input_row + I;
+ if (!(FULL_TILE || input_row < params.batch_size)) {
+ return;
+ }
+ device DT* output = ops.d + input_row * params.out_vec_size + tile.row0;
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ const uint global_row = tile.row0 + R;
+ if (!OutputTile<Tile, FULL_TILE>::row_in_range(global_row, params.out_vec_size)) {
+ return;
+ }
+ if (tile.clamped && params.output_transform.contains(GemmDTransform::ACCUMULATE)) {
+ return;
+ }
+ if constexpr (STAGE_HADAMARD) {
+ shared_results[I * Tile::OUTPUT_ROWS + tile.local_row + R] =
+ transform<false>(result[I][R], output + R, global_row, ops, params);
+ } else {
+ output[R] = static_cast<DT>(transform<true>(result[I][R], output + R, global_row, ops, params));
}
+ });
+ });
+ }
+
+ template <bool APPLY_BIAS>
+ static METAL_FUNC U transform(
+ U value,
+ const device DT* output,
+ uint global_row,
+ const thread GemvOperands<AT, BT, DT>& ops,
+ const thread GemvParams& params
+ ) {
+ if (params.output_transform.contains(GemmDTransform::SCALE)) {
+ value *= params.ab_scale;
+ }
+ if (params.output_transform.contains(GemmDTransform::ACCUMULATE)) {
+ value += static_cast<U>(*output);
+ }
+ if constexpr (APPLY_BIAS) {
+ if (params.output_transform.contains(GemmDTransform::BIAS)) {
+ value += static_cast<U>(ops.output_bias[global_row]);
}
}
+ if (params.output_transform.contains(GemmDTransform::SOFT_CAP)) {
+ value = apply_soft_cap(value, params.soft_cap);
+ }
+ return value;
}
};
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/full_precision_b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/full_precision_b_source.h
index 9305379a5..3dc7e1f48 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/full_precision_b_source.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/full_precision_b_source.h
@@ -1,67 +1,75 @@
#pragma once
-#include "../../common/defines.h"
+#include "arguments.h"
+#include "tile.h"
namespace uzu {
namespace gemm {
-template <typename BT, typename AT, typename U, uint RESULTS_PER_SIMDGROUP, uint K_SPLIT, bool INPUT_ALIGNED>
+template <typename Tile, typename AT, typename BT, typename DT, bool INPUT_ALIGNED, bool FULL_TILE>
struct FullPrecisionBSource {
+ using U = float;
+
static METAL_FUNC void accumulate(
- thread U (&result)[RESULTS_PER_SIMDGROUP],
- const device uint32_t* b,
- const device AT* a,
- const device uint* gather_indices,
- bool gathered,
- uint in_vec_size,
- uint out_vec_size,
- uint out_row,
- uint batch_idx,
- uint simd_lane,
- uint k_slice
+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],
+ const thread GemvOperands<AT, BT, DT>& ops,
+ const thread GemvParams& params,
+ const thread OutputTile<Tile, FULL_TILE>& tile
) {
- constexpr uint values_per_thread = 4;
- constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;
- typedef vec<BT, 4> W4;
- typedef vec<AT, 4> I4;
-
- const uint k_stride = K_SPLIT * block_size;
- const uint k_start = k_slice * block_size;
- const uint thread_k = simd_lane * values_per_thread + k_start;
- const device AT* input = a + batch_idx * in_vec_size + thread_k;
+ static_assert(Tile::INPUT_ROWS == 1, "full-precision GEMV uses one input row");
+ constexpr uint VALUES_PER_THREAD = 4;
+ constexpr uint BLOCK_SIZE = VALUES_PER_THREAD * METAL_SIMD_SIZE;
+ using W4 = vec<BT, 4>;
+ using I4 = vec<AT, 4>;
+ const uint k_stride = Tile::K_SPLIT * BLOCK_SIZE;
+ const uint k_start = tile.k_slice * BLOCK_SIZE;
+ const uint thread_k = tile.reduction_lane * VALUES_PER_THREAD + k_start;
+ const uint weights_row_stride = params.in_vec_size;
+ const uint input_row = tile.input_row;
+ const device AT* input = ops.a + input_row * params.in_vec_size + thread_k;
+ thread const device BT* weight_rows[Tile::ROWS_PER_LANE];
+ const uint last_row = params.out_vec_size > 0 ? params.out_vec_size - 1 : 0;
- // One advancing weight pointer per output row; base row = out_row (dense) or gather index.
- const uint base_row = gathered ? 0u : out_row;
- const device BT* weights = reinterpret_cast<const device BT*>(b);
- weights += base_row * in_vec_size + thread_k;
- thread const device BT* weight_rows[RESULTS_PER_SIMDGROUP];
METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- const uint addr_row = gathered ? gather_indices[batch_idx * out_vec_size + out_row + row] : row;
- weight_rows[row] = weights + addr_row * in_vec_size;
+ for (uint output_index = 0; output_index < Tile::ROWS_PER_LANE; output_index++) {
+ const uint global_row = tile.row0 + output_index;
+ const uint lookup_row = FULL_TILE ? global_row : min(global_row, last_row);
+ const uint weight_row =
+ params.gathered ? ops.gather_indices[input_row * params.out_vec_size + lookup_row] : lookup_row;
+ weight_rows[output_index] =
+ reinterpret_cast<const device BT*>(ops.b) + weight_row * weights_row_stride + thread_k;
}
uint k = k_start;
- for (; k + block_size <= in_vec_size; k += k_stride) {
- float4 input_values = static_cast<float4>(*reinterpret_cast<const device I4*>(input));
+ for (; k + BLOCK_SIZE <= params.in_vec_size; k += k_stride) {
+ const float4 input_values = static_cast<float4>(*reinterpret_cast<const device I4*>(input));
METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- result[row] += dot(static_cast<float4>(*reinterpret_cast<const device W4*>(weight_rows[row])), input_values);
- weight_rows[row] += k_stride;
+ for (uint output_index = 0; output_index < Tile::ROWS_PER_LANE; output_index++) {
+ const uint global_row = tile.row0 + output_index;
+ if (FULL_TILE || global_row < params.out_vec_size) {
+ result[0][output_index] +=
+ dot(static_cast<float4>(*reinterpret_cast<const device W4*>(weight_rows[output_index])), input_values);
+ }
+ weight_rows[output_index] += k_stride;
}
input += k_stride;
}
- if constexpr (K_SPLIT == 1 && !INPUT_ALIGNED) {
- const uint thread_offset = simd_lane * values_per_thread;
- const int remaining = (k + thread_offset < in_vec_size) ? min(static_cast<int>(in_vec_size - k - thread_offset),
- static_cast<int>(values_per_thread))
- : 0;
+ if constexpr (Tile::K_SPLIT == 1 && !INPUT_ALIGNED) {
+ const uint thread_offset = tile.reduction_lane * VALUES_PER_THREAD;
+ const int remaining =
+ k + thread_offset < params.in_vec_size
+ ? min(static_cast<int>(params.in_vec_size - k - thread_offset), static_cast<int>(VALUES_PER_THREAD))
+ : 0;
if (remaining > 0) {
METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- for (int index = 0; index < remaining; index++) {
- result[row] += static_cast<U>(weight_rows[row][index]) * static_cast<U>(input[index]);
+ for (uint output_index = 0; output_index < Tile::ROWS_PER_LANE; output_index++) {
+ const uint global_row = tile.row0 + output_index;
+ if (OutputTile<Tile, FULL_TILE>::row_in_range(global_row, params.out_vec_size)) {
+ for (int index = 0; index < remaining; index++) {
+ result[0][output_index] +=
+ static_cast<U>(weight_rows[output_index][index]) * static_cast<U>(input[index]);
+ }
}
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/output_tile.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/output_tile.h
deleted file mode 100644
index bc25f94c3..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/output_tile.h
+++ /dev/null
@@ -1,29 +0,0 @@
-#pragma once
-
-namespace uzu {
-namespace gemm {
-
-template <uint K_SPLIT, uint NUM_SIMDGROUPS, uint RESULTS_PER_SIMDGROUP>
-struct OutputTile {
- uint out_row;
- uint row_group;
- uint k_slice;
- bool writer;
-
- static METAL_FUNC OutputTile make(uint out_block_idx, uint simd_group, uint out_vec_size) {
- constexpr uint rows_per_threadgroup = (NUM_SIMDGROUPS / K_SPLIT) * RESULTS_PER_SIMDGROUP;
- const uint row_group = simd_group / K_SPLIT;
- const uint k_slice = simd_group % K_SPLIT;
- uint out_row = out_block_idx * rows_per_threadgroup + row_group * RESULTS_PER_SIMDGROUP;
- out_row = out_vec_size > RESULTS_PER_SIMDGROUP ? min(out_row, out_vec_size - RESULTS_PER_SIMDGROUP) : 0u;
- OutputTile tile;
- tile.out_row = out_row;
- tile.row_group = row_group;
- tile.k_slice = k_slice;
- tile.writer = (K_SPLIT == 1) || (k_slice == 0);
- return tile;
- }
-};
-
-} // namespace gemm
-} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h
new file mode 100644
index 000000000..6df334cdb
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h
@@ -0,0 +1,62 @@
+#pragma once
+
+#include "arguments.h"
+#include "quant_slice.h"
+#include "tile.h"
+
+namespace uzu {
+namespace gemm {
+
+template <
+ typename Tile,
+ typename AT,
+ typename BT,
+ typename DT,
+ GemmBPrologueKind B_PROLOGUE,
+ uint GROUP_SIZE,
+ uint BITS,
+ bool INPUT_ALIGNED,
+ bool FULL_TILE>
+struct QuantBSource {
+ using U = float;
+ using Slice = QuantSlice<Tile, AT, BT, DT, B_PROLOGUE, GROUP_SIZE, BITS, INPUT_ALIGNED, FULL_TILE>;
+ using Metadata = QuantMetadata<Tile, AT, BT, DT, B_PROLOGUE, BITS>;
+
+ static METAL_FUNC void accumulate(
+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],
+ const thread GemvOperands<AT, BT, DT>& ops,
+ const thread GemvParams& params,
+ const thread OutputTile<Tile, FULL_TILE>& tile
+ ) {
+ const uint groups = Metadata::group_count(params, GROUP_SIZE);
+ const uint row_stride = Slice::row_stride(params);
+ const uint group_slot = tile.reduction_lane / Tile::GROUP_LANES;
+ const uint group_offset = (tile.reduction_lane % Tile::GROUP_LANES) * Slice::VALUES_PER_LANE;
+ uint weight_row_indices[Tile::ROWS_PER_LANE];
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ const uint output_row = tile.row0 + R;
+ weight_row_indices[R] =
+ params.gathered ? ops.gather_indices[tile.input_row * params.out_vec_size + output_row] : output_row;
+ });
+
+ const device uint8_t* weights = reinterpret_cast<const device uint8_t*>(ops.b);
+ const uint batch_remaining = params.batch_size - tile.input_row;
+ uint group = group_slot;
+
+ Slice current;
+ QuantPosition position = {group, 0};
+ while (position.valid(groups)) {
+ Metadata metadata;
+ metadata.load(position.group, groups, weight_row_indices, ops, params);
+ for (position.slice = 0; position.slice < Slice::SLICES_PER_LANE; position.slice++) {
+ current.load_weights(position, weights, weight_row_indices, row_stride, group_offset);
+ current.accumulate(result, position, ops, params, tile, group_offset, batch_remaining, metadata);
+ }
+ position.group += Tile::GROUPS_PER_STEP;
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h
new file mode 100644
index 000000000..75e58a995
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h
@@ -0,0 +1,120 @@
+#pragma once
+
+#include "../../common/quant_unpack.h"
+
+namespace uzu {
+namespace gemm {
+
+template <uint BITS>
+struct QuantChunk {
+ UZU_CONST uint VALUES = 8;
+ UZU_CONST uint FLOAT_MANTISSA_BITS = 23;
+ UZU_CONST uint FLOAT_EXPONENT_BIAS = 127;
+ UZU_CONST uint MANTISSA_BASE_BITS = (FLOAT_EXPONENT_BIAS + FLOAT_MANTISSA_BITS) << FLOAT_MANTISSA_BITS;
+ UZU_CONST float MANTISSA_BASE = float(1u << FLOAT_MANTISSA_BITS);
+ UZU_CONST uint BITS_PER_BYTE = 8;
+ UZU_CONST uint WORD_BITS = 32;
+ UZU_CONST uint WORD_LANES = 4;
+ UZU_CONST uint WORD_BYTES = WORD_BITS / BITS_PER_BYTE;
+ UZU_CONST uint VECTOR_BYTES = WORD_BYTES * WORD_LANES;
+ UZU_CONST uint BF16_BITS = 16;
+ UZU_CONST uint BF16_VALUES_PER_WORD = WORD_BITS / BF16_BITS;
+
+ template <uint WORDS>
+ static METAL_FUNC void decode(
+ const thread uint4 (&words)[WORDS],
+ uint chunk,
+ thread float (&values)[VALUES],
+ float origin,
+ bool signed_codes
+ ) {
+ const uint word = chunk * (BITS == 4 ? 1u : 2u);
+ const uint word0 = words[word / 4u][word % 4u];
+ const uint word1 = BITS == 4 ? 0u : words[(word + 1u) / 4u][(word + 1u) % 4u];
+ decode_words(word0, word1, values, origin, signed_codes);
+ }
+
+ template <bool ALIGNED, typename AT>
+ static METAL_FUNC void load(
+ const device AT* source,
+ thread float (&values)[VALUES],
+ uint offset,
+ uint extent,
+ uint row
+ ) {
+ static_assert(
+ metal::is_same_v<AT, bfloat> || metal::is_same_v<AT, float>,
+ "GEMV quantized input must be bfloat or float; half cannot use bfloat widening"
+ );
+ constexpr bool EXPANDS_BF16 = metal::is_same_v<AT, bfloat>;
+ if constexpr (ALIGNED) {
+ load_values<false, EXPANDS_BF16>(source, values, VALUES);
+ } else {
+ const uint valid = offset < extent ? min(extent - offset, VALUES) : 0u;
+ const bool source_aligned = !EXPANDS_BF16 || ((row * extent + offset) & 7u) == 0;
+ if (valid == VALUES && source_aligned) {
+ load_values<false, EXPANDS_BF16>(source, values, VALUES);
+ } else {
+ load_values<true, EXPANDS_BF16>(source, values, valid);
+ }
+ }
+ }
+
+private:
+ static METAL_FUNC void decode_words(
+ uint word0,
+ uint word1,
+ thread float (&values)[VALUES],
+ float origin,
+ bool signed_codes
+ ) {
+ constexpr uint CODES_PER_WORD = WORD_BITS / BITS;
+ constexpr uint W4_SIGN_BITS = 0x88888888u;
+ constexpr uint W8_SIGN_BITS = 0x80808080u;
+ if constexpr (BITS == 4) {
+ word0 ^= signed_codes ? W4_SIGN_BITS : 0u;
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < VALUES; i++) {
+ values[i] = as_type<float>(extract_bits(word0, BITS * i, BITS) | MANTISSA_BASE_BITS) - origin;
+ }
+ } else {
+ const uint mask = signed_codes ? W8_SIGN_BITS : 0u;
+ word0 ^= mask;
+ word1 ^= mask;
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < CODES_PER_WORD; i++) {
+ values[i] = as_type<float>(extract_bits(word0, BITS * i, BITS) | MANTISSA_BASE_BITS) - origin;
+ values[CODES_PER_WORD + i] = as_type<float>(extract_bits(word1, BITS * i, BITS) | MANTISSA_BASE_BITS) - origin;
+ }
+ }
+ }
+
+ template <bool BOUNDED, bool EXPANDS_BF16, typename AT>
+ static METAL_FUNC void load_values(const device AT* source, thread float (&values)[VALUES], uint valid) {
+ if constexpr (!BOUNDED && EXPANDS_BF16) {
+ constexpr uint BF16_UPPER_BITS = 0xffff0000u;
+ const uint4 packed = *reinterpret_cast<const device uint4*>(source);
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < WORD_LANES; i++) {
+ values[BF16_VALUES_PER_WORD * i] = as_type<float>(packed[i] << BF16_BITS);
+ values[BF16_VALUES_PER_WORD * i + 1] = as_type<float>(packed[i] & BF16_UPPER_BITS);
+ }
+ } else {
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < VALUES; i++) {
+ float value = 0.0f;
+ if constexpr (BOUNDED) {
+ if (i < valid) {
+ value = float(source[i]);
+ }
+ } else {
+ value = float(source[i]);
+ }
+ values[i] = value;
+ }
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_metadata.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_metadata.h
new file mode 100644
index 000000000..1ef1039c6
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_metadata.h
@@ -0,0 +1,87 @@
+#pragma once
+
+#include "arguments.h"
+#include "quant_chunk.h"
+#include "tile.h"
+
+namespace uzu {
+namespace gemm {
+
+template <typename Tile, typename AT, typename BT, typename DT, GemmBPrologueKind B_PROLOGUE, uint BITS>
+struct QuantMetadata {
+private:
+ float scale[Tile::ROWS_PER_LANE];
+ float origin[Tile::ROWS_PER_LANE];
+ float bias[Tile::ROWS_PER_LANE];
+ bool signed_codes;
+
+public:
+ static METAL_FUNC uint group_count(const thread GemvParams& params, uint group_size) {
+ return (params.in_vec_size + group_size - 1) / group_size;
+ }
+
+ METAL_FUNC void load(
+ uint group_index,
+ uint group_count,
+ const thread uint (&weight_rows)[Tile::ROWS_PER_LANE],
+ const thread GemvOperands<AT, BT, DT>& ops,
+ const thread GemvParams& params
+ ) thread {
+ signed_codes = params.signed_codes;
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ const uint row = weight_rows[R];
+ scale[R] = float(ops.scales[row * group_count + group_index]);
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {
+ constexpr uint ZERO_POINTS_PER_BYTE = QuantChunk<BITS>::BITS_PER_BYTE / BITS;
+ const uint byte_index = group_index / ZERO_POINTS_PER_BYTE;
+ const uint8_t packed = ops.zero_points[row * zero_point_row_stride<BITS>(group_count) + byte_index];
+ origin[R] = QuantChunk<BITS>::MANTISSA_BASE + float(decode_zero_point<BITS>(packed, group_index));
+ bias[R] = 0.0f;
+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant) {
+ origin[R] = QuantChunk<BITS>::MANTISSA_BASE + float(symmetric_zero_point<BITS>());
+ bias[R] = 0.0f;
+ } else {
+ static_assert(B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant);
+ origin[R] = QuantChunk<BITS>::MANTISSA_BASE;
+ bias[R] = float(ops.biases[row * group_count + group_index]);
+ }
+ });
+ }
+
+ template <uint WORDS>
+ METAL_FUNC void decode(
+ const thread uint4 (&words)[WORDS],
+ uint chunk,
+ uint row,
+ thread float (&values)[QuantChunk<BITS>::VALUES]
+ ) const thread {
+ QuantChunk<BITS>::decode(words, chunk, values, origin[row], signed_codes);
+ }
+
+ METAL_FUNC void fold(
+ thread float (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],
+ const thread float (&partial)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],
+ const thread float (&input_sum)[Tile::INPUT_ROWS]
+ ) const thread {
+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {
+ constexpr uint I = decltype(input_index)::value;
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ result[I][R] = finish(result[I][R], partial[I][R], input_sum[I], R);
+ });
+ });
+ }
+
+private:
+ METAL_FUNC float finish(float prior, float partial, float input_sum, uint row) const thread {
+ float result = fma(scale[row], partial, prior);
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
+ result = fma(bias[row], input_sum, result);
+ }
+ return result;
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
new file mode 100644
index 000000000..fef42725d
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
@@ -0,0 +1,170 @@
+#pragma once
+
+#include "arguments.h"
+#include "quant_chunk.h"
+#include "quant_metadata.h"
+#include "tile.h"
+
+namespace uzu {
+namespace gemm {
+
+template <
+ typename Tile,
+ typename AT,
+ typename BT,
+ typename DT,
+ GemmBPrologueKind B_PROLOGUE,
+ uint GROUP_SIZE,
+ uint BITS,
+ bool INPUT_ALIGNED,
+ bool FULL_TILE>
+struct QuantSlice {
+ using U = float;
+
+public:
+ UZU_CONST uint VALUES_PER_LANE = GROUP_SIZE / Tile::GROUP_LANES;
+
+private:
+ UZU_CONST uint CHUNK_VALUES = QuantChunk<BITS>::VALUES;
+ UZU_CONST uint MAX_SLICE_VALUES = 32;
+ UZU_CONST uint SLICE_VALUES = VALUES_PER_LANE > MAX_SLICE_VALUES ? MAX_SLICE_VALUES : VALUES_PER_LANE;
+
+public:
+ UZU_CONST uint SLICES_PER_LANE = VALUES_PER_LANE / SLICE_VALUES;
+
+private:
+ UZU_CONST uint SLICE_BYTES = SLICE_VALUES * BITS / QuantChunk<BITS>::BITS_PER_BYTE;
+ UZU_CONST uint SLICE_WORDS = (SLICE_BYTES + QuantChunk<BITS>::VECTOR_BYTES - 1) / QuantChunk<BITS>::VECTOR_BYTES;
+ UZU_CONST uint CHUNKS_PER_SLICE = SLICE_VALUES / CHUNK_VALUES;
+
+ static_assert(GROUP_SIZE % Tile::GROUP_LANES == 0, "group lanes must divide the quantization group");
+ static_assert(VALUES_PER_LANE % CHUNK_VALUES == 0, "group lane slices must contain complete chunks");
+ static_assert(SLICE_VALUES <= MAX_SLICE_VALUES, "QMV slice exceeds its staged-value limit");
+ static_assert(VALUES_PER_LANE % SLICE_VALUES == 0, "a lane loads complete group slices");
+
+ uint4 weights[Tile::ROWS_PER_LANE][SLICE_WORDS];
+
+public:
+ static METAL_FUNC uint row_stride(const thread GemvParams& params) {
+ return params.in_vec_size * BITS / QuantChunk<BITS>::BITS_PER_BYTE;
+ }
+
+ METAL_FUNC void load_weights(
+ const thread QuantPosition& position,
+ const device uint8_t* weights_base,
+ const thread uint (&weight_row_indices)[Tile::ROWS_PER_LANE],
+ uint row_stride,
+ uint group_offset
+ ) thread {
+ const uint k = position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES;
+ const uint offset_bytes = k * BITS / QuantChunk<BITS>::BITS_PER_BYTE;
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ const uint row = weight_row_indices[R];
+ const device uint8_t* source = weights_base + row * row_stride + offset_bytes;
+ if constexpr (INPUT_ALIGNED) {
+ load_words<false>(weights[R], source);
+ } else {
+ const uint valid_bytes = offset_bytes < row_stride ? min(row_stride - offset_bytes, SLICE_BYTES) : 0u;
+ constexpr uint LOAD_ALIGNMENT =
+ SLICE_BYTES < QuantChunk<BITS>::VECTOR_BYTES ? SLICE_BYTES : QuantChunk<BITS>::VECTOR_BYTES;
+ const bool load_aligned = ((row * row_stride + offset_bytes) & (LOAD_ALIGNMENT - 1)) == 0;
+ if (valid_bytes == SLICE_BYTES && load_aligned) {
+ load_words<false>(weights[R], source);
+ } else {
+ load_words<true>(weights[R], source, valid_bytes);
+ }
+ }
+ });
+ }
+
+ METAL_FUNC void accumulate(
+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],
+ const thread QuantPosition& position,
+ const thread GemvOperands<AT, BT, DT>& ops,
+ const thread GemvParams& params,
+ const thread OutputTile<Tile, FULL_TILE>& tile,
+ uint group_offset,
+ uint batch_remaining,
+ const thread QuantMetadata<Tile, AT, BT, DT, B_PROLOGUE, BITS>& metadata
+ ) const thread {
+ float partial[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE] = {{0}};
+ float input_sum[Tile::INPUT_ROWS] = {0};
+ METAL_PRAGMA_UNROLL
+ for (uint chunk = 0; chunk < CHUNKS_PER_SLICE; chunk++) {
+ float weight_values[Tile::ROWS_PER_LANE][CHUNK_VALUES];
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ metadata.decode(weights[R], chunk, R, weight_values[R]);
+ });
+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {
+ constexpr uint I = decltype(input_index)::value;
+ const uint source_row = FULL_TILE ? I : min(I, batch_remaining - 1);
+ const uint k =
+ position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES + chunk * CHUNK_VALUES;
+ const uint input_row = tile.input_row + source_row;
+ const device AT* input = ops.a + input_row * params.in_vec_size + k;
+ float input_values[CHUNK_VALUES];
+ QuantChunk<BITS>::template load<INPUT_ALIGNED>(input, input_values, k, params.in_vec_size, input_row);
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < CHUNK_VALUES; i++) {
+ input_sum[I] += input_values[i];
+ }
+ }
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < CHUNK_VALUES; i++) {
+ partial[I][R] = fma(input_values[i], weight_values[R][i], partial[I][R]);
+ }
+ });
+ });
+ }
+ metadata.fold(result, partial, input_sum);
+ }
+
+private:
+ template <bool BOUNDED>
+ static METAL_FUNC void load_words(
+ thread uint4 (&words)[SLICE_WORDS],
+ const device uint8_t* source,
+ uint valid_bytes = SLICE_BYTES
+ ) {
+ if constexpr (!BOUNDED && SLICE_BYTES == QuantChunk<BITS>::WORD_BYTES) {
+ const uint packed = *reinterpret_cast<const device uint*>(source);
+ words[0] = uint4(packed, 0u, 0u, 0u);
+ } else if constexpr (!BOUNDED && SLICE_BYTES == 2 * QuantChunk<BITS>::WORD_BYTES) {
+ const uint2 packed = *reinterpret_cast<const device uint2*>(source);
+ words[0] = uint4(packed[0], packed[1], 0u, 0u);
+ } else if constexpr (!BOUNDED) {
+ static_assert(SLICE_BYTES % QuantChunk<BITS>::VECTOR_BYTES == 0, "large slices contain complete uint4 words");
+ const device uint4* packed = reinterpret_cast<const device uint4*>(source);
+ METAL_PRAGMA_UNROLL
+ for (uint word = 0; word < SLICE_WORDS; word++) {
+ words[word] = packed[word];
+ }
+ } else {
+ METAL_PRAGMA_UNROLL
+ for (uint word = 0; word < SLICE_WORDS; word++) {
+ uint4 packed = uint4(0u);
+ METAL_PRAGMA_UNROLL
+ for (uint component = 0; component < QuantChunk<BITS>::WORD_LANES; component++) {
+ uint value = 0u;
+ METAL_PRAGMA_UNROLL
+ for (uint byte = 0; byte < QuantChunk<BITS>::WORD_BYTES; byte++) {
+ const uint index = QuantChunk<BITS>::VECTOR_BYTES * word + QuantChunk<BITS>::WORD_BYTES * component + byte;
+ if (index < SLICE_BYTES && index < valid_bytes) {
+ value |= uint(source[index]) << (QuantChunk<BITS>::BITS_PER_BYTE * byte);
+ }
+ }
+ packed[component] = value;
+ }
+ words[word] = packed;
+ }
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_b_source.h
deleted file mode 100644
index cda4f41a3..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_b_source.h
+++ /dev/null
@@ -1,115 +0,0 @@
-#pragma once
-
-#include "../../common/qdot.h"
-#include "../../common/quant_pack.h"
-#include "quantized_row_state.h"
-
-namespace uzu {
-namespace gemm {
-
-template <
- typename BT,
- typename AT,
- typename U,
- GemmBPrologueKind B_PROLOGUE,
- uint GROUP_SIZE,
- uint BITS,
- uint RESULTS_PER_SIMDGROUP,
- bool INPUT_ALIGNED>
-struct QuantizedBSource {
- static METAL_FUNC void accumulate(
- thread U (&result)[RESULTS_PER_SIMDGROUP],
- const device uint32_t* b,
- const device BT* scales,
- const device uint8_t* zero_points,
- const device BT* biases,
- const device AT* a,
- const device uint* gather_indices,
- bool gathered,
- uint in_vec_size,
- uint out_vec_size,
- uint out_row,
- uint batch_idx,
- uint simd_lane,
- const bool signed_codes
- ) {
- constexpr uint pack_factor = get_pack_factor<BITS, 32>();
- constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();
- constexpr uint packs_per_thread = 2;
- constexpr uint values_per_thread = pack_factor * packs_per_thread;
- constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;
- constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;
- using RowState = QuantizedRowState<BT, U, B_PROLOGUE, BITS, RESULTS_PER_SIMDGROUP>;
- using RowParams = typename RowState::Params;
-
- const uint weights_row_stride = in_vec_size * bytes_per_pack / pack_factor;
- const uint group_count = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;
- const uint group_offset = simd_lane / scale_step_per_thread;
-
- // Base row = out_row (dense) or 0 (gather); rows indexed relative to it.
- const uint base_row = gathered ? 0u : out_row;
-
- const device uint8_t* weights = reinterpret_cast<const device uint8_t*>(b);
- weights += base_row * weights_row_stride + simd_lane * packs_per_thread * bytes_per_pack;
-
- RowState row_state(scales, zero_points, biases, base_row, group_count, group_offset);
-
- const device AT* input = a + batch_idx * in_vec_size + simd_lane * values_per_thread;
- thread U input_values[values_per_thread];
-
- uint k = 0;
- for (; k + block_size <= in_vec_size; k += block_size) {
- U input_sum = load_vector<AT, U, values_per_thread, BITS>(input, input_values, signed_codes);
-
- RowParams row_params;
- row_state.load(row_params, gather_indices, gathered, batch_idx, out_vec_size, out_row);
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- const uint addr_row = gathered ? gather_indices[batch_idx * out_vec_size + out_row + row] : row;
- const device uint8_t* weight_row = weights + addr_row * weights_row_stride;
- result[row] += qdot<U, values_per_thread, BITS>(
- weight_row,
- input_values,
- row_params.scale[row],
- row_params.offset[row],
- input_sum,
- signed_codes
- );
- }
-
- weights += block_size * bytes_per_pack / pack_factor;
- row_state.advance(block_size / GROUP_SIZE);
- input += block_size;
- }
-
- if constexpr (!INPUT_ALIGNED) {
- const uint thread_offset = simd_lane * values_per_thread;
- const int remaining = (k + thread_offset < in_vec_size) ? min(static_cast<int>(in_vec_size - k - thread_offset),
- static_cast<int>(values_per_thread))
- : 0;
- if (remaining > 0) {
- U input_sum = load_vector_safe<AT, U, values_per_thread>(input, input_values, remaining);
-
- RowParams row_params;
- row_state.load(row_params, gather_indices, gathered, batch_idx, out_vec_size, out_row);
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- const uint addr_row = gathered ? gather_indices[batch_idx * out_vec_size + out_row + row] : row;
- const device uint8_t* weight_row = weights + addr_row * weights_row_stride;
- result[row] += qdot_safe<U, values_per_thread, BITS>(
- weight_row,
- input_values,
- row_params.scale[row],
- row_params.offset[row],
- input_sum,
- remaining,
- signed_codes
- );
- }
- }
- }
- }
-};
-
-} // namespace gemm
-} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_row_state.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_row_state.h
deleted file mode 100644
index e62b01d62..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_row_state.h
+++ /dev/null
@@ -1,143 +0,0 @@
-#pragma once
-
-#include "../../../generated/gemm.h"
-#include "../../common/defines.h"
-#include "../../common/quant_pack.h"
-
-namespace uzu {
-namespace gemm {
-
-template <typename U, uint RESULTS_PER_SIMDGROUP>
-struct QuantizedAffineRowParams {
- U scale[RESULTS_PER_SIMDGROUP];
- U offset[RESULTS_PER_SIMDGROUP];
-};
-
-template <typename T, typename U>
-struct QuantizedGroupRows {
- const device T* values;
- uint group_stride;
-
- QuantizedGroupRows(const device T* values_base, uint base_row, uint group_count, uint group_offset)
- : values(values_base + base_row * group_count + group_offset), group_stride(group_count) {}
-
- METAL_FUNC U value(uint row) const { return static_cast<U>(values[row * group_stride]); }
-
- void advance(uint groups) { values += groups; }
-};
-
-template <typename BT, typename U, GemmBPrologueKind B_PROLOGUE, uint BITS>
-struct QuantizedOffsetState;
-
-template <typename BT, typename U, uint BITS>
-struct QuantizedOffsetState<BT, U, GemmBPrologueKind::ScaleBiasDequant, BITS> {
- QuantizedGroupRows<BT, U> biases;
-
- QuantizedOffsetState(
- const device uint8_t*,
- const device BT* biases_base,
- uint base_row,
- uint group_count,
- uint group_offset
- )
- : biases(biases_base, base_row, group_count, group_offset) {}
-
- METAL_FUNC U value(uint row, U) const { return biases.value(row); }
-
- void advance(uint groups) { biases.advance(groups); }
-};
-
-template <typename BT, typename U, uint BITS>
-struct QuantizedOffsetState<BT, U, GemmBPrologueKind::ScaleZeroPointDequant, BITS> {
- const device uint8_t* zero_points;
- uint zero_point_stride;
- uint zero_point_shift;
-
- QuantizedOffsetState(
- const device uint8_t* zero_points_base,
- const device BT*,
- uint base_row,
- uint group_count,
- uint group_offset
- ) {
- if constexpr (BITS == 4) {
- zero_point_stride = (group_count + 1) / 2;
- zero_points = zero_points_base + base_row * zero_point_stride + group_offset / 2;
- zero_point_shift = (group_offset & 1) * 4u;
- } else {
- zero_point_stride = group_count;
- zero_points = zero_points_base + base_row * zero_point_stride + group_offset;
- }
- }
-
- METAL_FUNC U value(uint row, U scale) const {
- uint8_t zero_point_value = zero_points[row * zero_point_stride];
- if constexpr (BITS == 4) {
- zero_point_value = (zero_point_value >> zero_point_shift) & 0x0F;
- }
- return -scale * static_cast<U>(zero_point_value);
- }
-
- void advance(uint groups) {
- constexpr uint zero_points_per_byte = get_pack_factor<BITS, 8>();
- zero_points += groups / zero_points_per_byte;
- }
-};
-
-template <typename BT, typename U, uint BITS>
-struct QuantizedOffsetState<BT, U, GemmBPrologueKind::ScaleSymmetricDequant, BITS> {
- QuantizedOffsetState(const device uint8_t*, const device BT*, uint, uint, uint) {}
-
- METAL_FUNC U value(uint, U scale) const {
- constexpr U midpoint = U(1u << (BITS - 1));
- return -scale * midpoint;
- }
-};
-
-template <typename BT, typename U, GemmBPrologueKind B_PROLOGUE, uint BITS, uint RESULTS_PER_SIMDGROUP>
-struct QuantizedRowState {
- static_assert(BITS == 4 || BITS == 8, "QMV supports 4- and 8-bit only");
-
- using Params = QuantizedAffineRowParams<U, RESULTS_PER_SIMDGROUP>;
-
- QuantizedGroupRows<BT, U> scale_rows;
- QuantizedOffsetState<BT, U, B_PROLOGUE, BITS> offset_state;
-
- QuantizedRowState(
- const device BT* scales_base,
- const device uint8_t* zero_points_base,
- const device BT* biases_base,
- uint base_row,
- uint group_count,
- uint group_offset
- )
- : scale_rows(scales_base, base_row, group_count, group_offset),
- offset_state(zero_points_base, biases_base, base_row, group_count, group_offset) {}
-
- // Row index: dense = local row, gather = absolute gather index.
- void load(
- thread Params& params,
- const device uint* gather_indices,
- bool gathered,
- uint batch_idx,
- uint out_vec_size,
- uint out_row
- ) const {
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- const uint addr_row = gathered ? gather_indices[batch_idx * out_vec_size + out_row + row] : row;
- const U scale = scale_rows.value(addr_row);
- params.scale[row] = scale;
- params.offset[row] = offset_state.value(addr_row, scale);
- }
- }
-
- void advance(uint groups) {
- scale_rows.advance(groups);
- if constexpr (B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant)
- offset_state.advance(groups);
- }
-};
-
-} // namespace gemm
-} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/reduce.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/reduce.h
index 30c79fd14..6620775b3 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/reduce.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/reduce.h
@@ -1,41 +1,64 @@
#pragma once
+#include "tile.h"
+
namespace uzu {
namespace gemm {
-template <typename U, uint K_SPLIT, uint NUM_SIMDGROUPS, uint RESULTS_PER_SIMDGROUP>
+template <typename Tile, bool FULL_TILE>
struct Reduce {
+ using U = float;
+
static METAL_FUNC void run(
- thread U (&result)[RESULTS_PER_SIMDGROUP],
+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],
threadgroup U* shared_results,
- uint simd_group,
- uint simd_lane,
- uint row_group,
- uint k_slice
+ const thread OutputTile<Tile, FULL_TILE>& tile
) {
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- result[row] = simd_sum(result[row]);
- }
-
- if constexpr (K_SPLIT > 1) {
- if (simd_lane == 0) {
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] = result[row];
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (k_slice == 0 && simd_lane == 0) {
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {
- U acc = 0;
+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {
+ constexpr uint I = decltype(input_index)::value;
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ if constexpr (Tile::REDUCTION_LANES == METAL_SIMD_SIZE) {
+ result[I][R] = simd_sum(result[I][R]);
+ } else {
+ U reduced = result[I][R];
METAL_PRAGMA_UNROLL
- for (uint s = 0; s < K_SPLIT; s++) {
- acc += shared_results[(row_group * K_SPLIT + s) * RESULTS_PER_SIMDGROUP + row];
+ for (uint offset = Tile::REDUCTION_LANES / 2; offset > 0; offset /= 2) {
+ reduced += simd_shuffle_xor(reduced, offset);
}
- result[row] = acc;
+ result[I][R] = reduced;
}
+ });
+ });
+
+ if constexpr (Tile::K_SPLIT > 1) {
+ if (tile.reduction_lane == 0) {
+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {
+ constexpr uint I = decltype(input_index)::value;
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ const uint row = tile.local_row + R;
+ shared_results
+ [I * Tile::OUTPUT_ROWS * Tile::K_SPLIT + tile.simd_group * Tile::SIMDGROUP_OUTPUT_ROWS +
+ (row % Tile::SIMDGROUP_OUTPUT_ROWS)] = result[I][R];
+ });
+ });
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tile.k_slice == 0 && tile.reduction_lane == 0) {
+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {
+ constexpr uint I = decltype(input_index)::value;
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ U sum = 0;
+ for (uint slice = 0; slice < Tile::K_SPLIT; slice++) {
+ const uint group = tile.row_group * Tile::K_SPLIT + slice;
+ const uint row = group * Tile::SIMDGROUP_OUTPUT_ROWS + (tile.local_row % Tile::SIMDGROUP_OUTPUT_ROWS) + R;
+ sum += shared_results[I * Tile::OUTPUT_ROWS * Tile::K_SPLIT + row];
+ }
+ result[I][R] = sum;
+ });
+ });
}
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/tile.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/tile.h
new file mode 100644
index 000000000..d980352e9
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/tile.h
@@ -0,0 +1,101 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../common/integral_constant.h"
+#include "../../common/defines.h"
+
+namespace uzu {
+namespace gemm {
+
+struct QuantPosition {
+ uint group;
+ uint slice;
+
+ METAL_FUNC bool valid(uint groups) const thread { return group < groups; }
+};
+
+template <
+ uint INPUT_ROW_TILE_,
+ uint OUTPUT_ROW_TILE_,
+ uint REDUCTION_LANES_,
+ uint GROUP_LANES_,
+ uint NUM_SIMDGROUPS_,
+ uint K_SPLIT_>
+struct GemvTile {
+ // Compile-time geometry.
+ UZU_CONST uint INPUT_ROWS = INPUT_ROW_TILE_;
+ UZU_CONST uint OUTPUT_ROWS = OUTPUT_ROW_TILE_;
+ UZU_CONST uint REDUCTION_LANES = REDUCTION_LANES_;
+ UZU_CONST uint GROUP_LANES = GROUP_LANES_;
+ UZU_CONST uint NUM_SIMDGROUPS = NUM_SIMDGROUPS_;
+ UZU_CONST uint K_SPLIT = K_SPLIT_;
+ UZU_CONST uint ROW_BLOCKS = METAL_SIMD_SIZE / REDUCTION_LANES;
+ UZU_CONST uint SIMDGROUP_OUTPUT_ROWS = OUTPUT_ROWS / (NUM_SIMDGROUPS / K_SPLIT);
+ UZU_CONST uint ROWS_PER_LANE = SIMDGROUP_OUTPUT_ROWS / ROW_BLOCKS;
+ UZU_CONST uint GROUPS_PER_STEP = REDUCTION_LANES / GROUP_LANES;
+
+ static_assert(INPUT_ROWS > 0 && OUTPUT_ROWS > 0, "GEMV tiles must be non-empty");
+ static_assert(
+ REDUCTION_LANES == 4 || REDUCTION_LANES == 8 || REDUCTION_LANES == 16 || REDUCTION_LANES == 32,
+ "reduction lanes must divide one simdgroup"
+ );
+ static_assert(METAL_SIMD_SIZE % REDUCTION_LANES == 0, "reduction lanes must divide one simdgroup");
+ static_assert(GROUP_LANES > 0 && REDUCTION_LANES % GROUP_LANES == 0, "group lanes must divide reduction lanes");
+ static_assert(NUM_SIMDGROUPS > 0 && NUM_SIMDGROUPS % K_SPLIT == 0, "K split must divide simdgroups");
+ static_assert(OUTPUT_ROWS % (NUM_SIMDGROUPS / K_SPLIT) == 0, "output rows must divide row groups");
+ static_assert(SIMDGROUP_OUTPUT_ROWS % ROW_BLOCKS == 0, "output rows must divide reduction blocks");
+
+ template <typename Fn>
+ static METAL_FUNC void for_each_input_row(Fn fn) {
+ const_for_loop<0, INPUT_ROWS, 1>(fn);
+ }
+
+ template <typename Fn>
+ static METAL_FUNC void for_each_output_row(Fn fn) {
+ const_for_loop<0, ROWS_PER_LANE, 1>(fn);
+ }
+};
+
+template <typename Tile, bool FULL_TILE>
+struct OutputTile {
+ uint simd_group;
+ uint simd_lane;
+ uint reduction_lane;
+ uint row_group;
+ uint k_slice;
+ uint input_row;
+ uint tile_row;
+ uint local_row;
+ uint row0;
+ bool writer;
+ bool clamped;
+
+ static METAL_FUNC OutputTile
+ make(uint output_tile_idx, uint input_tile_idx, uint simd_group, uint simd_lane, uint out_vec_size) {
+ const uint row_group = simd_group / Tile::K_SPLIT;
+ const uint k_slice = simd_group % Tile::K_SPLIT;
+ const uint row_block = simd_lane / Tile::REDUCTION_LANES;
+ const uint local_row = row_group * Tile::SIMDGROUP_OUTPUT_ROWS + row_block * Tile::ROWS_PER_LANE;
+ const uint unclamped = output_tile_idx * Tile::OUTPUT_ROWS + local_row;
+ const uint last = out_vec_size > Tile::ROWS_PER_LANE ? out_vec_size - Tile::ROWS_PER_LANE : 0;
+ OutputTile tile;
+ tile.simd_group = simd_group;
+ tile.simd_lane = simd_lane;
+ tile.reduction_lane = simd_lane % Tile::REDUCTION_LANES;
+ tile.row_group = row_group;
+ tile.k_slice = k_slice;
+ tile.input_row = input_tile_idx * Tile::INPUT_ROWS;
+ tile.tile_row = output_tile_idx * Tile::OUTPUT_ROWS;
+ tile.local_row = local_row;
+ tile.row0 = FULL_TILE ? unclamped : min(unclamped, last);
+ tile.writer = Tile::K_SPLIT == 1 || k_slice == 0;
+ tile.clamped = !FULL_TILE && unclamped > last;
+ return tile;
+ }
+
+ static METAL_FUNC bool row_in_range(uint row, uint out_vec_size) { return FULL_TILE || row < out_vec_size; }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 08de3feeb..940dbd67c 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -1,11 +1,15 @@
#include "../../common/dsl.h"
+#include "../../common/integral_constant.h"
#include "../../generated/gemm.h"
-#include "common/b_source.h"
+#include "common/arguments.h"
#include "common/epilogue.h"
-#include "common/output_tile.h"
+#include "common/full_precision_b_source.h"
+#include "common/tile.h"
+#include "common/quant_b_source.h"
#include "common/reduce.h"
using namespace metal;
+using namespace uzu;
using namespace uzu::gemm;
template <
@@ -17,7 +21,10 @@ template <
uint BITS,
uint K_SPLIT,
bool INPUT_ALIGNED,
- uint RESULTS_PER_SIMDGROUP,
+ uint INPUT_ROW_TILE,
+ uint OUTPUT_ROW_TILE,
+ uint REDUCTION_LANES,
+ uint GROUP_LANES,
uint NUM_SIMDGROUPS>
VARIANTS(AT, bfloat, float)
VARIANTS(BT, bfloat, float)
@@ -33,21 +40,75 @@ VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)
VARIANTS(BITS, 0, 4, 8)
VARIANTS(K_SPLIT, 1, 2, 4, 8)
VARIANTS(INPUT_ALIGNED, false, true)
-VARIANTS(RESULTS_PER_SIMDGROUP, 1, 2, 4, 8)
+VARIANTS(INPUT_ROW_TILE, 1, 2, 3, 4, 5, 6, 7, 8)
+VARIANTS(OUTPUT_ROW_TILE, 1, 2, 4, 8, 16, 32, 64)
+VARIANTS(REDUCTION_LANES, 8, 16, 32)
+VARIANTS(GROUP_LANES, 1, 2, 4, 8, 16)
VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)
+
CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))
CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")
-CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || K_SPLIT == 1)
-CONSTRAINT(K_SPLIT <= NUM_SIMDGROUPS)
-// Only selector-reachable tiles are instantiated (fleet-tuned tables): fp
-// always runs 8 simdgroups with 1 or 4 rows each; non-default quantized
-// tiles exist for bf16 IO only. Widen locally when sweeping new configs.
-CONSTRAINT(BITS != 0 || NUM_SIMDGROUPS == 8)
-CONSTRAINT(BITS != 0 || RESULTS_PER_SIMDGROUP == 1 || RESULTS_PER_SIMDGROUP == 4)
+CONSTRAINT(BITS == 0 || K_SPLIT == 1)
+CONSTRAINT(BITS != 0 || (INPUT_ROW_TILE == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && GROUP_LANES == 1))
+CONSTRAINT(INPUT_ROW_TILE != 1 || REDUCTION_LANES == 32)
+CONSTRAINT(
+ INPUT_ROW_TILE == 1 ||
+ (K_SPLIT == 1 && INPUT_ALIGNED && AT == "bfloat" && DT == "bfloat" && GROUP_LANES == 1))
+
+#define GEMV_TILE(OUTPUT_ROWS, LANES, SIMD_GROUPS) \
+ (OUTPUT_ROW_TILE == OUTPUT_ROWS && REDUCTION_LANES == LANES && NUM_SIMDGROUPS == SIMD_GROUPS)
+#define INPUT_ROWS(FIRST, LAST) (INPUT_ROW_TILE >= FIRST && INPUT_ROW_TILE <= LAST)
+
+CONSTRAINT(
+ INPUT_ROW_TILE == 1 ||
+ (((B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8) ||
+ (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant && BITS == 4)) &&
+ (GROUP_SIZE == 32 || GROUP_SIZE == 64) &&
+ ((GEMV_TILE(16, 8, 2) && INPUT_ROWS(2, 7)) ||
+ (GEMV_TILE(16, 16, 4) && INPUT_ROWS(2, 6) &&
+ ((GROUP_SIZE == 32 && (BITS == 4 || INPUT_ROW_TILE >= 3)) ||
+ (BITS == 4 && GROUP_SIZE == 64 && INPUT_ROW_TILE >= 3))) ||
+ (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant && GEMV_TILE(8, 8, 2) &&
+ ((GROUP_SIZE == 32 && (INPUT_ROW_TILE == 4 || INPUT_ROW_TILE == 8)) ||
+ (GROUP_SIZE == 64 && INPUT_ROW_TILE == 2))))))
+
+// Keep only selector-reachable geometry families.
CONSTRAINT(
- BITS == 0 || (NUM_SIMDGROUPS == 8 && RESULTS_PER_SIMDGROUP == 4) ||
- (AT == "bfloat" && DT == "bfloat"))
+ BITS == 0 ||
+ (NUM_SIMDGROUPS == 2 &&
+ (OUTPUT_ROW_TILE == 2 || OUTPUT_ROW_TILE == 4 || OUTPUT_ROW_TILE == 8 ||
+ (INPUT_ROW_TILE > 1 && OUTPUT_ROW_TILE == 16))) ||
+ (NUM_SIMDGROUPS == 4 && (OUTPUT_ROW_TILE == 8 || OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32)) ||
+ (NUM_SIMDGROUPS == 8 &&
+ (OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32 ||
+ (BITS == 4 && INPUT_ALIGNED && OUTPUT_ROW_TILE == 64))))
+CONSTRAINT(
+ BITS == 0 || (AT == "bfloat" && DT == "bfloat") ||
+ (NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32))
+CONSTRAINT(BITS != 8 || (NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32) || INPUT_ROW_TILE > 1)
+CONSTRAINT(INPUT_ALIGNED || K_SPLIT == 1)
+CONSTRAINT(K_SPLIT <= NUM_SIMDGROUPS && NUM_SIMDGROUPS % K_SPLIT == 0)
+CONSTRAINT(OUTPUT_ROW_TILE % (NUM_SIMDGROUPS / K_SPLIT) == 0)
+CONSTRAINT(OUTPUT_ROW_TILE / (NUM_SIMDGROUPS / K_SPLIT) % (32 / REDUCTION_LANES) == 0)
+CONSTRAINT(
+ (BITS == 0 && (OUTPUT_ROW_TILE == (NUM_SIMDGROUPS / K_SPLIT) ||
+ OUTPUT_ROW_TILE == 4 * (NUM_SIMDGROUPS / K_SPLIT))) ||
+ (BITS != 0 &&
+ (OUTPUT_ROW_TILE == NUM_SIMDGROUPS || OUTPUT_ROW_TILE == 2 * NUM_SIMDGROUPS ||
+ OUTPUT_ROW_TILE == 4 * NUM_SIMDGROUPS || OUTPUT_ROW_TILE == 8 * NUM_SIMDGROUPS)))
+CONSTRAINT(REDUCTION_LANES % GROUP_LANES == 0)
+CONSTRAINT(BITS == 0 || GROUP_SIZE % GROUP_LANES == 0)
+CONSTRAINT(BITS != 4 || (GROUP_SIZE / GROUP_LANES) % 16 == 0)
+CONSTRAINT(BITS != 8 || (GROUP_SIZE / GROUP_LANES) % 8 == 0)
+CONSTRAINT(BITS != 0 || REDUCTION_LANES == 32)
+CONSTRAINT(
+ BITS == 0 || INPUT_ROW_TILE > 1 || (BITS == 4 &&
+ ((GROUP_SIZE == 16 && GROUP_LANES == 1) || (GROUP_SIZE == 32 && GROUP_LANES == 2) ||
+ (GROUP_SIZE == 64 && GROUP_LANES == 4) || (GROUP_SIZE == 128 && GROUP_LANES == 8))) ||
+ (BITS == 8 &&
+ ((GROUP_SIZE == 16 && GROUP_LANES == 2) || (GROUP_SIZE == 32 && GROUP_LANES == 4) ||
+ (GROUP_SIZE == 64 && GROUP_LANES == 8) || (GROUP_SIZE == 128 && GROUP_LANES == 16))))
KERNEL(Gemv)(
const device uint32_t* b,
const device BT* scales
@@ -73,60 +134,36 @@ KERNEL(Gemv)(
const GemmDTransform output_transform SPECIALIZE,
const bool gathered SPECIALIZE,
const bool signed_codes SPECIALIZE,
- threadgroup float shared_results[NUM_SIMDGROUPS * RESULTS_PER_SIMDGROUP],
- const uint batch_idx GROUPS(batch_size),
- const uint out_block_idx GROUPS(group_count_x),
+ const bool full_tile SPECIALIZE,
+ threadgroup float shared_results[INPUT_ROW_TILE * OUTPUT_ROW_TILE * K_SPLIT],
+ const uint input_tile_idx GROUPS(batch_size.div_ceil(INPUT_ROW_TILE)),
+ const uint output_tile_idx GROUPS(group_count_x),
const uint simd_lane THREADS(32),
const uint simd_group THREADS(NUM_SIMDGROUPS)
) {
- typedef float U;
- thread U result[RESULTS_PER_SIMDGROUP] = {0};
-
- OutputTile<K_SPLIT, NUM_SIMDGROUPS, RESULTS_PER_SIMDGROUP> tile =
- OutputTile<K_SPLIT, NUM_SIMDGROUPS, RESULTS_PER_SIMDGROUP>::make(out_block_idx, simd_group, out_vec_size);
- d += batch_idx * out_vec_size + tile.out_row;
-
- BSource<BT, AT, U, B_PROLOGUE, GROUP_SIZE, BITS, K_SPLIT, RESULTS_PER_SIMDGROUP, INPUT_ALIGNED>::accumulate(
- result,
- b,
- scales,
- zero_points,
- biases,
- a,
- gather_indices,
- gathered,
- in_vec_size,
- out_vec_size,
- tile.out_row,
- batch_idx,
- simd_lane,
- tile.k_slice,
- signed_codes
- );
+ using Ops = GemvOperands<AT, BT, DT>;
+ const Ops ops = {b, scales, zero_points, biases, a, d, output_bias, hadamard_factors, gather_indices};
+ const GemvParams params =
+ {in_vec_size, out_vec_size, batch_size, ab_scale, soft_cap, output_transform, gathered, signed_codes};
+ dispatch_bool(full_tile, [&](auto full_tile_constant) {
+ constexpr bool FullTile = decltype(full_tile_constant)::value;
+ using Tile = GemvTile<INPUT_ROW_TILE, OUTPUT_ROW_TILE, REDUCTION_LANES, GROUP_LANES, NUM_SIMDGROUPS, K_SPLIT>;
+ const OutputTile<Tile, FullTile> tile =
+ OutputTile<Tile, FullTile>::make(output_tile_idx, input_tile_idx, simd_group, simd_lane, out_vec_size);
+ thread float result[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE] = {{0}};
- Reduce<U, K_SPLIT, NUM_SIMDGROUPS, RESULTS_PER_SIMDGROUP>::run(
- result,
- shared_results,
- simd_group,
- simd_lane,
- tile.row_group,
- tile.k_slice
- );
+ if constexpr (BITS == 0) {
+ FullPrecisionBSource<Tile, AT, BT, DT, INPUT_ALIGNED, FullTile>::accumulate(result, ops, params, tile);
+ } else {
+ QuantBSource<Tile, AT, BT, DT, B_PROLOGUE, GROUP_SIZE, BITS, INPUT_ALIGNED, FullTile>::accumulate(
+ result,
+ ops,
+ params,
+ tile
+ );
+ }
- Epilogue<BT, DT, U, RESULTS_PER_SIMDGROUP>::store(
- result,
- d,
- output_bias,
- hadamard_factors,
- shared_results,
- ab_scale,
- soft_cap,
- output_transform,
- tile.out_row,
- out_vec_size,
- out_block_idx,
- simd_group,
- simd_lane,
- tile.writer
- );
+ Reduce<Tile, FullTile>::run(result, shared_results, tile);
+ Epilogue<Tile, AT, BT, DT, FullTile>::store(result, ops, params, tile, shared_results);
+ });
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index f9326cf43..56ceb4e6a 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -1,56 +1,97 @@
-use std::{
- collections::{HashMap, hash_map::Entry},
- sync::OnceLock,
-};
-
use super::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK};
use crate::{
backends::{
common::{
- Allocation, BufferArg, Encoder,
gpu_types::{
HADAMARD_TRANSFORM_BLOCK_SIZE,
gemm::{GemmBPrologueKind, GemmDTransform},
},
- kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulShape},
+ kernel::matmul::MatmulShape,
},
- metal::{Metal, context::MetalContext, device_profile::DeviceProfile, kernel::GemvMetalKernel},
+ metal::{context::MetalContext, device_profile::DeviceProfile, error::MetalError, kernel::GemvMetalKernel},
},
data_type::DataType,
};
-const DEFAULT_GEMV_MAX_BATCH: u32 = 8;
-static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();
-
-fn max_gemv_batch_threshold() -> u32 {
- *GEMV_MAX_BATCH.get_or_init(|| {
- // TODO: remove magic env var
- std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)
- })
-}
+const GEMV_MAX_BATCH: u32 = 8;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub(crate) struct GemvSpecialization {
+pub struct GemvSpecialization {
b_prologue: GemmBPrologueKind,
group_size: u32,
bits: u32,
output_transform: GemmDTransform,
input_aligned: bool,
k_split: u32,
- results_per_simdgroup: u32,
+ output_row_tile: u32,
num_simdgroups: u32,
+ input_row_tile: u32,
+ reduction_lanes: u32,
+ group_lanes: u32,
gathered: bool,
signed_codes: bool,
+ full_tile: bool,
}
impl GemvSpecialization {
- pub(crate) fn select_shape(
+ #[cfg(test)]
+ pub fn tile(self) -> policy::GemvTile {
+ policy::GemvTile {
+ num_simdgroups: self.num_simdgroups,
+ k_split: self.k_split,
+ results_per_simdgroup: self.output_row_tile / (self.num_simdgroups / self.k_split),
+ input_row_tile: self.input_row_tile,
+ reduction_lanes: self.reduction_lanes,
+ group_lanes: self.group_lanes,
+ }
+ }
+
+ pub fn select_shape(
shape: &MatmulShape,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
device_profile: DeviceProfile,
- ) -> Option<GemvSpecialization> {
+ ) -> Option<Self> {
+ let is_quant = shape.is_quant();
+ let bits = shape.b_bits.unwrap_or(0);
+ let bf16_io = input_data_type == DataType::BF16 && output_data_type == DataType::BF16;
+ let tile = if is_quant && shape.gathered {
+ policy::gathered_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n)
+ } else if is_quant {
+ policy::quantized_tile(
+ device_profile,
+ bits,
+ shape.b_group_size.unwrap_or(0),
+ shape.m,
+ shape.n,
+ shape.k,
+ shape.d_transform,
+ bf16_io,
+ )
+ } else {
+ let mixed_precision = weights_data_type == DataType::F32
+ && (input_data_type != DataType::F32 || output_data_type != DataType::F32);
+ if mixed_precision || shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {
+ return None;
+ }
+ let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);
+ if shape.d_transform.contains(GemmDTransform::RHT) {
+ Some(policy::DEFAULT_TILE)
+ } else {
+ Some(policy::fp_tile(shape.m, shape.n, shape.k, input_aligned, device_profile))
+ }
+ };
+ Self::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile?)
+ }
+
+ pub fn select_tile(
+ shape: &MatmulShape,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ tile: policy::GemvTile,
+ ) -> Option<Self> {
if !shape.b_transpose || !shape.a_full_precision {
return None;
}
@@ -63,24 +104,20 @@ impl GemvSpecialization {
if bad_leading_dimension {
return None;
}
- if shape.d_transform.contains(GemmDTransform::ACCUMULATE) && !shape.n.is_multiple_of(32) {
- return None;
- }
if shape.d_transform.contains(GemmDTransform::RHT) && !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE) {
return None;
}
- if shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > max_gemv_batch_threshold() {
+ if shape.d_transform.contains(GemmDTransform::ACCUMULATE) && !shape.n.is_multiple_of(32) {
return None;
}
+ let bits = shape.b_bits.unwrap_or(0);
if !is_quant {
let mixed_precision = weights_data_type == DataType::F32
&& (input_data_type != DataType::F32 || output_data_type != DataType::F32);
- if mixed_precision {
+ if mixed_precision || shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {
return None;
}
}
-
- let bits = shape.b_bits.unwrap_or(0);
let block_size = if !is_quant {
FP_K_BLOCK
} else if bits == 4 {
@@ -89,49 +126,90 @@ impl GemvSpecialization {
256
};
let input_aligned = shape.k.is_multiple_of(block_size);
- let has_rht = shape.d_transform.contains(GemmDTransform::RHT);
- let bf16_io = input_data_type == DataType::BF16 && output_data_type == DataType::BF16;
- let tile = if is_quant && bf16_io {
- policy::quant_tile(shape.m, shape.n, shape.k, bits, has_rht, device_profile)
- } else if is_quant || has_rht {
- // Non-bf16 quant IO and fp+RHT keep the default tile (the only
- // one instantiated for those modes).
- policy::DEFAULT_TILE
- } else {
- policy::fp_tile(shape.m, shape.n, shape.k, input_aligned, device_profile)
- };
- Some(Self {
+ // Gathered quantized rows cannot share one input tile.
+ if is_quant && shape.gathered && tile.input_row_tile > 1 {
+ return None;
+ }
+ let specialization = Self {
b_prologue: shape.b_prologue,
group_size: shape.b_group_size.unwrap_or(0),
bits,
output_transform: shape.d_transform,
input_aligned,
k_split: tile.k_split,
- results_per_simdgroup: tile.results_per_simdgroup,
+ output_row_tile: tile.output_row_tile(),
num_simdgroups: tile.num_simdgroups,
+ input_row_tile: tile.input_row_tile,
+ reduction_lanes: tile.reduction_lanes,
+ group_lanes: tile.group_lanes,
gathered: shape.gathered,
signed_codes: shape.signed_codes,
- })
+ full_tile: full_tile(shape, tile),
+ };
+ Some(specialization)
+ }
+
+ pub fn output_row_tile(&self) -> u32 {
+ self.output_row_tile
+ }
+
+ fn create_pipeline(
+ &self,
+ context: &MetalContext,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ ) -> Result<GemvMetalKernel, MetalError> {
+ GemvMetalKernel::new(
+ context,
+ input_data_type,
+ weights_data_type,
+ output_data_type,
+ self.b_prologue,
+ self.group_size,
+ self.bits,
+ self.k_split,
+ self.input_aligned,
+ self.input_row_tile,
+ self.output_row_tile(),
+ self.reduction_lanes,
+ self.group_lanes,
+ self.num_simdgroups,
+ self.output_transform,
+ self.gathered,
+ self.signed_codes,
+ self.full_tile,
+ )
}
}
-fn rows_per_threadgroup(
- k_split: u32,
- results_per_simdgroup: u32,
- num_simdgroups: u32,
-) -> u32 {
- (num_simdgroups / k_split) * results_per_simdgroup
+fn full_tile(
+ shape: &MatmulShape,
+ tile: policy::GemvTile,
+) -> bool {
+ shape.m.is_multiple_of(tile.input_row_tile) && shape.n.is_multiple_of(tile.output_row_tile())
}
-pub(crate) struct GemvDispatch {
+use std::collections::{HashMap, hash_map::Entry};
+
+use crate::backends::{
+ common::{
+ BufferArg, Encoder,
+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError},
+ },
+ metal::Metal,
+};
+
+/// GEMV pipelines compiled on first use.
+pub struct GemvKernel {
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
pipelines: HashMap<GemvSpecialization, GemvMetalKernel>,
}
-impl GemvDispatch {
- pub(crate) fn new(
+impl GemvKernel {
+ pub fn new(
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
@@ -152,29 +230,15 @@ impl GemvDispatch {
match self.pipelines.entry(specialization) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
- let kernel = GemvMetalKernel::new(
- context,
- self.input_data_type,
- self.weights_data_type,
- self.output_data_type,
- specialization.b_prologue,
- specialization.group_size,
- specialization.bits,
- specialization.k_split,
- specialization.input_aligned,
- specialization.results_per_simdgroup,
- specialization.num_simdgroups,
- specialization.output_transform,
- specialization.gathered,
- specialization.signed_codes,
- )
- .map_err(MatmulError::BackendError)?;
+ let kernel = specialization
+ .create_pipeline(context, self.weights_data_type, self.input_data_type, self.output_data_type)
+ .map_err(MatmulError::BackendError)?;
Ok(entry.insert(kernel))
},
}
}
- pub(crate) fn encode<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(
+ pub fn encode<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(
&mut self,
arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,
specialization: GemvSpecialization,
@@ -206,88 +270,80 @@ impl GemvDispatch {
});
};
- let group_count_x = n.div_ceil(rows_per_threadgroup(
- specialization.k_split,
- specialization.results_per_simdgroup,
- specialization.num_simdgroups,
- ));
+ // Preserve each weight buffer's residency range.
+ let (scales, zero_points, biases) = match &b {
+ MatmulB::FullPrecision {
+ ..
+ } => (None, None, None),
+ MatmulB::ScaleBiasDequant {
+ scales,
+ biases,
+ ..
+ } => (Some(*scales), None, Some(*biases)),
+ MatmulB::ScaleZeroPointDequant {
+ scales,
+ zero_points,
+ ..
+ } => (Some(*scales), Some(*zero_points), None),
+ MatmulB::ScaleSymmetricDequant {
+ scales,
+ ..
+ } => (Some(*scales), None, None),
+ };
+ let output_group_count = n.div_ceil(specialization.output_row_tile());
let context = encoder.context();
let pipeline = self.get_or_create(context, specialization)?;
-
match b {
MatmulB::FullPrecision {
b: weights,
- } => {
- pipeline.encode(
- weights,
- None::<&Allocation<Metal>>,
- None::<&Allocation<Metal>>,
- None::<&Allocation<Metal>>,
- (a, a_offset),
- &mut *d,
- output_bias,
- rht_factors,
- gather_indices,
- k,
- n,
- m,
- ab_scale,
- group_count_x,
- soft_cap,
- encoder,
- );
- },
- quant_b @ (MatmulB::ScaleBiasDequant {
+ } => pipeline.encode(
+ weights,
+ scales,
+ zero_points,
+ biases,
+ (a, a_offset),
+ &mut *d,
+ output_bias,
+ rht_factors,
+ gather_indices,
+ k,
+ n,
+ m,
+ ab_scale,
+ output_group_count,
+ soft_cap,
+ encoder,
+ ),
+ MatmulB::ScaleBiasDequant {
+ b: weights,
..
}
| MatmulB::ScaleZeroPointDequant {
+ b: weights,
..
}
| MatmulB::ScaleSymmetricDequant {
+ b: weights,
..
- }) => {
- let (weights, scales, zero_points, biases) = match quant_b {
- MatmulB::ScaleBiasDequant {
- b: w,
- scales,
- biases,
- ..
- } => (w, scales, None, Some(biases)),
- MatmulB::ScaleZeroPointDequant {
- b: w,
- scales,
- zero_points,
- ..
- } => (w, scales, Some(zero_points), None),
- MatmulB::ScaleSymmetricDequant {
- b: w,
- scales,
- ..
- } => (w, scales, None, None),
- MatmulB::FullPrecision {
- ..
- } => unreachable!(),
- };
- pipeline.encode(
- weights,
- Some(scales),
- zero_points,
- biases,
- (a, a_offset),
- &mut *d,
- output_bias,
- rht_factors,
- gather_indices,
- k,
- n,
- m,
- ab_scale,
- group_count_x,
- soft_cap,
- encoder,
- );
- },
+ } => pipeline.encode(
+ weights,
+ scales,
+ zero_points,
+ biases,
+ (a, a_offset),
+ &mut *d,
+ output_bias,
+ rht_factors,
+ gather_indices,
+ k,
+ n,
+ m,
+ ab_scale,
+ output_group_count,
+ soft_cap,
+ encoder,
+ ),
}
Ok(())
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/mod.rs
index 7f9bc76ef..8a71486a7 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/mod.rs
@@ -1,4 +1,5 @@
mod kernel;
mod policy;
-pub(crate) use kernel::{GemvDispatch, GemvSpecialization};
+pub(super) use kernel::{GemvKernel, GemvSpecialization};
+pub(super) use policy::GemvTile;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
index 43d33ba89..c410c67ed 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
@@ -1,24 +1,33 @@
-use crate::backends::metal::device_profile::{DeviceGeneration, DeviceProfile, DeviceSize};
+use crate::backends::metal::device_profile::{DeviceProfile, DeviceSize, GpuFamily};
+
+mod quantized;
+
+pub(super) use quantized::{gathered_tile, select as quantized_tile};
// Full-precision GEMV accumulates four K values per SIMD lane, so one full
// vectorized K block is 4 * 32 lanes.
-pub(crate) const FP_K_BLOCK: u32 = 128;
-pub(crate) const DEFAULT_RESULTS_PER_SIMDGROUP: u32 = 4;
-pub(crate) const DEFAULT_NUM_SIMDGROUPS: u32 = 8;
+pub(super) const FP_K_BLOCK: u32 = 128;
+pub(super) const DEFAULT_RESULTS_PER_SIMDGROUP: u32 = 4;
+pub(super) const DEFAULT_NUM_SIMDGROUPS: u32 = 8;
-/// Tile specialization selected for one GEMV dispatch.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(crate) struct GemvTile {
+pub struct GemvTile {
/// SIMD groups launched in one threadgroup.
- pub num_simdgroups: u32,
+ pub(super) num_simdgroups: u32,
/// Number of split-K slices reduced by one threadgroup.
- pub k_split: u32,
+ pub(super) k_split: u32,
/// Output rows computed by each SIMD group.
- pub results_per_simdgroup: u32,
+ pub(super) results_per_simdgroup: u32,
+ /// Input rows processed by one threadgroup.
+ pub(super) input_row_tile: u32,
+ /// SIMD lanes cooperating over K for one output-row block.
+ pub(super) reduction_lanes: u32,
+ /// SIMD lanes cooperating on one quantization group.
+ pub(super) group_lanes: u32,
}
const SMALL_G13_HUGE_N: u32 = 32768;
-const SMALL_G13_WIDE_ROW_N: u32 = 6144;
+const SMALL_G13_BROAD_ROW_N: u32 = 6144;
const DEEP_K: u32 = 8192;
const FP_LARGE_SPLIT_K_MIN_DEPTH: u32 = 4 * FP_K_BLOCK;
const FP_K_DEPTH_N_MAX: u32 = 4095;
@@ -34,37 +43,50 @@ const fn tile(
num_simdgroups,
k_split,
results_per_simdgroup,
+ input_row_tile: 1,
+ reduction_lanes: 32,
+ group_lanes: 1,
}
}
-const fn qtile(
- num_simdgroups: u32,
- results_per_simdgroup: u32,
-) -> GemvTile {
- // Q4 table sweeps only covered KS1; quant split-K is a separate future grid.
- tile(num_simdgroups, 1, results_per_simdgroup)
-}
+pub(super) const DEFAULT_TILE: GemvTile = tile(DEFAULT_NUM_SIMDGROUPS, 1, DEFAULT_RESULTS_PER_SIMDGROUP);
+
+impl GemvTile {
+ pub const fn quantized(
+ num_simdgroups: u32,
+ results_per_simdgroup: u32,
+ input_row_tile: u32,
+ reduction_lanes: u32,
+ group_lanes: u32,
+ ) -> Self {
+ Self {
+ num_simdgroups,
+ k_split: 1,
+ results_per_simdgroup,
+ input_row_tile,
+ reduction_lanes,
+ group_lanes,
+ }
+ }
-pub(crate) const DEFAULT_TILE: GemvTile = qtile(DEFAULT_NUM_SIMDGROUPS, DEFAULT_RESULTS_PER_SIMDGROUP);
-// Qxy = qtile(num_simdgroups=x, results_per_simdgroup=y), with KS1.
-const Q21: GemvTile = qtile(2, 1);
-const Q22: GemvTile = qtile(2, 2);
-const Q24: GemvTile = qtile(2, 4);
-const Q42: GemvTile = qtile(4, 2);
-const Q44: GemvTile = qtile(4, 4);
-const Q48: GemvTile = qtile(4, 8);
-const Q82: GemvTile = qtile(8, 2);
-const QUANT_N_BUCKET_MAXES: [u32; 6] = [512, 2048, 4096, 8192, 16384, 32768];
-const QUANT_K_BUCKET_MAXES: [u32; 3] = [512, 2048, 8192];
-const QUANT_RHT_TUNED_N_MIN_EXCLUSIVE: u32 = 2048;
-const QUANT_RHT_TUNED_N_MAX: u32 = 4096;
-const QUANT_RHT_TUNED_K_MIN: u32 = 2048;
-
-fn table_bucket_index(
- value: u32,
- bucket_maxes: &[u32],
-) -> usize {
- bucket_maxes.partition_point(|&max| value > max)
+ pub const fn quantized_output_tile(
+ num_simdgroups: u32,
+ output_row_tile: u32,
+ input_row_tile: u32,
+ reduction_lanes: u32,
+ group_lanes: u32,
+ ) -> Self {
+ assert!(output_row_tile.is_multiple_of(num_simdgroups));
+ Self::quantized(num_simdgroups, output_row_tile / num_simdgroups, input_row_tile, reduction_lanes, group_lanes)
+ }
+
+ pub(super) const fn output_row_tile(self) -> u32 {
+ (self.num_simdgroups / self.k_split) * self.results_per_simdgroup
+ }
+
+ pub(super) const fn rows_per_lane(self) -> u32 {
+ self.results_per_simdgroup / (32 / self.reduction_lanes)
+ }
}
fn cap_k_split_to_complete_fp_k_blocks(
@@ -119,7 +141,7 @@ fn preferred_fp_k_split(
/// Selects the full-precision GEMV tile. `m` is the input-vector count,
/// `n` is the output row count, and `k` is the reduction depth.
-pub(crate) fn fp_tile(
+pub(super) fn fp_tile(
m: u32,
n: u32,
k: u32,
@@ -127,12 +149,12 @@ pub(crate) fn fp_tile(
profile: DeviceProfile,
) -> GemvTile {
let size = profile.size();
- let is_small_g13 = size == DeviceSize::Small && profile.generation() == DeviceGeneration::Legacy;
- // FP sweeps covered SG2/SG4/SG8; SG changes did not produce portable
- // confirmed wins, so shipped FP policy keeps SG8 and tunes KS/R only.
+ let gpu_family = profile.gpu_family();
+ let is_small_legacy = size == DeviceSize::Small && gpu_family == GpuFamily::Legacy;
+ // SG8 is the portable full-precision geometry.
let should_disable_k_split = !input_aligned
|| (m == 1 && size == DeviceSize::Large && k < FP_LARGE_SPLIT_K_MIN_DEPTH)
- || (m == 1 && is_small_g13 && n >= SMALL_G13_HUGE_N);
+ || (m == 1 && is_small_legacy && n >= SMALL_G13_HUGE_N);
let k_split = if should_disable_k_split {
1
@@ -140,9 +162,7 @@ pub(crate) fn fp_tile(
cap_k_split_to_complete_fp_k_blocks(k, preferred_fp_k_split(m, n, k))
};
- // R1 won most single-row FP sweeps; Large devices only switch back to R4
- // for deep-K rows, while legacy wide rows keep R4.
- let results_per_simdgroup = if is_small_g13 && m == 1 && n >= SMALL_G13_WIDE_ROW_N {
+ let results_per_simdgroup = if is_small_legacy && m == 1 && n >= SMALL_G13_BROAD_ROW_N {
DEFAULT_RESULTS_PER_SIMDGROUP
} else if m == 1 && (k <= DEEP_K || size != DeviceSize::Large) {
1
@@ -153,77 +173,6 @@ pub(crate) fn fp_tile(
tile(DEFAULT_NUM_SIMDGROUPS, k_split, results_per_simdgroup)
}
-/// Selects the quantized GEMV tile. `m` is the input-vector count, `n` is the
-/// output row count, `k` is the reduction depth, and `bits` is the quant width.
-pub(crate) fn quant_tile(
- m: u32,
- n: u32,
- k: u32,
- bits: u32,
- has_rht: bool,
- profile: DeviceProfile,
-) -> GemvTile {
- let size = profile.size();
- let generation = profile.generation();
- // These tables are fitted for batch-1 Q4 only; Q8/future widths keep the
- // deterministic default until they have their own cold sweep.
- if m != 1 || bits != 4 {
- return DEFAULT_TILE;
- }
- if has_rht {
- // This special case mirrors quant bucket edges: n in (2048, 4096]
- // and k at or above the 2048 boundary.
- return if size == DeviceSize::Large
- && n > QUANT_RHT_TUNED_N_MIN_EXCLUSIVE
- && n <= QUANT_RHT_TUNED_N_MAX
- && k >= QUANT_RHT_TUNED_K_MIN
- {
- qtile(4, 8)
- } else {
- DEFAULT_TILE
- };
- }
-
- let k_bucket = table_bucket_index(k, &QUANT_K_BUCKET_MAXES);
- let n_bucket = table_bucket_index(n, &QUANT_N_BUCKET_MAXES);
- // Q4 BF16 decode choices from June 2026 gemv_fine_tune sweeps; omitted
- // cells keep SG8_KS1_R4. Other quant widths keep DEFAULT_TILE until swept.
- let selected = match (size, generation, k_bucket, n_bucket) {
- (DeviceSize::Large, _, 0, 1) => Q42,
- (DeviceSize::Large, _, 1, 0) => Q21,
- (DeviceSize::Large, _, 1, 1..=3) => Q22,
- (DeviceSize::Large, _, 1, 4) => Q21,
- (DeviceSize::Large, _, 1, 5) => Q24,
- (DeviceSize::Large, _, 2, 1) => Q42,
- (DeviceSize::Large, _, 3, 1) => Q22,
-
- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 0, 1) => Q44,
- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 1, 0) => Q42,
- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 1, 1) => Q22,
- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 1, 2) => Q42,
- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 1, 4) => Q22,
- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 1, 5) => Q42,
- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 2, 1) => Q42,
- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 3, 1) => Q22,
-
- (DeviceSize::Small, DeviceGeneration::Apple8, 0, 1) => Q44,
- (DeviceSize::Small, DeviceGeneration::Apple8, 1, _) | (DeviceSize::Small, DeviceGeneration::Apple8, 2, 1) => {
- Q82
- },
-
- (DeviceSize::Small, DeviceGeneration::Legacy, 0, 1) => Q48,
- (DeviceSize::Small, DeviceGeneration::Legacy, 1, 0..=3) => Q82,
-
- _ => DEFAULT_TILE,
- };
- if n < selected.results_per_simdgroup {
- // Coarse N buckets can include n < R; keep the default R4 tile for tiny rows.
- DEFAULT_TILE
- } else {
- selected
- }
-}
-
#[cfg(test)]
#[path = "../../../../../../unit/backends/metal/kernel/matmul/gemv/policy_test.rs"]
mod tests;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
new file mode 100644
index 000000000..56bafba17
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
@@ -0,0 +1,148 @@
+use super::{DEFAULT_RESULTS_PER_SIMDGROUP, GemvTile};
+use crate::backends::{
+ common::gpu_types::gemm::GemmDTransform,
+ metal::device_profile::{DeviceProfile, DeviceSize, GpuFamily},
+};
+
+const QUANT_N_BUCKET_MAXES: [u32; 6] = [512, 2048, 4096, 8192, 16384, 32768];
+const QUANT_K_BUCKET_MAXES: [u32; 3] = [512, 2048, 8192];
+const QUANT_RHT_TUNED_N_MIN_EXCLUSIVE: u32 = 2048;
+const QUANT_RHT_TUNED_N_MAX: u32 = 4096;
+const QUANT_RHT_TUNED_K_MIN: u32 = 2048;
+
+/// Lane-sliced tiles split each quantization group across reduction lanes.
+/// A lane stages 8 bytes: 16 W4 values or 8 W8 values.
+const fn lane_tile(
+ num_simdgroups: u32,
+ results_per_simdgroup: u32,
+ bits: u32,
+ group: u32,
+) -> GemvTile {
+ let pack_factor = match bits {
+ 4 => 8,
+ 8 => 4,
+ _ => 1,
+ };
+ GemvTile::quantized(num_simdgroups, results_per_simdgroup, 1, 32, group / (2 * pack_factor))
+}
+
+const fn lane_default(
+ bits: u32,
+ group: u32,
+) -> GemvTile {
+ lane_tile(8, DEFAULT_RESULTS_PER_SIMDGROUP, bits, group)
+}
+
+pub fn gathered_tile(
+ bits: u32,
+ group: u32,
+ m: u32,
+ n: u32,
+) -> Option<GemvTile> {
+ if !matches!(bits, 4 | 8) || !matches!(group, 16 | 32 | 64 | 128) || !(1..=8).contains(&m) {
+ return None;
+ }
+ let tile = lane_default(bits, group);
+ (n >= tile.rows_per_lane()).then_some(tile)
+}
+
+fn table_bucket_index(
+ value: u32,
+ bucket_maxes: &[u32],
+) -> usize {
+ bucket_maxes.partition_point(|&max| value > max)
+}
+
+fn lane_policy(
+ m: u32,
+ n: u32,
+ k: u32,
+ bits: u32,
+ group: u32,
+ has_rht: bool,
+ profile: DeviceProfile,
+) -> GemvTile {
+ let size = profile.size();
+ let gpu_family = profile.gpu_family();
+ if m != 1 || bits != 4 {
+ return lane_default(bits, group);
+ }
+ if has_rht {
+ return if size == DeviceSize::Large
+ && n > QUANT_RHT_TUNED_N_MIN_EXCLUSIVE
+ && n <= QUANT_RHT_TUNED_N_MAX
+ && k >= QUANT_RHT_TUNED_K_MIN
+ {
+ lane_tile(4, 8, bits, group)
+ } else {
+ lane_default(bits, group)
+ };
+ }
+
+ let k_bucket = table_bucket_index(k, &QUANT_K_BUCKET_MAXES);
+ let n_bucket = table_bucket_index(n, &QUANT_N_BUCKET_MAXES);
+ let (num_simdgroups, results_per_simdgroup) = match (size, gpu_family, k_bucket, n_bucket) {
+ (DeviceSize::Large, _, 0, 1) => (4, 2),
+ (DeviceSize::Large, _, 1, 0) => (2, 1),
+ (DeviceSize::Large, _, 1, 1..=3) => (2, 2),
+ (DeviceSize::Large, _, 1, 4) => (2, 1),
+ (DeviceSize::Large, _, 1, 5) => (2, 4),
+ (DeviceSize::Large, _, 2, 1) => (4, 2),
+ (DeviceSize::Large, _, 3, 1) => (2, 2),
+ (DeviceSize::Small, GpuFamily::Apple9, 0, 1) => (4, 4),
+ (DeviceSize::Small, GpuFamily::Apple9, 1, 0) => (4, 2),
+ (DeviceSize::Small, GpuFamily::Apple9, 1, 1) => (2, 2),
+ (DeviceSize::Small, GpuFamily::Apple9, 1, 2) => (4, 2),
+ (DeviceSize::Small, GpuFamily::Apple9, 1, 4) => (2, 2),
+ (DeviceSize::Small, GpuFamily::Apple9, 1, 5) => (4, 2),
+ (DeviceSize::Small, GpuFamily::Apple9, 2, 1) => (4, 2),
+ (DeviceSize::Small, GpuFamily::Apple9, 3, 1) => (2, 2),
+ (DeviceSize::Small, GpuFamily::Apple8, 0, 1) => (4, 4),
+ (DeviceSize::Small, GpuFamily::Apple8, 1, _) | (DeviceSize::Small, GpuFamily::Apple8, 2, 1) => (8, 2),
+ (DeviceSize::Small, GpuFamily::Legacy, 0, 1) => (4, 8),
+ (DeviceSize::Small, GpuFamily::Legacy, 1, 0..=3) => (8, 2),
+ _ => (8, 4),
+ };
+ let selected = lane_tile(num_simdgroups, results_per_simdgroup, bits, group);
+ if n < selected.results_per_simdgroup {
+ lane_default(bits, group)
+ } else {
+ selected
+ }
+}
+
+pub fn select(
+ profile: DeviceProfile,
+ bits: u32,
+ group: u32,
+ m: u32,
+ n: u32,
+ k: u32,
+ d_transform: GemmDTransform,
+ bf16_io: bool,
+) -> Option<GemvTile> {
+ if !matches!(bits, 4 | 8) || !matches!(group, 16 | 32 | 64 | 128) || !(1..=8).contains(&m) {
+ return None;
+ }
+
+ if !bf16_io {
+ let tile = lane_default(bits, group);
+ return (m <= 4 && n >= tile.rows_per_lane()).then_some(tile);
+ }
+
+ let has_rht = d_transform.contains(GemmDTransform::RHT);
+ let tile = match m {
+ 1..=4 => lane_policy(m, n, k, bits, group, has_rht, profile),
+ 5..=8 if n <= 64 => lane_default(bits, group),
+ _ => return None,
+ };
+ let rows = tile.output_row_tile();
+ // Input rows are tiled independently; partial tiles clamp loads and stores.
+ if n < tile.rows_per_lane() {
+ return None;
+ }
+ if has_rht && (!rows.is_multiple_of(32) || !n.is_multiple_of(rows)) {
+ return None;
+ }
+ Some(tile)
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
index e90e9346c..2139f6cca 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
@@ -1,10 +1,12 @@
pub mod gemm;
-pub mod gemv;
+mod gemv;
+mod qmv;
pub use self::gemm::GemmKernel;
use self::{
gemm::{GemmPlan, GemmProblem},
- gemv::{GemvDispatch, GemvSpecialization},
+ gemv::{GemvKernel, GemvSpecialization},
+ qmv::QmvRoute,
};
use crate::{
backends::{
@@ -22,7 +24,7 @@ use crate::{
};
pub struct MatmulMetalKernel {
- gemv: GemvDispatch,
+ gemv: GemvKernel,
pub gemm: GemmKernel,
weights_data_type: DataType,
input_data_type: DataType,
@@ -64,42 +66,55 @@ impl MatmulMetalKernel {
matches!(plan.tiling, GemmTiling::Tile16x32x256_Simdgroups1x1 | GemmTiling::Tile16x128x256_Simdgroups1x4)
}
- fn select_dispatch(
- &self,
+ fn choose_dispatch(
shape: &MatmulShape,
- context: &MetalContext,
+ profile: crate::backends::metal::device_profile::DeviceProfile,
+ supports_mxu: bool,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
) -> MatmulDispatch {
- let gemv = GemvSpecialization::select_shape(
- shape,
- self.weights_data_type,
- self.input_data_type,
- self.output_data_type,
- context.device_profile(),
- );
- let problem = GemmProblem::new(
- *shape,
- self.weights_data_type,
- self.output_data_type,
- context.supports_mxu(),
- context.device_profile(),
- );
+ let all_bf16 = weights_data_type == DataType::BF16
+ && input_data_type == DataType::BF16
+ && output_data_type == DataType::BF16;
+ if let Some(route) = qmv::route(profile, shape, all_bf16) {
+ return match route {
+ QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(
+ GemvSpecialization::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile)
+ .expect("typed QMV route must contain a legal GEMV tile"),
+ ),
+ QmvRoute::MainGemm(plan) => MatmulDispatch::Gemm(plan),
+ };
+ }
+ let gemv =
+ GemvSpecialization::select_shape(shape, weights_data_type, input_data_type, output_data_type, profile);
+ let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, profile);
let plan = problem.select_plan();
match gemv {
None => MatmulDispatch::Gemm(plan),
Some(_)
- if Self::prefer_gemm_over_gemv(
- *shape,
- plan,
- self.weights_data_type,
- self.input_data_type,
- self.output_data_type,
- ) =>
+ if Self::prefer_gemm_over_gemv(*shape, plan, weights_data_type, input_data_type, output_data_type) =>
{
MatmulDispatch::Gemm(plan)
},
Some(gemv) => MatmulDispatch::Gemv(gemv),
}
}
+
+ fn select_dispatch(
+ &self,
+ shape: &MatmulShape,
+ context: &MetalContext,
+ ) -> MatmulDispatch {
+ Self::choose_dispatch(
+ shape,
+ context.device_profile(),
+ context.supports_mxu(),
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
+ )
+ }
}
impl MatmulKernel for MatmulMetalKernel {
@@ -118,7 +133,7 @@ impl MatmulKernel for MatmulMetalKernel {
}
let gemm = GemmKernel::new(context, weights_data_type, input_data_type, output_data_type)?;
- let gemv = GemvDispatch::new(weights_data_type, input_data_type, output_data_type);
+ let gemv = GemvKernel::new(weights_data_type, input_data_type, output_data_type);
Ok(Self {
gemv,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rs
new file mode 100644
index 000000000..b75f223f9
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rs
@@ -0,0 +1,12 @@
+mod routes;
+
+pub(super) use routes::route;
+
+use super::{gemm::GemmPlan, gemv::GemvTile};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum QmvRoute {
+ Tuned(GemvTile),
+ MainGemv(GemvTile),
+ MainGemm(GemmPlan),
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs
new file mode 100644
index 000000000..2ace699b2
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs
@@ -0,0 +1,415 @@
+// Qwen3.6-27B W4/ZP and W8/Symmetric, G32/G64.
+// Tuned on M1, M2, M2 Pro, M3 Max, M4, M4 Pro, M5 Max.
+use super::{
+ super::{
+ gemm::{GemmEngine, GemmPlan},
+ gemv::GemvTile,
+ },
+ QmvRoute,
+};
+use crate::backends::{
+ common::{
+ gpu_types::gemm::{GemmBPrologueKind, GemmTiling},
+ kernel::matmul::MatmulShape,
+ },
+ metal::device_profile::{DeviceIdentity, DeviceProfile},
+};
+
+const DOWN: u8 = 1 << 0;
+const GATE: u8 = 1 << 1;
+const GATE_UP: u8 = 1 << 2;
+const PROJECTION_IN: u8 = 1 << 3;
+const PROJECTION_OUT: u8 = 1 << 4;
+const QKV: u8 = 1 << 5;
+const READOUT: u8 = 1 << 6;
+
+#[derive(Clone, Copy)]
+struct RouteRow {
+ identity: DeviceIdentity,
+ bits: u32,
+ group: u32,
+ m: u32,
+ shapes: u8,
+ route: QmvRoute,
+}
+
+const fn shape(
+ n: u32,
+ k: u32,
+) -> u8 {
+ match (n, k) {
+ (5120, 17408) => DOWN,
+ (6144, 5120) => GATE,
+ (34816, 5120) => GATE_UP,
+ (16480, 5120) => PROJECTION_IN,
+ (5120, 6144) => PROJECTION_OUT,
+ (8192, 5120) => QKV,
+ (248320, 5120) => READOUT,
+ _ => 0,
+ }
+}
+
+#[rustfmt::skip]
+macro_rules! tuned { ($input:literal, $output:literal, $lanes:literal, $group_lanes:literal, $simdgroups:literal) => { QmvRoute::Tuned(GemvTile::quantized_output_tile($simdgroups, $output, $input, $lanes, $group_lanes)) }; }
+#[rustfmt::skip]
+macro_rules! main_gemv { ($input:literal, $results_per_simdgroup:literal, $lanes:literal, $group_lanes:literal, $simdgroups:literal) => { QmvRoute::MainGemv(GemvTile::quantized($simdgroups, $results_per_simdgroup, $input, $lanes, $group_lanes)) }; }
+#[rustfmt::skip]
+macro_rules! main_gemm { ($engine:ident, $tiling:ident, $split:literal) => { QmvRoute::MainGemm(GemmPlan { engine: GemmEngine::$engine, tiling: GemmTiling::$tiling, split_k: $split }) }; }
+#[rustfmt::skip]
+macro_rules! qmv_format { (w4_zp_g32) => { (4, 32) }; (w4_zp_g64) => { (4, 64) }; (w8_sym_g32) => { (8, 32) }; (w8_sym_g64) => { (8, 64) }; }
+#[rustfmt::skip]
+macro_rules! row { ($identity:ident, $format:ident, $m:literal, $shapes:expr, $route:expr) => { RouteRow { identity: DeviceIdentity::$identity, bits: qmv_format!($format).0, group: qmv_format!($format).1, m: $m, shapes: $shapes, route: $route } }; }
+
+#[rustfmt::skip]
+const ROWS: &[RouteRow] = &[
+ row!(M1, w4_zp_g32, 2, READOUT, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M1, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(2, 16, 8, 1, 2)),
+ row!(M1, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M1, w4_zp_g32, 4, DOWN, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M1, w4_zp_g32, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M1, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M1, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),
+ row!(M1, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M1, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M1, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M1, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 16, 1, 4)),
+ row!(M1, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),
+ row!(M1, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),
+ row!(M1, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M1, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M1, w8_sym_g32, 3, PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g32, 4, GATE_UP, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M1, w8_sym_g32, 4, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g32, 6, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g32, 6, DOWN, tuned!(6, 16, 16, 1, 4)),
+ row!(M1, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M1, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M1, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g32, 2, PROJECTION_IN, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M2, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g32, 7, READOUT, tuned!(4, 8, 8, 1, 2)),
+ row!(M2, w4_zp_g32, 7, GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(7, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g32, 7, DOWN | GATE, tuned!(8, 8, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 2, PROJECTION_IN, tuned!(2, 8, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g32, 2, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M2, w8_sym_g32, 2, DOWN, tuned!(2, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M2, w8_sym_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 2, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M2, w8_sym_g64, 2, DOWN, tuned!(2, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M2, w8_sym_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g32, 2, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M2Pro, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g32, 3, DOWN | GATE_UP | READOUT, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M2Pro, w4_zp_g32, 3, PROJECTION_IN, tuned!(3, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g32, 3, GATE | PROJECTION_OUT | QKV, tuned!(3, 16, 16, 1, 4)),
+ row!(M2Pro, w4_zp_g32, 4, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M2Pro, w4_zp_g32, 4, PROJECTION_IN, tuned!(4, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g32, 4, GATE, tuned!(4, 16, 16, 1, 4)),
+ row!(M2Pro, w4_zp_g32, 5, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
+ row!(M2Pro, w4_zp_g32, 5, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M2Pro, w4_zp_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M2Pro, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g32, 6, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
+ row!(M2Pro, w4_zp_g32, 6, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M2Pro, w4_zp_g32, 6, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M2Pro, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g32, 7, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
+ row!(M2Pro, w4_zp_g32, 7, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M2Pro, w4_zp_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M2Pro, w4_zp_g32, 7, PROJECTION_IN, tuned!(7, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g64, 3, PROJECTION_OUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M2Pro, w4_zp_g64, 4, GATE_UP | PROJECTION_IN | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV, tuned!(4, 16, 16, 1, 4)),
+ row!(M2Pro, w4_zp_g64, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),
+ row!(M2Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M2Pro, w8_sym_g32, 3, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M2Pro, w8_sym_g32, 3, PROJECTION_IN, tuned!(3, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g32, 3, GATE, tuned!(3, 16, 16, 1, 4)),
+ row!(M2Pro, w8_sym_g32, 4, DOWN | GATE_UP | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M2Pro, w8_sym_g32, 4, PROJECTION_IN, tuned!(4, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g32, 4, GATE | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),
+ row!(M2Pro, w8_sym_g32, 5, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
+ row!(M2Pro, w8_sym_g32, 5, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M2Pro, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M2Pro, w8_sym_g32, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g32, 6, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
+ row!(M2Pro, w8_sym_g32, 6, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M2Pro, w8_sym_g32, 6, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M2Pro, w8_sym_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
+ row!(M2Pro, w8_sym_g32, 7, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M2Pro, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M2Pro, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M2Pro, w8_sym_g64, 3, DOWN | GATE | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M2Pro, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g64, 4, DOWN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M2Pro, w8_sym_g64, 4, GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M3Max, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M3Max, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M3Max, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M3Max, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
+ row!(M3Max, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),
+ row!(M3Max, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),
+ row!(M3Max, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g64, 2, GATE_UP | PROJECTION_IN | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M3Max, w4_zp_g64, 2, DOWN | GATE | PROJECTION_OUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g64, 3, DOWN | GATE | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g64, 3, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M3Max, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g64, 4, GATE_UP | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
+ row!(M3Max, w4_zp_g64, 5, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g64, 5, GATE_UP, tuned!(5, 16, 16, 1, 4)),
+ row!(M3Max, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M3Max, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M3Max, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M3Max, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),
+ row!(M3Max, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M3Max, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M3Max, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),
+ row!(M3Max, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M3Max, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),
+ row!(M3Max, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M3Max, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M3Max, w8_sym_g64, 3, GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M3Max, w8_sym_g64, 3, DOWN | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g64, 4, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M3Max, w8_sym_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M4, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M4, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M4, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M4, w4_zp_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g64, 4, DOWN | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g64, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g64, 5, DOWN | GATE_UP | PROJECTION_IN | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g64, 5, GATE | PROJECTION_OUT | QKV, tuned!(5, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M4, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M4, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M4, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),
+ row!(M4, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M4, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M4, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),
+ row!(M4, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M4, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),
+ row!(M4, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M4, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M4, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M4, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M4, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M4Pro, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M4Pro, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M4Pro, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
+ row!(M4Pro, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
+ row!(M4Pro, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),
+ row!(M4Pro, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),
+ row!(M4Pro, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M4Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M4Pro, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g64, 4, PROJECTION_IN | QKV, tuned!(4, 16, 16, 1, 4)),
+ row!(M4Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g64, 5, PROJECTION_IN | QKV, tuned!(5, 16, 16, 1, 4)),
+ row!(M4Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M4Pro, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M4Pro, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M4Pro, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),
+ row!(M4Pro, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M4Pro, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M4Pro, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),
+ row!(M4Pro, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
+ row!(M4Pro, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),
+ row!(M4Pro, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
+ row!(M4Pro, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M4Pro, w8_sym_g64, 2, READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M4Pro, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g32, 2, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(2, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g32, 2, GATE_UP | READOUT, tuned!(2, 16, 16, 1, 4)),
+ row!(M5Max, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g32, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
+ row!(M5Max, w4_zp_g32, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
+ row!(M5Max, w4_zp_g32, 6, GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),
+ row!(M5Max, w4_zp_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
+ row!(M5Max, w4_zp_g32, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
+ row!(M5Max, w4_zp_g32, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
+ row!(M5Max, w4_zp_g32, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
+ row!(M5Max, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g64, 6, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
+ row!(M5Max, w4_zp_g64, 6, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
+ row!(M5Max, w4_zp_g64, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
+ row!(M5Max, w4_zp_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
+ row!(M5Max, w4_zp_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
+ row!(M5Max, w4_zp_g64, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
+ row!(M5Max, w4_zp_g64, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
+ row!(M5Max, w4_zp_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
+ row!(M5Max, w8_sym_g32, 2, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M5Max, w8_sym_g32, 2, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g32, 4, PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),
+ row!(M5Max, w8_sym_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
+ row!(M5Max, w8_sym_g32, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
+ row!(M5Max, w8_sym_g32, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
+ row!(M5Max, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
+ row!(M5Max, w8_sym_g64, 2, GATE | GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M5Max, w8_sym_g64, 2, DOWN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 4, GATE | PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 6, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
+ row!(M5Max, w8_sym_g64, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
+ row!(M5Max, w8_sym_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
+ row!(M5Max, w8_sym_g64, 6, GATE_UP | PROJECTION_IN | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
+ row!(M5Max, w8_sym_g64, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
+ row!(M5Max, w8_sym_g64, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
+ row!(M5Max, w8_sym_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
+];
+
+fn qmv_format(shape: &MatmulShape) -> Option<(u32, u32)> {
+ match (shape.b_prologue, shape.b_bits, shape.b_group_size) {
+ (GemmBPrologueKind::ScaleZeroPointDequant, Some(4), Some(group @ (32 | 64))) => Some((4, group)),
+ (GemmBPrologueKind::ScaleSymmetricDequant, Some(8), Some(group @ (32 | 64))) => Some((8, group)),
+ _ => None,
+ }
+}
+
+pub fn route(
+ device: DeviceProfile,
+ shape: &MatmulShape,
+ all_bf16: bool,
+) -> Option<QmvRoute> {
+ if !all_bf16
+ || shape.m < 2
+ || shape.m > 7
+ || !shape.a_full_precision
+ || !shape.b_transpose
+ || shape.b_leading_dimension.is_some()
+ || shape.gathered
+ {
+ return None;
+ }
+ let (bits, group) = qmv_format(shape)?;
+ let mask = self::shape(shape.n, shape.k);
+ if mask == 0 {
+ return None;
+ }
+ let matches =
+ |row: &&RouteRow| row.bits == bits && row.group == group && row.m == shape.m && row.shapes & mask != 0;
+ let route =
+ ROWS.iter().filter(matches).find(|row| row.identity == device.identity()).map(|row| row.route).or_else(|| {
+ let same_family = |row: &&RouteRow| device.gpu_family().contains(row.identity);
+ let mut routes = ROWS.iter().filter(matches).filter(same_family).map(|row| row.route);
+ let route = routes.next()?;
+ routes.all(|candidate| candidate == route).then_some(route)
+ });
+ route.filter(|route| {
+ !matches!(
+ route,
+ QmvRoute::MainGemm(GemmPlan {
+ engine: GemmEngine::Mxu,
+ ..
+ })
+ ) || device.supports_mxu()
+ })
+}
+
+#[cfg(test)]
+#[path = "../../../../../../unit/backends/metal/kernel/matmul/qmv/routes_test.rs"]
+mod tests;
diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rs
index c6483b9c1..d03f39fce 100644
--- a/crates/uzu-engine/src/backends/metal/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/mod.rs
@@ -20,8 +20,4 @@ const _: () = {
pub use backend::Metal;
pub use context::MetalContext;
#[cfg(test)]
-pub use device_profile::{DeviceGeneration, DeviceProfile, DeviceSize};
-#[cfg(test)]
pub use kernel::matmul::gemm::GemmEngine;
-#[cfg(test)]
-pub(crate) use kernel::matmul::gemv::{GemvDispatch, GemvSpecialization};
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
index 2c9d41e0c..ae7a6de50 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
@@ -14,10 +14,10 @@ use crate::{
kernel::{
ActivationTransform, Kernels,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulShape},
+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
- metal::{DeviceProfile, GemmEngine, GemvDispatch, GemvSpecialization, Metal, MetalContext},
+ metal::{GemmEngine, Metal, MetalContext},
},
data_type::DataType,
tests::{
@@ -33,7 +33,7 @@ type MetalMatmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel;
enum BenchPath {
A8GemmMxu,
Bf16GemmMxu,
- Bf16Gemv,
+ Bf16Routed,
}
impl BenchPath {
@@ -41,7 +41,7 @@ impl BenchPath {
match self {
BenchPath::A8GemmMxu => "a8_gemm_mxu",
BenchPath::Bf16GemmMxu => "abf16_gemm_mxu",
- BenchPath::Bf16Gemv => "abf16_gemv",
+ BenchPath::Bf16Routed => "abf16_routed",
}
}
}
@@ -91,16 +91,15 @@ impl BenchmarkData {
let rht_factors = alloc_allocation_with_data::<Metal, i32>(context, &rht);
let groups = k / group_size;
- let a_elements = m as usize * k as usize;
Self {
unsigned_weights,
signed_weights,
weight_scales,
activations,
rht_factors,
- a_working: alloc_allocation::<Metal, bf16>(context, a_elements),
- a_int8: alloc_allocation::<Metal, i8>(context, a_elements),
- a_scales: alloc_allocation::<Metal, f32>(context, m as usize * groups as usize),
+ a_working: alloc_allocation::<Metal, bf16>(context, (m * k) as usize),
+ a_int8: alloc_allocation::<Metal, i8>(context, (m * k) as usize),
+ a_scales: alloc_allocation::<Metal, f32>(context, (m * groups) as usize),
m,
k,
n,
@@ -149,8 +148,6 @@ fn encode_step(
prepare: &ActivationTransform<Metal>,
hadamard: &ActivationTransform<Metal>,
matmul: &mut MetalMatmul,
- gemv: &mut GemvDispatch,
- device_profile: DeviceProfile,
encoder: &mut Encoder<Metal>,
) {
match path {
@@ -196,19 +193,11 @@ fn encode_step(
let args = data.bf16_arguments(output);
matmul.gemm.encode_with_engine(args, GemmEngine::Mxu, encoder).expect("bf16 gemm mxu encode");
},
- BenchPath::Bf16Gemv => {
+ BenchPath::Bf16Routed => {
encoder.encode_copy(&data.activations, .., &mut data.a_working, ..);
hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, data.m, data.k, encoder);
let args = data.bf16_arguments(output);
- let spec = GemvSpecialization::select_shape(
- &MatmulShape::from_arguments(&args),
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- device_profile,
- )
- .expect("bf16 gemv specialization");
- gemv.encode(args, spec, encoder).expect("bf16 gemv encode");
+ matmul.encode(args, encoder).expect("routed bf16 matmul encode");
},
}
}
@@ -216,15 +205,12 @@ fn encode_step(
fn bench_bits(
c: &mut Criterion,
context: &MetalContext,
- device_profile: DeviceProfile,
prepare: &ActivationTransform<Metal>,
hadamard: &ActivationTransform<Metal>,
bits: u32,
) {
let mut matmul = <MetalMatmul as MatmulKernel>::new(context, DataType::BF16, DataType::BF16, DataType::BF16)
.expect("matmul kernel");
- let mut gemv = GemvDispatch::new(DataType::BF16, DataType::BF16, DataType::BF16);
-
let mut group = c.benchmark_group(format!("{}/Kernel/A8W/w{bits}", type_short_name::<Metal>()));
group.sample_size(10);
group.warm_up_time(Duration::from_millis(100));
@@ -241,22 +227,10 @@ fn bench_bits(
HADAMARD_TRANSFORM_BLOCK_SIZE,
0xA8_00 ^ u64::from(bits) ^ k as u64 ^ n as u64,
);
- let mut output = alloc_allocation::<Metal, bf16>(context, m as usize * n as usize);
+ let mut output = alloc_allocation::<Metal, bf16>(context, (m * n) as usize);
let shape_label = format!("{layer}_m{m}_k{k}_n{n}");
- let gemv_eligible = GemvSpecialization::select_shape(
- &MatmulShape::from_arguments(&data.bf16_arguments(&mut output)),
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- device_profile,
- )
- .is_some();
-
- let mut paths = vec![BenchPath::A8GemmMxu, BenchPath::Bf16GemmMxu];
- if gemv_eligible {
- paths.push(BenchPath::Bf16Gemv);
- }
+ let paths = [BenchPath::A8GemmMxu, BenchPath::Bf16GemmMxu, BenchPath::Bf16Routed];
group.throughput(Throughput::Elements((m * k * n) as u64));
for path in paths {
@@ -264,17 +238,7 @@ fn bench_bits(
let benchmark_path =
format!("{}/Kernel/A8W/w{bits}/{}/{shape_label}", type_short_name::<Metal>(), path.label());
iter_encode_loop_named::<Metal, _>(context, bench, &benchmark_path, |encoder| {
- encode_step(
- path,
- &mut data,
- &mut output,
- prepare,
- hadamard,
- &mut matmul,
- &mut gemv,
- device_profile,
- encoder,
- );
+ encode_step(path, &mut data, &mut output, prepare, hadamard, &mut matmul, encoder);
});
});
}
@@ -288,12 +252,10 @@ fn bench_a8w(c: &mut Criterion) {
if !context.supports_mxu() {
return;
}
- let device_profile = context.device_profile();
-
let prepare = ActivationTransform::<Metal>::quantize(&context, DataType::BF16, 128, None).expect("prepare kernel");
let hadamard = ActivationTransform::<Metal>::input_rht(&context, DataType::BF16, true).expect("hadamard kernel");
for bits in [8u32, 4u32] {
- bench_bits(c, &context, device_profile, &prepare, &hadamard, bits);
+ bench_bits(c, &context, &prepare, &hadamard, bits);
}
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
index 40e268fd9..6f8197bfd 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
@@ -253,6 +253,13 @@ fn parity_bf16_gemv_qmv_fused_scale_bias() {
#[case::gs64_4bit(1, 96, 64, 64, 4, QuantizationMethod::ScaleBias)]
#[case::gs64_4bit_zp(2, 96, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs128_8bit(1, 192, 64, 128, 8, QuantizationMethod::ScaleBias)]
+// Whole quantization groups with an unaligned K block.
+#[case::gs64_4bit_k1152_zp(1, 1152, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs64_8bit_k1152_zp(1, 1152, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
+// Same, plus a final partial quantization group (1056 = 16 * 64 + 32).
+#[case::gs64_4bit_k1056_partial_group(1, 1056, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs64_4bit_k514_unaligned_rows(2, 514, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs64_8bit_k257_unaligned_rows(2, 257, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
fn parity_gemv_partial_group_bf16(
#[case] m: u32,
#[case] k: u32,
diff --git a/crates/uzu-engine/unit/backends/metal/device_profile_test.rs b/crates/uzu-engine/unit/backends/metal/device_profile_test.rs
index 5f59a9b15..ab2cc7a8a 100644
--- a/crates/uzu-engine/unit/backends/metal/device_profile_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/device_profile_test.rs
@@ -4,17 +4,68 @@ use super::*;
#[uzu_test]
fn device_profile_detection() {
- assert_eq!(classify_device(40, true, true, false), DeviceProfile::new(40, DeviceGeneration::Apple9));
- assert_eq!(classify_device(40, true, true, true), DeviceProfile::new(40, DeviceGeneration::M5Plus));
- assert_eq!(classify_device(20, true, true, false), DeviceProfile::new(20, DeviceGeneration::Apple9));
- assert_eq!(classify_device(10, true, false, false), DeviceProfile::new(10, DeviceGeneration::Apple8));
- assert_eq!(classify_device(8, false, false, false), DeviceProfile::new(8, DeviceGeneration::Legacy));
+ let cases = [
+ ("Apple M4 Max", 40, false, DeviceIdentity::M4Max, GpuFamily::Apple9, DeviceSize::Large),
+ ("Apple M5 Max", 40, true, DeviceIdentity::M5Max, GpuFamily::M5Plus, DeviceSize::Large),
+ ("Apple M3 Pro", 20, false, DeviceIdentity::M3Pro, GpuFamily::Apple9, DeviceSize::Small),
+ ("Apple M2", 10, false, DeviceIdentity::M2, GpuFamily::Apple8, DeviceSize::Small),
+ ("Apple M1", 8, false, DeviceIdentity::M1, GpuFamily::Legacy, DeviceSize::Small),
+ ];
+ for (name, cores, mxu, identity, family, size) in cases {
+ let profile = classify_device(name, cores, mxu);
+ assert_eq!(profile.identity(), identity);
+ assert_eq!(profile.gpu_family(), family);
+ assert_eq!(profile.size(), size);
+ assert_eq!(profile.supports_mxu(), mxu);
+ }
}
#[uzu_test]
-fn size_derives_from_core_count() {
- // M1 Max ships in two bins that straddle the cutoff, on identical silicon.
- assert_eq!(classify_device(24, false, false, false).size(), DeviceSize::Small);
- assert_eq!(classify_device(32, false, false, false).size(), DeviceSize::Large);
- assert_eq!(classify_device(8, false, false, false).gpu_core_count(), 8);
+fn family_detection_does_not_use_mxu_as_generation_proxy() {
+ let m5_without_mxu = classify_device("Apple M5", 12, false);
+ let m4_with_mxu = classify_device("Apple M4", 10, true);
+
+ assert_eq!(m5_without_mxu.gpu_family(), GpuFamily::M5Plus);
+ assert!(!m5_without_mxu.supports_mxu());
+ assert_eq!(m4_with_mxu.gpu_family(), GpuFamily::Apple9);
+ assert!(m4_with_mxu.supports_mxu());
+}
+
+#[uzu_test]
+fn device_identities_remain_distinct() {
+ let cases = [
+ ("Apple M1", DeviceIdentity::M1),
+ ("Apple M1 Pro", DeviceIdentity::M1Pro),
+ ("Apple M1 Max", DeviceIdentity::M1Max),
+ ("Apple M1 Ultra", DeviceIdentity::M1Ultra),
+ ("Apple M2", DeviceIdentity::M2),
+ ("Apple M2 Pro", DeviceIdentity::M2Pro),
+ ("Apple M2 Max", DeviceIdentity::M2Max),
+ ("Apple M2 Ultra", DeviceIdentity::M2Ultra),
+ ("Apple M3", DeviceIdentity::M3),
+ ("Apple M3 Pro", DeviceIdentity::M3Pro),
+ ("Apple M3 Max", DeviceIdentity::M3Max),
+ ("Apple M4", DeviceIdentity::M4),
+ ("Apple M4 Pro", DeviceIdentity::M4Pro),
+ ("Apple M4 Max", DeviceIdentity::M4Max),
+ ("Apple M5", DeviceIdentity::M5),
+ ("Apple M5 Pro", DeviceIdentity::M5Pro),
+ ("Apple M5 Max", DeviceIdentity::M5Max),
+ ];
+ for (name, expected) in cases {
+ assert_eq!(classify_device(name, 16, false).identity(), expected);
+ }
+}
+
+#[uzu_test]
+fn untuned_devices_keep_their_identity_and_family() {
+ let m1_ultra = classify_device("Apple M1 Ultra", 64, false);
+ assert_eq!(m1_ultra.identity(), DeviceIdentity::M1Ultra);
+ assert_eq!(m1_ultra.gpu_family(), GpuFamily::Legacy);
+}
+
+#[uzu_test]
+fn core_count_is_reduced_to_device_size() {
+ assert_eq!(classify_device("Apple M1 Max", 24, false).size(), DeviceSize::Small);
+ assert_eq!(classify_device("Apple M1 Max", 32, false).size(), DeviceSize::Large);
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index c1a50399d..018f92f72 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -4,13 +4,13 @@ use super::{super::specialization::GemmSpecialization, *};
use crate::backends::{
common::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmDTransform},
metal::{
- device_profile::{DeviceGeneration, DeviceProfile},
+ device_profile::{DeviceIdentity, DeviceProfile, DeviceSize},
kernel::matmul::MatmulMetalKernel,
},
};
-const LEGACY_PROFILE: DeviceProfile = DeviceProfile::new(8, DeviceGeneration::Legacy);
-const APPLE9_PROFILE: DeviceProfile = DeviceProfile::new(10, DeviceGeneration::Apple9);
+const LEGACY_PROFILE: DeviceProfile = DeviceProfile::new(DeviceIdentity::M1, DeviceSize::Small, false);
+const APPLE9_PROFILE: DeviceProfile = DeviceProfile::new(DeviceIdentity::M4, DeviceSize::Small, false);
fn shape(
m: u32,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
index 9118473c9..df438a596 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
@@ -1,59 +1,198 @@
use uzu_engine_macros::uzu_test;
use super::*;
-use crate::backends::metal::device_profile::DeviceGeneration;
+use crate::{
+ backends::{
+ common::{
+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
+ kernel::matmul::MatmulShape,
+ },
+ metal::device_profile::{DeviceIdentity, DeviceProfile, DeviceSize},
+ },
+ data_type::DataType,
+};
-const LARGE: DeviceProfile = DeviceProfile::new(40, DeviceGeneration::Apple9);
-const SMALL_APPLE9: DeviceProfile = DeviceProfile::new(10, DeviceGeneration::Apple9);
-const SMALL_APPLE8: DeviceProfile = DeviceProfile::new(10, DeviceGeneration::Apple8);
-const SMALL_LEGACY: DeviceProfile = DeviceProfile::new(8, DeviceGeneration::Legacy);
-const SMALL_M5: DeviceProfile = DeviceProfile::new(10, DeviceGeneration::M5Plus);
-/// 32-core M1 Max: Legacy generation on a Large die, so the G13 rules must not fire.
-const LARGE_LEGACY: DeviceProfile = DeviceProfile::new(32, DeviceGeneration::Legacy);
+fn profile(
+ size: DeviceSize,
+ identity: DeviceIdentity,
+) -> DeviceProfile {
+ DeviceProfile::new(
+ identity,
+ size,
+ matches!(identity, DeviceIdentity::M5 | DeviceIdentity::M5Pro | DeviceIdentity::M5Max),
+ )
+}
+
+fn qtile(
+ num_simdgroups: u32,
+ results_per_simdgroup: u32,
+) -> GemvTile {
+ GemvTile::quantized(num_simdgroups, results_per_simdgroup, 1, 32, 2)
+}
#[uzu_test]
fn fp_policy_cases() {
- #[rustfmt::skip]
let cases = [
- (LARGE, 1, 12288, 1536, true, tile(8, 8, 1)),
- (LARGE, 1, 12288, 1536, false, tile(8, 1, 1)),
- (LARGE, 1, 1536, 256, true, tile(8, 1, 1)),
- (SMALL_APPLE9, 1, 1536, 256, true, tile(8, 2, 1)),
- (LARGE, 8, 12288, 1536, true, tile(8, 1, 4)),
- (LARGE, 1, 1536, 12288, true, tile(8, 8, 4)),
- (SMALL_LEGACY, 1, 262144, 1536, true, tile(8, 1, 4)),
- (LARGE_LEGACY, 1, 262144, 1536, true, tile(8, 8, 1)),
- (LARGE_LEGACY, 1, 6144, 1536, true, tile(8, 8, 1)),
+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 12288, 1536, true, tile(8, 8, 1)),
+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 12288, 1536, false, tile(8, 1, 1)),
+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 1536, 256, true, tile(8, 1, 1)),
+ (profile(DeviceSize::Small, DeviceIdentity::M3), 1, 1536, 256, true, tile(8, 2, 1)),
+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 8, 12288, 1536, true, tile(8, 1, 4)),
+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 1536, 12288, true, tile(8, 8, 4)),
+ (profile(DeviceSize::Small, DeviceIdentity::M1), 1, 262144, 1536, true, tile(8, 1, 4)),
];
for (profile, m, n, k, aligned, expected) in cases {
- assert_eq!(fp_tile(m, n, k, aligned, profile), expected, "profile={profile:?} m={m} n={n} k={k}");
+ assert_eq!(fp_tile(m, n, k, aligned, profile), expected);
}
}
#[uzu_test]
fn quant_policy_cases() {
- #[rustfmt::skip]
let cases = [
- (LARGE, 1, 256, 1536, 4, false, qtile(2, 1)),
- (LARGE, 1, 262144, 1536, 4, false, DEFAULT_TILE),
- (SMALL_APPLE9, 1, 1536, 256, 4, false, qtile(4, 4)),
- (SMALL_APPLE8, 1, 2048, 1536, 4, false, qtile(8, 2)),
- (SMALL_LEGACY, 1, 256, 1536, 4, false, qtile(8, 2)),
- (SMALL_LEGACY, 1, 1536, 256, 4, false, qtile(4, 8)),
- (LARGE, 2, 2048, 1536, 4, false, DEFAULT_TILE),
- (LARGE, 1, 2048, 1536, 8, false, DEFAULT_TILE),
- (LARGE, 1, 2560, 9216, 4, true, qtile(4, 8)),
- // Small M5 dies classified as SmallApple9 before the split; keep those rows.
- (SMALL_M5, 1, 1536, 256, 4, false, qtile(4, 4)),
- (SMALL_M5, 1, 256, 1536, 4, false, qtile(4, 2)),
+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 256, 1536, 4, false, qtile(2, 1)),
+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 262144, 1536, 4, false, qtile(8, 4)),
+ (profile(DeviceSize::Small, DeviceIdentity::M3), 1, 1536, 256, 4, false, qtile(4, 4)),
+ (profile(DeviceSize::Small, DeviceIdentity::M2), 1, 2048, 1536, 4, false, qtile(8, 2)),
+ (profile(DeviceSize::Small, DeviceIdentity::M1), 1, 256, 1536, 4, false, qtile(8, 2)),
+ (profile(DeviceSize::Small, DeviceIdentity::M1), 1, 1536, 256, 4, false, qtile(4, 8)),
+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 2, 2048, 1536, 4, false, qtile(8, 4)),
+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 4, 5120, 5120, 4, false, qtile(8, 4)),
+ (
+ profile(DeviceSize::Large, DeviceIdentity::M3Max),
+ 1,
+ 2048,
+ 1536,
+ 8,
+ false,
+ GemvTile::quantized(8, 4, 1, 32, 4),
+ ),
+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 2560, 9216, 4, true, qtile(4, 8)),
];
for (profile, m, n, k, bits, has_rht, expected) in cases {
- assert_eq!(
- quant_tile(m, n, k, bits, has_rht, profile),
- expected,
- "profile={profile:?} m={m} n={n} k={k} bits={bits}"
- );
+ let actual = quantized_tile(
+ profile,
+ bits,
+ 32,
+ m,
+ n,
+ k,
+ if has_rht {
+ GemmDTransform::RHT
+ } else {
+ GemmDTransform::empty()
+ },
+ true,
+ )
+ .expect("quantized route");
+ assert_eq!(actual, expected, "m={m} n={n} k={k} bits={bits} profile={profile:?}");
}
}
+
+fn quantized(
+ profile: DeviceProfile,
+ bits: u32,
+ group: u32,
+ m: u32,
+ n: u32,
+ k: u32,
+ d_transform: GemmDTransform,
+) -> Option<GemvTile> {
+ quantized_tile(profile, bits, group, m, n, k, d_transform, true)
+}
+
+#[uzu_test]
+fn quantized_policy_edges() {
+ for (profile, m, group, expected) in [
+ (profile(DeviceSize::Large, DeviceIdentity::M5Max), 2, 32, GemvTile::quantized(8, 4, 1, 32, 4)),
+ (profile(DeviceSize::Large, DeviceIdentity::M5Max), 3, 32, GemvTile::quantized(8, 4, 1, 32, 4)),
+ (profile(DeviceSize::Small, DeviceIdentity::M2), 4, 64, GemvTile::quantized(8, 4, 1, 32, 8)),
+ ] {
+ assert_eq!(quantized(profile, 8, group, m, 4096, 4096, GemmDTransform::RHT), Some(expected));
+ }
+ let profile = profile(DeviceSize::Large, DeviceIdentity::M5Max);
+ let none = GemmDTransform::empty();
+ assert_eq!(quantized(profile, 4, 64, 4, 8192, 4100, none), Some(GemvTile::quantized(8, 4, 1, 32, 4)));
+ assert!(quantized(profile, 4, 32, 4, 8, 4096, none).is_some());
+ assert!(quantized(profile, 4, 32, 1, 8192, 4096, none).is_some());
+ assert_eq!(quantized(profile, 4, 32, 9, 8192, 4096, none), None);
+ assert_eq!(quantized_tile(profile, 4, 32, 4, 8192, 4096, none, true), Some(GemvTile::quantized(8, 4, 1, 32, 2)));
+}
+
+#[uzu_test]
+fn untuned_quantized_io_uses_only_the_generated_fallback() {
+ let profile = profile(DeviceSize::Large, DeviceIdentity::M5Max);
+ assert_eq!(quantized_tile(profile, 4, 32, 4, 3, 4096, GemmDTransform::empty(), false), None);
+ let tile =
+ quantized_tile(profile, 4, 32, 4, 4096, 4096, GemmDTransform::empty(), false).expect("mixed-IO fallback");
+ assert_eq!(tile, GemvTile::quantized(8, 4, 1, 32, 2));
+ assert_eq!(quantized_tile(profile, 4, 32, 5, 4096, 4096, GemmDTransform::empty(), false), None);
+}
+
+fn quant_shape(
+ m: u32,
+ n: u32,
+ d_transform: GemmDTransform,
+) -> MatmulShape {
+ MatmulShape {
+ m,
+ n,
+ k: 4096,
+ b_transpose: true,
+ b_leading_dimension: None,
+ b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,
+ b_bits: Some(4),
+ b_group_size: Some(32),
+ signed_codes: false,
+ a_full_precision: true,
+ gathered: false,
+ d_transform,
+ }
+}
+
+#[uzu_test]
+fn block_unaligned_quantized_k_stays_on_gemv() {
+ let profile = profile(DeviceSize::Large, DeviceIdentity::M5Max);
+ let shape = MatmulShape {
+ m: 1,
+ n: 64,
+ k: 1152,
+ b_transpose: true,
+ b_leading_dimension: None,
+ b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,
+ b_bits: Some(4),
+ b_group_size: Some(64),
+ signed_codes: false,
+ a_full_precision: true,
+ gathered: false,
+ d_transform: GemmDTransform::empty(),
+ };
+ assert!(
+ super::super::kernel::GemvSpecialization::select_shape(
+ &shape,
+ DataType::BF16,
+ DataType::BF16,
+ DataType::BF16,
+ profile,
+ )
+ .is_some()
+ );
+}
+
+#[uzu_test]
+fn specialization_preserves_quantized_route_and_accumulate_tail() {
+ let profile = profile(DeviceSize::Large, DeviceIdentity::M5Max);
+ let select = |n, transform| {
+ super::super::kernel::GemvSpecialization::select_shape(
+ &quant_shape(4, n, transform),
+ DataType::BF16,
+ DataType::BF16,
+ DataType::BF16,
+ profile,
+ )
+ };
+ let clean = select(8192, GemmDTransform::empty()).expect("quantized specialization");
+ assert!(clean.output_row_tile() > DEFAULT_RESULTS_PER_SIMDGROUP);
+ assert_eq!(select(8192 + clean.output_row_tile() / 2, GemmDTransform::ACCUMULATE), None);
+}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
new file mode 100644
index 000000000..7e016371c
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
@@ -0,0 +1,233 @@
+use std::fmt::Write;
+
+use uzu_engine_macros::uzu_test;
+use xxhash_rust::xxh3::xxh3_64;
+
+use super::*;
+use crate::{
+ backends::{
+ common::{
+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
+ kernel::matmul::MatmulShape,
+ },
+ metal::{
+ device_profile::{DeviceIdentity, DeviceProfile, DeviceSize},
+ kernel::matmul::{MatmulDispatch, MatmulMetalKernel, gemm::GemmProblem, gemv::GemvSpecialization},
+ },
+ },
+ data_type::DataType,
+};
+
+const FROZEN_PLAN_FINGERPRINT: u64 = 5_839_212_743_558_880_136;
+
+const DEVICES: [(&str, DeviceIdentity); 7] = [
+ ("m1", DeviceIdentity::M1),
+ ("m2", DeviceIdentity::M2),
+ ("m2-pro", DeviceIdentity::M2Pro),
+ ("m3-max", DeviceIdentity::M3Max),
+ ("m4", DeviceIdentity::M4),
+ ("m4-pro", DeviceIdentity::M4Pro),
+ ("m5-max", DeviceIdentity::M5Max),
+];
+const FORMATS: [(&str, u32, u32, GemmBPrologueKind); 4] = [
+ ("W4/ZP G32", 4, 32, GemmBPrologueKind::ScaleZeroPointDequant),
+ ("W4/ZP G64", 4, 64, GemmBPrologueKind::ScaleZeroPointDequant),
+ ("W8/Symmetric G32", 8, 32, GemmBPrologueKind::ScaleSymmetricDequant),
+ ("W8/Symmetric G64", 8, 64, GemmBPrologueKind::ScaleSymmetricDequant),
+];
+const SHAPES: [(&str, u32, u32); 7] = [
+ ("down", 5120, 17408),
+ ("gate", 6144, 5120),
+ ("gate-up", 34816, 5120),
+ ("projection-in", 16480, 5120),
+ ("projection-out", 5120, 6144),
+ ("qkv", 8192, 5120),
+ ("readout", 248320, 5120),
+];
+
+fn problem(
+ m: u32,
+ n: u32,
+ k: u32,
+ bits: u32,
+ group: u32,
+ prologue: GemmBPrologueKind,
+) -> MatmulShape {
+ MatmulShape {
+ m,
+ n,
+ k,
+ b_transpose: true,
+ b_leading_dimension: None,
+ b_prologue: prologue,
+ b_bits: Some(bits),
+ b_group_size: Some(group),
+ signed_codes: false,
+ a_full_precision: true,
+ gathered: false,
+ d_transform: GemmDTransform::empty(),
+ }
+}
+
+#[uzu_test]
+fn table_is_complete_and_fingerprint_is_stable() {
+ let mut canonical = Vec::new();
+ let mut matched_rows = vec![false; ROWS.len()];
+ let mut tuned = 0;
+ let mut main_gemv = 0;
+ let mut main_gemm = 0;
+ for &(device_name, identity) in &DEVICES {
+ for &(format_name, bits, group, prologue) in &FORMATS {
+ for m in 2..=7 {
+ for &(shape_name, n, k) in &SHAPES {
+ let mask = shape(n, k);
+ let matches: Vec<_> = ROWS
+ .iter()
+ .enumerate()
+ .filter(|row| {
+ row.1.identity == identity
+ && row.1.bits == bits
+ && row.1.group == group
+ && row.1.m == m
+ && row.1.shapes & mask != 0
+ })
+ .collect();
+ assert_eq!(matches.len(), 1, "route coverage for {device_name} {format_name} M={m} {shape_name}");
+ let (row_index, row) = matches[0];
+ matched_rows[row_index] = true;
+ let selected = row.route;
+ match selected {
+ QmvRoute::Tuned(_) => tuned += 1,
+ QmvRoute::MainGemv(_) => main_gemv += 1,
+ QmvRoute::MainGemm(_) => main_gemm += 1,
+ }
+ canonical.push(format!("{device_name}|{format_name}|{shape_name}|{m}|{n}|{k}|{selected:?}"));
+ let profile = device_profile(identity);
+ let problem = problem(m, n, k, bits, group, prologue);
+ assert_eq!(route(profile, &problem, true), Some(selected));
+ let runtime = MatmulMetalKernel::choose_dispatch(
+ &problem,
+ profile,
+ profile.supports_mxu(),
+ DataType::BF16,
+ DataType::BF16,
+ DataType::BF16,
+ );
+ if let QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) = selected {
+ let specialization = GemvSpecialization::select_tile(
+ &problem,
+ DataType::BF16,
+ DataType::BF16,
+ DataType::BF16,
+ tile,
+ )
+ .expect("stored GEMV tile must be legal");
+ assert_eq!(specialization.tile(), tile);
+ assert!(matches!(runtime, MatmulDispatch::Gemv(actual) if actual == specialization));
+ } else if let QmvRoute::MainGemm(plan) = selected {
+ let problem =
+ GemmProblem::new(problem, DataType::BF16, DataType::BF16, profile.supports_mxu(), profile);
+ assert!(problem.plan_is_legal(plan), "stored GEMM plan must be legal");
+ assert!(matches!(runtime, MatmulDispatch::Gemm(actual) if actual == plan));
+ }
+ }
+ }
+ }
+ }
+ assert!(matched_rows.into_iter().all(|matched| matched), "route table contains an orphaned row");
+ canonical.sort();
+ assert_eq!(canonical.len(), 1176);
+ let mut expanded = String::new();
+ for line in canonical {
+ writeln!(&mut expanded, "{line}").unwrap();
+ }
+ assert_eq!(tuned, 864);
+ assert_eq!(main_gemv, 221);
+ assert_eq!(main_gemm, 91);
+ assert_eq!(xxh3_64(expanded.trim_end().as_bytes()), FROZEN_PLAN_FINGERPRINT);
+}
+
+fn device_profile(identity: DeviceIdentity) -> DeviceProfile {
+ let large = matches!(identity, DeviceIdentity::M3Max | DeviceIdentity::M5Max);
+ DeviceProfile::new(
+ identity,
+ if large {
+ DeviceSize::Large
+ } else {
+ DeviceSize::Small
+ },
+ matches!(identity, DeviceIdentity::M5Max),
+ )
+}
+
+#[uzu_test]
+fn exact_lookup_rejects_non_matrix_inputs() {
+ let profile = device_profile(DeviceIdentity::M1);
+ let p = problem(2, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
+ assert!(route(profile, &p, true).is_some());
+ for mutate in [
+ |p: &mut MatmulShape| p.m = 1,
+ |p: &mut MatmulShape| p.n = 1,
+ |p: &mut MatmulShape| p.b_bits = Some(8),
+ |p: &mut MatmulShape| p.gathered = true,
+ ] {
+ let mut rejected = p;
+ mutate(&mut rejected);
+ assert!(route(profile, &rejected, true).is_none());
+ }
+ assert!(route(profile, &p, false).is_none());
+
+ let mut rht = p;
+ rht.d_transform = GemmDTransform::RHT;
+ rht.signed_codes = true;
+ let selected = route(profile, &rht, true).expect("RHT must preserve the exact route");
+ let QmvRoute::Tuned(tile) = selected else {
+ panic!("test anchor must use a tuned tile");
+ };
+ assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_some());
+ rht.n -= 1;
+ assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_none());
+
+ let m5_without_mxu = DeviceProfile::new(DeviceIdentity::M5Max, DeviceSize::Large, false);
+ let p = problem(7, 6144, 5120, 8, 64, GemmBPrologueKind::ScaleSymmetricDequant);
+ assert!(matches!(route(device_profile(DeviceIdentity::M5Max), &p, true), Some(QmvRoute::MainGemm(_))));
+ assert_eq!(route(m5_without_mxu, &p, true), None);
+}
+
+#[uzu_test]
+fn normal_routing_handles_inputs_outside_the_frozen_matrix() {
+ let profile = device_profile(DeviceIdentity::M1);
+ for (m, n, k) in [(1, 5120, 17408), (2, 4096, 5120)] {
+ let problem = problem(m, n, k, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
+ assert_eq!(route(profile, &problem, true), None);
+ let specialization =
+ GemvSpecialization::select_shape(&problem, DataType::BF16, DataType::BF16, DataType::BF16, profile)
+ .expect("normal M1 policy should select GEMV for this anchor");
+ assert!(matches!(
+ MatmulMetalKernel::choose_dispatch(
+ &problem,
+ profile,
+ false,
+ DataType::BF16,
+ DataType::BF16,
+ DataType::BF16,
+ ),
+ MatmulDispatch::Gemv(actual) if actual == specialization
+ ));
+ }
+}
+
+#[uzu_test]
+fn family_lookup_requires_one_unanimous_route() {
+ let m1_max = DeviceProfile::new(DeviceIdentity::M1Max, DeviceSize::Large, false);
+ let m1_route = problem(4, 8192, 5120, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
+ assert_eq!(route(m1_max, &m1_route, true), route(device_profile(DeviceIdentity::M1), &m1_route, true));
+
+ let family = DeviceProfile::new(DeviceIdentity::M2Max, DeviceSize::Small, false);
+ let unanimous = problem(6, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
+ assert_eq!(route(family, &unanimous, true), route(device_profile(DeviceIdentity::M2), &unanimous, true));
+
+ let disagreement = problem(3, 5120, 6144, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
+ assert_eq!(route(family, &disagreement, true), None);
+ assert!(route(device_profile(DeviceIdentity::M2), &disagreement, true).is_some());
+}