A high-performance inference engine for AI models
New generic qmv + tuning (#741)
CCYehbd3c8983f5f1
diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rsindex 310f88b24..a81773318 100644--- a/crates/uzu-engine/src/backends/metal/context.rs+++ b/crates/uzu-engine/src/backends/metal/context.rs@@ -51,10 +51,10 @@ pub struct MetalContext {impl MetalContext {pub fn supports_mxu(&self) -> bool {- self.device.supports_mxu()+ self.device_profile.supports_mxu()}- pub fn device_profile(&self) -> DeviceProfile {+ pub(super) fn device_profile(&self) -> DeviceProfile {self.device_profile}@@ -157,12 +157,7 @@ impl Context for MetalContext {let command_queue4 = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueueMtl4)?;let gpu_core_count = device.gpu_core_count();- let device_profile = classify_device(- gpu_core_count,- device.supports_family(metal::MTLGPUFamily::Apple8),- device.supports_family(metal::MTLGPUFamily::Apple9),- device.supports_mxu(),- );+ let device_profile = classify_device(&device.name(), gpu_core_count, device.supports_mxu());let page_size = MTLSparsePageSize::KB256;let heap_capacity = Metal::ALLOCATION_GRANULARITY;let sparse_pool = MetalSparseHeapPool::new(page_size, heap_capacity);diff --git a/crates/uzu-engine/src/backends/metal/device_profile.rs b/crates/uzu-engine/src/backends/metal/device_profile.rsindex 074a4055c..4177bb767 100644--- a/crates/uzu-engine/src/backends/metal/device_profile.rs+++ b/crates/uzu-engine/src/backends/metal/device_profile.rs@@ -1,76 +1,141 @@-//TODO: remove after retune with gpu core counts#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum DeviceSize {- /// Below Max/Ultra class: base and Pro dies, plus Max parts binned under- /// the core-count cutoff (a 24-core M1 Max lands here, a 32-core one does not).- Small,- /// Max/Ultra-class GPUs, currently classified by >= 30 GPU cores.- Large,+pub(super) enum DeviceIdentity {+ M1,+ M1Pro,+ M1Max,+ M1Ultra,+ M2,+ M2Pro,+ M2Max,+ M2Ultra,+ M3,+ M3Pro,+ M3Max,+ M4,+ M4Pro,+ M4Max,+ M5,+ M5Pro,+ M5Max,+}++impl DeviceIdentity {+ fn from_name(device_name: &str) -> Self {+ let Some(chip) = device_name.strip_prefix("Apple M") else {+ panic!("unsupported Metal device {device_name:?}; add its DeviceIdentity mapping");+ };+ match chip {+ "1" => Self::M1,+ "1 Pro" => Self::M1Pro,+ "1 Max" => Self::M1Max,+ "1 Ultra" => Self::M1Ultra,+ "2" => Self::M2,+ "2 Pro" => Self::M2Pro,+ "2 Max" => Self::M2Max,+ "2 Ultra" => Self::M2Ultra,+ "3" => Self::M3,+ "3 Pro" => Self::M3Pro,+ "3 Max" => Self::M3Max,+ "4" => Self::M4,+ "4 Pro" => Self::M4Pro,+ "4 Max" => Self::M4Max,+ "5" => Self::M5,+ "5 Pro" => Self::M5Pro,+ "5 Max" => Self::M5Max,+ _ => panic!("unsupported Metal device {device_name:?}; add its DeviceIdentity mapping"),+ }+ }++ const fn gpu_family(self) -> GpuFamily {+ match self {+ Self::M1 | Self::M1Pro | Self::M1Max | Self::M1Ultra => GpuFamily::Legacy,+ Self::M2 | Self::M2Pro | Self::M2Max | Self::M2Ultra => GpuFamily::Apple8,+ Self::M3 | Self::M3Pro | Self::M3Max | Self::M4 | Self::M4Pro | Self::M4Max => GpuFamily::Apple9,+ Self::M5 | Self::M5Pro | Self::M5Max => GpuFamily::M5Plus,+ }+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum DeviceGeneration {- Legacy, // M1 (G13) and A14 or older- Apple8, // M2, A15/16- Apple9, // M3/4, A17 Pro, A18- M5Plus, // M5 (Apple9 + MXU support)+pub(super) enum GpuFamily {+ Legacy,+ Apple8,+ Apple9,+ M5Plus,+}++impl GpuFamily {+ pub(super) fn contains(+ self,+ identity: DeviceIdentity,+ ) -> bool {+ identity.gpu_family() == self+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub struct DeviceProfile {- gpu_core_count: u32,- generation: DeviceGeneration,+pub(super) enum DeviceSize {+ Small,+ Large,}-const LARGE_MIN_GPU_CORES: u32 = 30;+impl DeviceSize {+ fn from_gpu_core_count(gpu_core_count: u32) -> Self {+ if gpu_core_count >= 30 {+ Self::Large+ } else {+ Self::Small+ }+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) struct DeviceProfile {+ identity: DeviceIdentity,+ size: DeviceSize,+ supports_mxu: bool,+}impl DeviceProfile {- pub const fn new(- gpu_core_count: u32,- generation: DeviceGeneration,+ pub(super) const fn new(+ identity: DeviceIdentity,+ size: DeviceSize,+ supports_mxu: bool,) -> Self {Self {- gpu_core_count,- generation,+ identity,+ size,+ supports_mxu,}}- pub const fn size(self) -> DeviceSize {- if self.gpu_core_count >= LARGE_MIN_GPU_CORES {- DeviceSize::Large- } else {- DeviceSize::Small- }+ pub(super) const fn identity(self) -> DeviceIdentity {+ self.identity+ }++ pub(super) const fn gpu_family(self) -> GpuFamily {+ self.identity.gpu_family()}- // TODO: retune based on gpu core counts- pub const fn gpu_core_count(self) -> u32 {- self.gpu_core_count+ pub(super) const fn size(self) -> DeviceSize {+ self.size}- pub const fn generation(self) -> DeviceGeneration {- self.generation+ pub(super) const fn supports_mxu(self) -> bool {+ self.supports_mxu}}pub(super) fn classify_device(+ device_name: &str,gpu_core_count: u32,- supports_apple8_family: bool,- supports_apple9_family: bool,supports_mxu: bool,) -> DeviceProfile {- // MXU is probed first: M5 also reports Apple9, so a family check alone- // cannot separate the two generations.- let generation = if supports_mxu {- DeviceGeneration::M5Plus- } else if supports_apple9_family {- DeviceGeneration::Apple9- } else if supports_apple8_family {- DeviceGeneration::Apple8- } else {- DeviceGeneration::Legacy- };- DeviceProfile::new(gpu_core_count, generation)+ DeviceProfile::new(+ DeviceIdentity::from_name(device_name),+ DeviceSize::from_gpu_core_count(gpu_core_count),+ supports_mxu,+ )}#[cfg(test)]diff --git a/crates/uzu-engine/src/backends/metal/kernel/common/integral_constant.h b/crates/uzu-engine/src/backends/metal/kernel/common/integral_constant.hindex fa36461fc..6fd581ec6 100644--- a/crates/uzu-engine/src/backends/metal/kernel/common/integral_constant.h+++ b/crates/uzu-engine/src/backends/metal/kernel/common/integral_constant.h@@ -6,6 +6,8 @@namespace uzu {+#define UZU_ALWAYS_INLINE __attribute__((always_inline))+template <typename T, T v>struct integral_constant {static constexpr constant T value = v;diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rsindex 475fc2254..5ee51c0e5 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs@@ -9,7 +9,7 @@ use crate::{},metal::{Metal, MetalContext,- device_profile::{DeviceGeneration, DeviceSize},+ device_profile::{DeviceSize, GpuFamily},error::MetalError,kernel::MetalKernels,},@@ -76,7 +76,7 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {let use_mxu = arguments.data_type == DataType::BF16 && context.supports_mxu();let transposed_h0 = !use_mxu&& context.device_profile().size() == DeviceSize::Small- && matches!(context.device_profile().generation(), DeviceGeneration::Legacy | DeviceGeneration::Apple8);+ && matches!(context.device_profile().gpu_family(), GpuFamily::Legacy | GpuFamily::Apple8);Ok(Self {arguments: *arguments,prefix: <MetalKernels as Kernels>::BuildTreePrefixKernel::new(context)?,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/qdot.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/qdot.hdeleted file mode 100644index a72149d1e..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/qdot.h+++ /dev/null@@ -1,139 +0,0 @@-#pragma once--#include <metal_stdlib>--#include "../../common/defines.h"-#include "quant_unpack.h"--using namespace metal;--namespace uzu {-namespace gemm {--// For packed unsigned weights, pre-scales each activation lane k by-// 2^-(BITS*k). qdot then reads the matching nibble/byte in place (value =-// q * 2^(BITS*k)); the factors cancel, so the dot product is unchanged. All-// factors are powers of two, so this is bit-exact.-template <typename T, typename U, int VALUES_PER_THREAD, int BITS>-METAL_FUNC U load_vector(const device T* x, thread U* x_thread, const bool signed_codes) {- using U4 = vec<U, 4>;- const U4 inv = (BITS == 8 && signed_codes)- ? U4(U(1))- : U4(U(1), U(1) / U(1u << BITS), U(1) / U(1u << (2u * BITS)), U(1) / U(1u << (3u * BITS)));- U sum = 0;- thread U4* x_vec4 = reinterpret_cast<thread U4*>(x_thread);- METAL_PRAGMA_UNROLL- for (int i = 0; i < VALUES_PER_THREAD / 4; i++) {- U4 v = U4(x[4 * i], x[4 * i + 1], x[4 * i + 2], x[4 * i + 3]);- sum += v[0] + v[1] + v[2] + v[3];- x_vec4[i] = v * inv;- }- return sum;-}--template <typename T, typename U, int VALUES_PER_THREAD>-METAL_FUNC U load_vector_safe(const device T* x, thread U* x_thread, int N) {- U sum = 0;- METAL_PRAGMA_UNROLL- for (int i = 0; i < VALUES_PER_THREAD; ++i) {- x_thread[i] = 0;- }- for (int i = 0; i < N; ++i) {- U v = x[i];- sum += v;- x_thread[i] = v;- }- return sum;-}--template <typename U, int VALUES_PER_THREAD, int BITS>-METAL_FUNC U qdot(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum, const bool signed_codes) {- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");-- U accumulator = 0;- if constexpr (BITS == 4) {- using U4 = vec<U, 4>;- const device ushort* weight_words = reinterpret_cast<const device ushort*>(w);- const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);- const ushort packed_mask = signed_codes ? 0x8888u : 0u;- METAL_PRAGMA_UNROLL- for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {- // Mask each nibble in place (no shifts); value of lane k is n_k << (4*k),- // i.e. n_k * 16^k, which is < 2^23 so the magic-number convert is valid.- // The matching x lane was pre-divided by 16^k in load_vector.- const uint4 lanes = uint4(ushort(weight_words[i] ^ packed_mask)) & uint4(0x000fu, 0x00f0u, 0x0f00u, 0xf000u);- const U4 weight_vec4 = U4(as_type<float4>(lanes | uint4(0x4b000000u)) - float4(8388608.0f));- accumulator += dot(x_vec4[i], weight_vec4);- }- } else if constexpr (BITS == 8) {- using U4 = vec<U, 4>;- const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);- if (signed_codes) {- const device char4* weight_vectors = reinterpret_cast<const device char4*>(w);- METAL_PRAGMA_UNROLL- for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {- accumulator += dot(x_vec4[i], U4(weight_vectors[i]));- }- } else {- const device uint* weight_words = reinterpret_cast<const device uint*>(w);- METAL_PRAGMA_UNROLL- for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {- // Keep each byte in place. Lane k is b_k * 256^k, while the- // matching activation lane was pre-divided by 256^k.- const uint4 lanes = uint4(weight_words[i]) & uint4(0x000000ffu, 0x0000ff00u, 0x00ff0000u, 0xff000000u);- accumulator += dot(x_vec4[i], U4(float4(lanes)));- }- }- }- const U adjusted_bias = (BITS == 8 && signed_codes) ? bias + scale * U(128) : bias;- return scale * accumulator + sum * adjusted_bias;-}--template <typename U, int VALUES_PER_THREAD, int BITS>-METAL_FUNC U-qdot_safe(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum, int N, const bool signed_codes) {- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");-- U accumulator = 0;- if constexpr (BITS == 4) {- using U4 = vec<U, 4>;- const device uint16_t* weight_words = reinterpret_cast<const device uint16_t*>(w);- const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);- const uint16_t packed_mask = signed_codes ? 0x8888u : 0u;-- int full_chunks = N / 4;- for (int i = 0; i < full_chunks; i++) {- uint16_t weight_word = weight_words[i] ^ packed_mask;- U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(weight_word, weight_word >> 4, weight_word >> 8, weight_word >> 12));- accumulator += dot(x_vec4[i], weight_vec4);- }-- int remainder = N & 3;- if (remainder > 0) {- uint16_t weight_word = weight_words[full_chunks] ^ packed_mask;- int base_index = 4 * full_chunks;- accumulator += x_thread[base_index] * uint_to_fp<U>(weight_word & 0xf);- if (remainder > 1)- accumulator += x_thread[base_index + 1] * uint_to_fp<U>((weight_word >> 4) & 0xf);- if (remainder > 2)- accumulator += x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);- }- } else if constexpr (BITS == 8) {- if (signed_codes) {- const device int8_t* signed_weights = reinterpret_cast<const device int8_t*>(w);- for (int i = 0; i < N; i++) {- accumulator += x_thread[i] * U(signed_weights[i]);- }- } else {- for (int i = 0; i < N; i++) {- accumulator += x_thread[i] * U(w[i]);- }- }- }-- const U adjusted_bias = (BITS == 8 && signed_codes) ? bias + scale * U(128) : bias;- return scale * accumulator + sum * adjusted_bias;-}--} // namespace gemm-} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.hindex 9b1677df9..ec8a95f03 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h@@ -61,16 +61,21 @@ METAL_FUNC constexpr Int zero_point_row_stride(Int groups_per_row) {}template <ushort BITS>-METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint group_index) {+METAL_FUNC uint decode_zero_point(uint8_t packed, uint group_index) {static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 zero points supported");if constexpr (BITS == 4) {- const uint packed = uint(zero_points_row[group_index >> 1]);- return (packed >> ((group_index & 1u) * 4u)) & 0x0Fu;+ return (uint(packed) >> ((group_index & 1u) * 4u)) & 0x0Fu;} else {- return uint(zero_points_row[group_index]);+ return uint(packed);}}+template <ushort BITS>+METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint group_index) {+ const uint byte_index = BITS == 4 ? group_index >> 1 : group_index;+ return decode_zero_point<BITS>(zero_points_row[byte_index], group_index);+}+METAL_FUNC char4 unpack_signed_nibbles_to_int8(uint packed) {uint spread = (packed | (packed << 8)) & 0x00FF00FFu;spread = (spread | (spread << 4)) & 0x0F0F0F0Fu;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rsindex 1e5091984..2350fd612 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -10,13 +10,13 @@ pub(super) use selection::GemmProblem;use crate::backends::common::gpu_types::gemm::GemmTiling;-#[derive(Debug, Clone, Copy, PartialEq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum GemmEngine {Simdgroup,Mxu,}-#[derive(Debug, Clone, Copy, PartialEq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub struct GemmPlan {pub(super) engine: GemmEngine,pub(super) tiling: GemmTiling,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rsindex 6b88dab0c..fffdc926d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs@@ -6,7 +6,7 @@use crate::backends::{common::gpu_types::gemm::GemmTiling,- metal::device_profile::{DeviceGeneration, DeviceProfile},+ metal::device_profile::{DeviceProfile, GpuFamily},};pub(super) const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;@@ -103,7 +103,7 @@ pub(super) fn simdgroup_fp_tile(/// A partial trailing M block costs a second pass over the weights. Older GPUs are bound by that;/// Apple9 and newer would rather keep the narrow tile's parallelism.fn prefers_wide_partial_m_tile(profile: DeviceProfile) -> bool {- matches!(profile.generation(), DeviceGeneration::Legacy | DeviceGeneration::Apple8)+ matches!(profile.gpu_family(), GpuFamily::Legacy | GpuFamily::Apple8)}pub(super) fn simdgroup_quant_tile(diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex d36e2f12a..8c3676138 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -77,6 +77,39 @@ impl GemmProblem {Ok(())}+ #[cfg(test)]+ pub fn plan_is_legal(+ self,+ plan: GemmPlan,+ ) -> bool {+ use super::specialization::GemmSpecialization;+ use crate::backends::common::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment};++ if self.validate_engine(plan.engine).is_err()+ || plan.engine == GemmEngine::Mxu && !plan.tiling.is_mxu_variant()+ || plan.engine == GemmEngine::Simdgroup && plan.tiling.is_mxu_variant()+ || plan.split_k == 0+ || !self.shape.k.is_multiple_of(plan.split_k)+ {+ return false;+ }+ let alignment = GemmAlignment::new(+ self.shape.m.is_multiple_of(plan.tiling.block_m()),+ self.shape.n.is_multiple_of(plan.tiling.block_n()),+ self.shape.k.is_multiple_of(plan.tiling.block_k()),+ );+ GemmSpecialization::from_plan(+ plan,+ self.shape,+ self.weights_data_type,+ self.shape.d_transform,+ alignment,+ GemmAPrologueKind::FullPrecision,+ None,+ )+ .is_ok()+ }+fn finish_plan(&self,engine: GemmEngine,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.hnew file mode 100644index 000000000..739a3697f--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h@@ -0,0 +1,38 @@+#pragma once++#include <metal_stdlib>++#include "../../common/defines.h"++namespace uzu {+namespace gemm {++template <typename AT_, typename BT_, typename DT_>+struct GemvOperands {+ using AT = AT_;+ using BT = BT_;+ using DT = DT_;+ const device uint32_t* b;+ const device BT* scales;+ const device uint8_t* zero_points;+ const device BT* biases;+ const device AT* a;+ device DT* d;+ const device BT* output_bias;+ const device int32_t* hadamard_factors;+ const device uint* gather_indices;+};++struct GemvParams {+ uint in_vec_size;+ uint out_vec_size;+ uint batch_size;+ float ab_scale;+ float soft_cap;+ GemmDTransform output_transform;+ bool gathered;+ bool signed_codes;+};++} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/b_source.hdeleted file mode 100644index 0c66e9157..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/b_source.h+++ /dev/null@@ -1,74 +0,0 @@-#pragma once--#include "../../../generated/gemm.h"-#include "full_precision_b_source.h"-#include "quantized_b_source.h"--namespace uzu {-namespace gemm {--template <- typename BT,- typename AT,- typename U,- GemmBPrologueKind B_PROLOGUE,- uint GROUP_SIZE,- uint BITS,- uint K_SPLIT,- uint RESULTS_PER_SIMDGROUP,- bool INPUT_ALIGNED>-struct BSource {- static METAL_FUNC void accumulate(- thread U (&result)[RESULTS_PER_SIMDGROUP],- const device uint32_t* b,- const device BT* scales,- const device uint8_t* zero_points,- const device BT* biases,- const device AT* a,- const device uint* gather_indices,- bool gathered,- uint in_vec_size,- uint out_vec_size,- uint out_row,- uint batch_idx,- uint simd_lane,- uint k_slice,- const bool signed_codes- ) {- if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {- FullPrecisionBSource<BT, AT, U, RESULTS_PER_SIMDGROUP, K_SPLIT, INPUT_ALIGNED>::accumulate(- result,- b,- a,- gather_indices,- gathered,- in_vec_size,- out_vec_size,- out_row,- batch_idx,- simd_lane,- k_slice- );- } else {- QuantizedBSource<BT, AT, U, B_PROLOGUE, GROUP_SIZE, BITS, RESULTS_PER_SIMDGROUP, INPUT_ALIGNED>::accumulate(- result,- b,- scales,- zero_points,- biases,- a,- gather_indices,- gathered,- in_vec_size,- out_vec_size,- out_row,- batch_idx,- simd_lane,- signed_codes- );- }- }-};--} // namespace gemm-} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.hindex f28e52208..0bc642579 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h@@ -2,89 +2,122 @@#include "../../../common/soft_cap.h"#include "../../../hadamard_transform/hadamard_transform.h"+#include "arguments.h"+#include "tile.h"namespace uzu {namespace gemm {-template <typename BT, typename DT, typename U, uint RESULTS_PER_SIMDGROUP>+template <typename Tile, typename AT, typename BT, typename DT, bool FULL_TILE>struct Epilogue {+ using U = float;+static METAL_FUNC void store(- thread U (&result)[RESULTS_PER_SIMDGROUP],- device DT* d,- const device BT* output_bias,- const device int32_t* hadamard_factors,- threadgroup U* shared_results,- float ab_scale,- float soft_cap,- GemmDTransform output_transform,- uint out_row,- uint out_vec_size,- uint out_block_idx,- uint simd_group,- uint simd_lane,- bool writer+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],+ const thread GemvOperands<AT, BT, DT>& ops,+ const thread GemvParams& params,+ const thread OutputTile<Tile, FULL_TILE>& tile,+ threadgroup U* shared_results) {- const bool is_scale = output_transform.contains(GemmDTransform::SCALE);- const bool is_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);- const bool is_bias = output_transform.contains(GemmDTransform::BIAS);- const bool is_soft_cap = output_transform.contains(GemmDTransform::SOFT_CAP);- const bool use_hadamard = output_transform.contains(GemmDTransform::RHT);-- if (writer && simd_lane == 0) {- METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- U value = result[row];- if (is_scale) {- value = static_cast<U>(ab_scale) * value;- }- const uint global_row = out_row + row;- if (is_accumulate && global_row < out_vec_size) {- value += static_cast<U>(d[row]);- }- if (is_bias && !use_hadamard && global_row < out_vec_size) {- value += static_cast<U>(output_bias[global_row]);- }- if (is_soft_cap) {- value = apply_soft_cap(value, soft_cap);- }- result[row] = value;- }+ const bool use_hadamard =+ Tile::OUTPUT_ROWS >= METAL_SIMD_SIZE && params.output_transform.contains(GemmDTransform::RHT);+ const bool output_writer = tile.writer && tile.reduction_lane == 0;+ if (!use_hadamard && !output_writer) {+ return;}- if (use_hadamard) {- if (simd_lane == 0) {- METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] = result[row];- }- }+ if (!use_hadamard) {+ write_results<false>(result, ops, params, tile, shared_results);+ return;+ }+ if (output_writer) {+ write_results<true>(result, ops, params, tile, shared_results);+ }+ if constexpr (Tile::OUTPUT_ROWS >= METAL_SIMD_SIZE) {+ static_assert(Tile::K_SPLIT == 1, "RHT reuses shared results after Reduce");threadgroup_barrier(mem_flags::mem_threadgroup);-- if (simd_group == 0) {- uint global_out_idx = out_block_idx * 32 + simd_lane;- if (global_out_idx < out_vec_size) {- DT transformed = simdgroup_output_random_hadamard_transform(- static_cast<ushort>(simd_lane),- static_cast<DT>(shared_results[simd_lane]),- hadamard_factors[global_out_idx]+ constexpr uint OUTPUT_BLOCKS = Tile::OUTPUT_ROWS / METAL_SIMD_SIZE;+ for (uint job = tile.simd_group; job < Tile::INPUT_ROWS * OUTPUT_BLOCKS; job += Tile::NUM_SIMDGROUPS) {+ const uint input_index = job / OUTPUT_BLOCKS;+ const uint output_block = job % OUTPUT_BLOCKS;+ const uint input_row = tile.input_row + input_index;+ const uint global_row = tile.tile_row + output_block * METAL_SIMD_SIZE + tile.simd_lane;+ if ((FULL_TILE || input_row < params.batch_size) && (FULL_TILE || global_row < params.out_vec_size)) {+ DT value = simdgroup_output_random_hadamard_transform(+ static_cast<ushort>(tile.simd_lane),+ static_cast<DT>(+ shared_results[input_index * Tile::OUTPUT_ROWS + output_block * METAL_SIMD_SIZE + tile.simd_lane]+ ),+ ops.hadamard_factors[global_row]);- if (is_bias) {- transformed += static_cast<DT>(output_bias[global_out_idx]);+ if (params.output_transform.contains(GemmDTransform::BIAS)) {+ value += static_cast<DT>(ops.output_bias[global_row]);}- d[simd_lane] = transformed;+ ops.d[input_row * params.out_vec_size + global_row] = value;}}- } else {- if (writer && simd_lane == 0) {- METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- if (out_row + row < out_vec_size) {- d[row] = static_cast<DT>(result[row]);- }+ }+ }++private:+ template <bool STAGE_HADAMARD>+ static METAL_FUNC void write_results(+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],+ const thread GemvOperands<AT, BT, DT>& ops,+ const thread GemvParams& params,+ const thread OutputTile<Tile, FULL_TILE>& tile,+ threadgroup U* shared_results+ ) {+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {+ constexpr uint I = decltype(input_index)::value;+ const uint input_row = tile.input_row + I;+ if (!(FULL_TILE || input_row < params.batch_size)) {+ return;+ }+ device DT* output = ops.d + input_row * params.out_vec_size + tile.row0;+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ const uint global_row = tile.row0 + R;+ if (!OutputTile<Tile, FULL_TILE>::row_in_range(global_row, params.out_vec_size)) {+ return;+ }+ if (tile.clamped && params.output_transform.contains(GemmDTransform::ACCUMULATE)) {+ return;+ }+ if constexpr (STAGE_HADAMARD) {+ shared_results[I * Tile::OUTPUT_ROWS + tile.local_row + R] =+ transform<false>(result[I][R], output + R, global_row, ops, params);+ } else {+ output[R] = static_cast<DT>(transform<true>(result[I][R], output + R, global_row, ops, params));}+ });+ });+ }++ template <bool APPLY_BIAS>+ static METAL_FUNC U transform(+ U value,+ const device DT* output,+ uint global_row,+ const thread GemvOperands<AT, BT, DT>& ops,+ const thread GemvParams& params+ ) {+ if (params.output_transform.contains(GemmDTransform::SCALE)) {+ value *= params.ab_scale;+ }+ if (params.output_transform.contains(GemmDTransform::ACCUMULATE)) {+ value += static_cast<U>(*output);+ }+ if constexpr (APPLY_BIAS) {+ if (params.output_transform.contains(GemmDTransform::BIAS)) {+ value += static_cast<U>(ops.output_bias[global_row]);}}+ if (params.output_transform.contains(GemmDTransform::SOFT_CAP)) {+ value = apply_soft_cap(value, params.soft_cap);+ }+ return value;}};diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/full_precision_b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/full_precision_b_source.hindex 9305379a5..3dc7e1f48 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/full_precision_b_source.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/full_precision_b_source.h@@ -1,67 +1,75 @@#pragma once-#include "../../common/defines.h"+#include "arguments.h"+#include "tile.h"namespace uzu {namespace gemm {-template <typename BT, typename AT, typename U, uint RESULTS_PER_SIMDGROUP, uint K_SPLIT, bool INPUT_ALIGNED>+template <typename Tile, typename AT, typename BT, typename DT, bool INPUT_ALIGNED, bool FULL_TILE>struct FullPrecisionBSource {+ using U = float;+static METAL_FUNC void accumulate(- thread U (&result)[RESULTS_PER_SIMDGROUP],- const device uint32_t* b,- const device AT* a,- const device uint* gather_indices,- bool gathered,- uint in_vec_size,- uint out_vec_size,- uint out_row,- uint batch_idx,- uint simd_lane,- uint k_slice+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],+ const thread GemvOperands<AT, BT, DT>& ops,+ const thread GemvParams& params,+ const thread OutputTile<Tile, FULL_TILE>& tile) {- constexpr uint values_per_thread = 4;- constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;- typedef vec<BT, 4> W4;- typedef vec<AT, 4> I4;-- const uint k_stride = K_SPLIT * block_size;- const uint k_start = k_slice * block_size;- const uint thread_k = simd_lane * values_per_thread + k_start;- const device AT* input = a + batch_idx * in_vec_size + thread_k;+ static_assert(Tile::INPUT_ROWS == 1, "full-precision GEMV uses one input row");+ constexpr uint VALUES_PER_THREAD = 4;+ constexpr uint BLOCK_SIZE = VALUES_PER_THREAD * METAL_SIMD_SIZE;+ using W4 = vec<BT, 4>;+ using I4 = vec<AT, 4>;+ const uint k_stride = Tile::K_SPLIT * BLOCK_SIZE;+ const uint k_start = tile.k_slice * BLOCK_SIZE;+ const uint thread_k = tile.reduction_lane * VALUES_PER_THREAD + k_start;+ const uint weights_row_stride = params.in_vec_size;+ const uint input_row = tile.input_row;+ const device AT* input = ops.a + input_row * params.in_vec_size + thread_k;+ thread const device BT* weight_rows[Tile::ROWS_PER_LANE];+ const uint last_row = params.out_vec_size > 0 ? params.out_vec_size - 1 : 0;- // One advancing weight pointer per output row; base row = out_row (dense) or gather index.- const uint base_row = gathered ? 0u : out_row;- const device BT* weights = reinterpret_cast<const device BT*>(b);- weights += base_row * in_vec_size + thread_k;- thread const device BT* weight_rows[RESULTS_PER_SIMDGROUP];METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- const uint addr_row = gathered ? gather_indices[batch_idx * out_vec_size + out_row + row] : row;- weight_rows[row] = weights + addr_row * in_vec_size;+ for (uint output_index = 0; output_index < Tile::ROWS_PER_LANE; output_index++) {+ const uint global_row = tile.row0 + output_index;+ const uint lookup_row = FULL_TILE ? global_row : min(global_row, last_row);+ const uint weight_row =+ params.gathered ? ops.gather_indices[input_row * params.out_vec_size + lookup_row] : lookup_row;+ weight_rows[output_index] =+ reinterpret_cast<const device BT*>(ops.b) + weight_row * weights_row_stride + thread_k;}uint k = k_start;- for (; k + block_size <= in_vec_size; k += k_stride) {- float4 input_values = static_cast<float4>(*reinterpret_cast<const device I4*>(input));+ for (; k + BLOCK_SIZE <= params.in_vec_size; k += k_stride) {+ const float4 input_values = static_cast<float4>(*reinterpret_cast<const device I4*>(input));METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- result[row] += dot(static_cast<float4>(*reinterpret_cast<const device W4*>(weight_rows[row])), input_values);- weight_rows[row] += k_stride;+ for (uint output_index = 0; output_index < Tile::ROWS_PER_LANE; output_index++) {+ const uint global_row = tile.row0 + output_index;+ if (FULL_TILE || global_row < params.out_vec_size) {+ result[0][output_index] +=+ dot(static_cast<float4>(*reinterpret_cast<const device W4*>(weight_rows[output_index])), input_values);+ }+ weight_rows[output_index] += k_stride;}input += k_stride;}- if constexpr (K_SPLIT == 1 && !INPUT_ALIGNED) {- const uint thread_offset = simd_lane * values_per_thread;- const int remaining = (k + thread_offset < in_vec_size) ? min(static_cast<int>(in_vec_size - k - thread_offset),- static_cast<int>(values_per_thread))- : 0;+ if constexpr (Tile::K_SPLIT == 1 && !INPUT_ALIGNED) {+ const uint thread_offset = tile.reduction_lane * VALUES_PER_THREAD;+ const int remaining =+ k + thread_offset < params.in_vec_size+ ? min(static_cast<int>(params.in_vec_size - k - thread_offset), static_cast<int>(VALUES_PER_THREAD))+ : 0;if (remaining > 0) {METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- for (int index = 0; index < remaining; index++) {- result[row] += static_cast<U>(weight_rows[row][index]) * static_cast<U>(input[index]);+ for (uint output_index = 0; output_index < Tile::ROWS_PER_LANE; output_index++) {+ const uint global_row = tile.row0 + output_index;+ if (OutputTile<Tile, FULL_TILE>::row_in_range(global_row, params.out_vec_size)) {+ for (int index = 0; index < remaining; index++) {+ result[0][output_index] +=+ static_cast<U>(weight_rows[output_index][index]) * static_cast<U>(input[index]);+ }}}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/output_tile.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/output_tile.hdeleted file mode 100644index bc25f94c3..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/output_tile.h+++ /dev/null@@ -1,29 +0,0 @@-#pragma once--namespace uzu {-namespace gemm {--template <uint K_SPLIT, uint NUM_SIMDGROUPS, uint RESULTS_PER_SIMDGROUP>-struct OutputTile {- uint out_row;- uint row_group;- uint k_slice;- bool writer;-- static METAL_FUNC OutputTile make(uint out_block_idx, uint simd_group, uint out_vec_size) {- constexpr uint rows_per_threadgroup = (NUM_SIMDGROUPS / K_SPLIT) * RESULTS_PER_SIMDGROUP;- const uint row_group = simd_group / K_SPLIT;- const uint k_slice = simd_group % K_SPLIT;- uint out_row = out_block_idx * rows_per_threadgroup + row_group * RESULTS_PER_SIMDGROUP;- out_row = out_vec_size > RESULTS_PER_SIMDGROUP ? min(out_row, out_vec_size - RESULTS_PER_SIMDGROUP) : 0u;- OutputTile tile;- tile.out_row = out_row;- tile.row_group = row_group;- tile.k_slice = k_slice;- tile.writer = (K_SPLIT == 1) || (k_slice == 0);- return tile;- }-};--} // namespace gemm-} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.hnew file mode 100644index 000000000..6df334cdb--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h@@ -0,0 +1,62 @@+#pragma once++#include "arguments.h"+#include "quant_slice.h"+#include "tile.h"++namespace uzu {+namespace gemm {++template <+ typename Tile,+ typename AT,+ typename BT,+ typename DT,+ GemmBPrologueKind B_PROLOGUE,+ uint GROUP_SIZE,+ uint BITS,+ bool INPUT_ALIGNED,+ bool FULL_TILE>+struct QuantBSource {+ using U = float;+ using Slice = QuantSlice<Tile, AT, BT, DT, B_PROLOGUE, GROUP_SIZE, BITS, INPUT_ALIGNED, FULL_TILE>;+ using Metadata = QuantMetadata<Tile, AT, BT, DT, B_PROLOGUE, BITS>;++ static METAL_FUNC void accumulate(+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],+ const thread GemvOperands<AT, BT, DT>& ops,+ const thread GemvParams& params,+ const thread OutputTile<Tile, FULL_TILE>& tile+ ) {+ const uint groups = Metadata::group_count(params, GROUP_SIZE);+ const uint row_stride = Slice::row_stride(params);+ const uint group_slot = tile.reduction_lane / Tile::GROUP_LANES;+ const uint group_offset = (tile.reduction_lane % Tile::GROUP_LANES) * Slice::VALUES_PER_LANE;+ uint weight_row_indices[Tile::ROWS_PER_LANE];+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ const uint output_row = tile.row0 + R;+ weight_row_indices[R] =+ params.gathered ? ops.gather_indices[tile.input_row * params.out_vec_size + output_row] : output_row;+ });++ const device uint8_t* weights = reinterpret_cast<const device uint8_t*>(ops.b);+ const uint batch_remaining = params.batch_size - tile.input_row;+ uint group = group_slot;++ Slice current;+ QuantPosition position = {group, 0};+ while (position.valid(groups)) {+ Metadata metadata;+ metadata.load(position.group, groups, weight_row_indices, ops, params);+ for (position.slice = 0; position.slice < Slice::SLICES_PER_LANE; position.slice++) {+ current.load_weights(position, weights, weight_row_indices, row_stride, group_offset);+ current.accumulate(result, position, ops, params, tile, group_offset, batch_remaining, metadata);+ }+ position.group += Tile::GROUPS_PER_STEP;+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.hnew file mode 100644index 000000000..75e58a995--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h@@ -0,0 +1,120 @@+#pragma once++#include "../../common/quant_unpack.h"++namespace uzu {+namespace gemm {++template <uint BITS>+struct QuantChunk {+ UZU_CONST uint VALUES = 8;+ UZU_CONST uint FLOAT_MANTISSA_BITS = 23;+ UZU_CONST uint FLOAT_EXPONENT_BIAS = 127;+ UZU_CONST uint MANTISSA_BASE_BITS = (FLOAT_EXPONENT_BIAS + FLOAT_MANTISSA_BITS) << FLOAT_MANTISSA_BITS;+ UZU_CONST float MANTISSA_BASE = float(1u << FLOAT_MANTISSA_BITS);+ UZU_CONST uint BITS_PER_BYTE = 8;+ UZU_CONST uint WORD_BITS = 32;+ UZU_CONST uint WORD_LANES = 4;+ UZU_CONST uint WORD_BYTES = WORD_BITS / BITS_PER_BYTE;+ UZU_CONST uint VECTOR_BYTES = WORD_BYTES * WORD_LANES;+ UZU_CONST uint BF16_BITS = 16;+ UZU_CONST uint BF16_VALUES_PER_WORD = WORD_BITS / BF16_BITS;++ template <uint WORDS>+ static METAL_FUNC void decode(+ const thread uint4 (&words)[WORDS],+ uint chunk,+ thread float (&values)[VALUES],+ float origin,+ bool signed_codes+ ) {+ const uint word = chunk * (BITS == 4 ? 1u : 2u);+ const uint word0 = words[word / 4u][word % 4u];+ const uint word1 = BITS == 4 ? 0u : words[(word + 1u) / 4u][(word + 1u) % 4u];+ decode_words(word0, word1, values, origin, signed_codes);+ }++ template <bool ALIGNED, typename AT>+ static METAL_FUNC void load(+ const device AT* source,+ thread float (&values)[VALUES],+ uint offset,+ uint extent,+ uint row+ ) {+ static_assert(+ metal::is_same_v<AT, bfloat> || metal::is_same_v<AT, float>,+ "GEMV quantized input must be bfloat or float; half cannot use bfloat widening"+ );+ constexpr bool EXPANDS_BF16 = metal::is_same_v<AT, bfloat>;+ if constexpr (ALIGNED) {+ load_values<false, EXPANDS_BF16>(source, values, VALUES);+ } else {+ const uint valid = offset < extent ? min(extent - offset, VALUES) : 0u;+ const bool source_aligned = !EXPANDS_BF16 || ((row * extent + offset) & 7u) == 0;+ if (valid == VALUES && source_aligned) {+ load_values<false, EXPANDS_BF16>(source, values, VALUES);+ } else {+ load_values<true, EXPANDS_BF16>(source, values, valid);+ }+ }+ }++private:+ static METAL_FUNC void decode_words(+ uint word0,+ uint word1,+ thread float (&values)[VALUES],+ float origin,+ bool signed_codes+ ) {+ constexpr uint CODES_PER_WORD = WORD_BITS / BITS;+ constexpr uint W4_SIGN_BITS = 0x88888888u;+ constexpr uint W8_SIGN_BITS = 0x80808080u;+ if constexpr (BITS == 4) {+ word0 ^= signed_codes ? W4_SIGN_BITS : 0u;+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < VALUES; i++) {+ values[i] = as_type<float>(extract_bits(word0, BITS * i, BITS) | MANTISSA_BASE_BITS) - origin;+ }+ } else {+ const uint mask = signed_codes ? W8_SIGN_BITS : 0u;+ word0 ^= mask;+ word1 ^= mask;+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < CODES_PER_WORD; i++) {+ values[i] = as_type<float>(extract_bits(word0, BITS * i, BITS) | MANTISSA_BASE_BITS) - origin;+ values[CODES_PER_WORD + i] = as_type<float>(extract_bits(word1, BITS * i, BITS) | MANTISSA_BASE_BITS) - origin;+ }+ }+ }++ template <bool BOUNDED, bool EXPANDS_BF16, typename AT>+ static METAL_FUNC void load_values(const device AT* source, thread float (&values)[VALUES], uint valid) {+ if constexpr (!BOUNDED && EXPANDS_BF16) {+ constexpr uint BF16_UPPER_BITS = 0xffff0000u;+ const uint4 packed = *reinterpret_cast<const device uint4*>(source);+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < WORD_LANES; i++) {+ values[BF16_VALUES_PER_WORD * i] = as_type<float>(packed[i] << BF16_BITS);+ values[BF16_VALUES_PER_WORD * i + 1] = as_type<float>(packed[i] & BF16_UPPER_BITS);+ }+ } else {+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < VALUES; i++) {+ float value = 0.0f;+ if constexpr (BOUNDED) {+ if (i < valid) {+ value = float(source[i]);+ }+ } else {+ value = float(source[i]);+ }+ values[i] = value;+ }+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_metadata.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_metadata.hnew file mode 100644index 000000000..1ef1039c6--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_metadata.h@@ -0,0 +1,87 @@+#pragma once++#include "arguments.h"+#include "quant_chunk.h"+#include "tile.h"++namespace uzu {+namespace gemm {++template <typename Tile, typename AT, typename BT, typename DT, GemmBPrologueKind B_PROLOGUE, uint BITS>+struct QuantMetadata {+private:+ float scale[Tile::ROWS_PER_LANE];+ float origin[Tile::ROWS_PER_LANE];+ float bias[Tile::ROWS_PER_LANE];+ bool signed_codes;++public:+ static METAL_FUNC uint group_count(const thread GemvParams& params, uint group_size) {+ return (params.in_vec_size + group_size - 1) / group_size;+ }++ METAL_FUNC void load(+ uint group_index,+ uint group_count,+ const thread uint (&weight_rows)[Tile::ROWS_PER_LANE],+ const thread GemvOperands<AT, BT, DT>& ops,+ const thread GemvParams& params+ ) thread {+ signed_codes = params.signed_codes;+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ const uint row = weight_rows[R];+ scale[R] = float(ops.scales[row * group_count + group_index]);+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {+ constexpr uint ZERO_POINTS_PER_BYTE = QuantChunk<BITS>::BITS_PER_BYTE / BITS;+ const uint byte_index = group_index / ZERO_POINTS_PER_BYTE;+ const uint8_t packed = ops.zero_points[row * zero_point_row_stride<BITS>(group_count) + byte_index];+ origin[R] = QuantChunk<BITS>::MANTISSA_BASE + float(decode_zero_point<BITS>(packed, group_index));+ bias[R] = 0.0f;+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant) {+ origin[R] = QuantChunk<BITS>::MANTISSA_BASE + float(symmetric_zero_point<BITS>());+ bias[R] = 0.0f;+ } else {+ static_assert(B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant);+ origin[R] = QuantChunk<BITS>::MANTISSA_BASE;+ bias[R] = float(ops.biases[row * group_count + group_index]);+ }+ });+ }++ template <uint WORDS>+ METAL_FUNC void decode(+ const thread uint4 (&words)[WORDS],+ uint chunk,+ uint row,+ thread float (&values)[QuantChunk<BITS>::VALUES]+ ) const thread {+ QuantChunk<BITS>::decode(words, chunk, values, origin[row], signed_codes);+ }++ METAL_FUNC void fold(+ thread float (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],+ const thread float (&partial)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],+ const thread float (&input_sum)[Tile::INPUT_ROWS]+ ) const thread {+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {+ constexpr uint I = decltype(input_index)::value;+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ result[I][R] = finish(result[I][R], partial[I][R], input_sum[I], R);+ });+ });+ }++private:+ METAL_FUNC float finish(float prior, float partial, float input_sum, uint row) const thread {+ float result = fma(scale[row], partial, prior);+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {+ result = fma(bias[row], input_sum, result);+ }+ return result;+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.hnew file mode 100644index 000000000..fef42725d--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h@@ -0,0 +1,170 @@+#pragma once++#include "arguments.h"+#include "quant_chunk.h"+#include "quant_metadata.h"+#include "tile.h"++namespace uzu {+namespace gemm {++template <+ typename Tile,+ typename AT,+ typename BT,+ typename DT,+ GemmBPrologueKind B_PROLOGUE,+ uint GROUP_SIZE,+ uint BITS,+ bool INPUT_ALIGNED,+ bool FULL_TILE>+struct QuantSlice {+ using U = float;++public:+ UZU_CONST uint VALUES_PER_LANE = GROUP_SIZE / Tile::GROUP_LANES;++private:+ UZU_CONST uint CHUNK_VALUES = QuantChunk<BITS>::VALUES;+ UZU_CONST uint MAX_SLICE_VALUES = 32;+ UZU_CONST uint SLICE_VALUES = VALUES_PER_LANE > MAX_SLICE_VALUES ? MAX_SLICE_VALUES : VALUES_PER_LANE;++public:+ UZU_CONST uint SLICES_PER_LANE = VALUES_PER_LANE / SLICE_VALUES;++private:+ UZU_CONST uint SLICE_BYTES = SLICE_VALUES * BITS / QuantChunk<BITS>::BITS_PER_BYTE;+ UZU_CONST uint SLICE_WORDS = (SLICE_BYTES + QuantChunk<BITS>::VECTOR_BYTES - 1) / QuantChunk<BITS>::VECTOR_BYTES;+ UZU_CONST uint CHUNKS_PER_SLICE = SLICE_VALUES / CHUNK_VALUES;++ static_assert(GROUP_SIZE % Tile::GROUP_LANES == 0, "group lanes must divide the quantization group");+ static_assert(VALUES_PER_LANE % CHUNK_VALUES == 0, "group lane slices must contain complete chunks");+ static_assert(SLICE_VALUES <= MAX_SLICE_VALUES, "QMV slice exceeds its staged-value limit");+ static_assert(VALUES_PER_LANE % SLICE_VALUES == 0, "a lane loads complete group slices");++ uint4 weights[Tile::ROWS_PER_LANE][SLICE_WORDS];++public:+ static METAL_FUNC uint row_stride(const thread GemvParams& params) {+ return params.in_vec_size * BITS / QuantChunk<BITS>::BITS_PER_BYTE;+ }++ METAL_FUNC void load_weights(+ const thread QuantPosition& position,+ const device uint8_t* weights_base,+ const thread uint (&weight_row_indices)[Tile::ROWS_PER_LANE],+ uint row_stride,+ uint group_offset+ ) thread {+ const uint k = position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES;+ const uint offset_bytes = k * BITS / QuantChunk<BITS>::BITS_PER_BYTE;+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ const uint row = weight_row_indices[R];+ const device uint8_t* source = weights_base + row * row_stride + offset_bytes;+ if constexpr (INPUT_ALIGNED) {+ load_words<false>(weights[R], source);+ } else {+ const uint valid_bytes = offset_bytes < row_stride ? min(row_stride - offset_bytes, SLICE_BYTES) : 0u;+ constexpr uint LOAD_ALIGNMENT =+ SLICE_BYTES < QuantChunk<BITS>::VECTOR_BYTES ? SLICE_BYTES : QuantChunk<BITS>::VECTOR_BYTES;+ const bool load_aligned = ((row * row_stride + offset_bytes) & (LOAD_ALIGNMENT - 1)) == 0;+ if (valid_bytes == SLICE_BYTES && load_aligned) {+ load_words<false>(weights[R], source);+ } else {+ load_words<true>(weights[R], source, valid_bytes);+ }+ }+ });+ }++ METAL_FUNC void accumulate(+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],+ const thread QuantPosition& position,+ const thread GemvOperands<AT, BT, DT>& ops,+ const thread GemvParams& params,+ const thread OutputTile<Tile, FULL_TILE>& tile,+ uint group_offset,+ uint batch_remaining,+ const thread QuantMetadata<Tile, AT, BT, DT, B_PROLOGUE, BITS>& metadata+ ) const thread {+ float partial[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE] = {{0}};+ float input_sum[Tile::INPUT_ROWS] = {0};+ METAL_PRAGMA_UNROLL+ for (uint chunk = 0; chunk < CHUNKS_PER_SLICE; chunk++) {+ float weight_values[Tile::ROWS_PER_LANE][CHUNK_VALUES];+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ metadata.decode(weights[R], chunk, R, weight_values[R]);+ });+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {+ constexpr uint I = decltype(input_index)::value;+ const uint source_row = FULL_TILE ? I : min(I, batch_remaining - 1);+ const uint k =+ position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES + chunk * CHUNK_VALUES;+ const uint input_row = tile.input_row + source_row;+ const device AT* input = ops.a + input_row * params.in_vec_size + k;+ float input_values[CHUNK_VALUES];+ QuantChunk<BITS>::template load<INPUT_ALIGNED>(input, input_values, k, params.in_vec_size, input_row);+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < CHUNK_VALUES; i++) {+ input_sum[I] += input_values[i];+ }+ }+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < CHUNK_VALUES; i++) {+ partial[I][R] = fma(input_values[i], weight_values[R][i], partial[I][R]);+ }+ });+ });+ }+ metadata.fold(result, partial, input_sum);+ }++private:+ template <bool BOUNDED>+ static METAL_FUNC void load_words(+ thread uint4 (&words)[SLICE_WORDS],+ const device uint8_t* source,+ uint valid_bytes = SLICE_BYTES+ ) {+ if constexpr (!BOUNDED && SLICE_BYTES == QuantChunk<BITS>::WORD_BYTES) {+ const uint packed = *reinterpret_cast<const device uint*>(source);+ words[0] = uint4(packed, 0u, 0u, 0u);+ } else if constexpr (!BOUNDED && SLICE_BYTES == 2 * QuantChunk<BITS>::WORD_BYTES) {+ const uint2 packed = *reinterpret_cast<const device uint2*>(source);+ words[0] = uint4(packed[0], packed[1], 0u, 0u);+ } else if constexpr (!BOUNDED) {+ static_assert(SLICE_BYTES % QuantChunk<BITS>::VECTOR_BYTES == 0, "large slices contain complete uint4 words");+ const device uint4* packed = reinterpret_cast<const device uint4*>(source);+ METAL_PRAGMA_UNROLL+ for (uint word = 0; word < SLICE_WORDS; word++) {+ words[word] = packed[word];+ }+ } else {+ METAL_PRAGMA_UNROLL+ for (uint word = 0; word < SLICE_WORDS; word++) {+ uint4 packed = uint4(0u);+ METAL_PRAGMA_UNROLL+ for (uint component = 0; component < QuantChunk<BITS>::WORD_LANES; component++) {+ uint value = 0u;+ METAL_PRAGMA_UNROLL+ for (uint byte = 0; byte < QuantChunk<BITS>::WORD_BYTES; byte++) {+ const uint index = QuantChunk<BITS>::VECTOR_BYTES * word + QuantChunk<BITS>::WORD_BYTES * component + byte;+ if (index < SLICE_BYTES && index < valid_bytes) {+ value |= uint(source[index]) << (QuantChunk<BITS>::BITS_PER_BYTE * byte);+ }+ }+ packed[component] = value;+ }+ words[word] = packed;+ }+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_b_source.hdeleted file mode 100644index cda4f41a3..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_b_source.h+++ /dev/null@@ -1,115 +0,0 @@-#pragma once--#include "../../common/qdot.h"-#include "../../common/quant_pack.h"-#include "quantized_row_state.h"--namespace uzu {-namespace gemm {--template <- typename BT,- typename AT,- typename U,- GemmBPrologueKind B_PROLOGUE,- uint GROUP_SIZE,- uint BITS,- uint RESULTS_PER_SIMDGROUP,- bool INPUT_ALIGNED>-struct QuantizedBSource {- static METAL_FUNC void accumulate(- thread U (&result)[RESULTS_PER_SIMDGROUP],- const device uint32_t* b,- const device BT* scales,- const device uint8_t* zero_points,- const device BT* biases,- const device AT* a,- const device uint* gather_indices,- bool gathered,- uint in_vec_size,- uint out_vec_size,- uint out_row,- uint batch_idx,- uint simd_lane,- const bool signed_codes- ) {- constexpr uint pack_factor = get_pack_factor<BITS, 32>();- constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();- constexpr uint packs_per_thread = 2;- constexpr uint values_per_thread = pack_factor * packs_per_thread;- constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;- constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;- using RowState = QuantizedRowState<BT, U, B_PROLOGUE, BITS, RESULTS_PER_SIMDGROUP>;- using RowParams = typename RowState::Params;-- const uint weights_row_stride = in_vec_size * bytes_per_pack / pack_factor;- const uint group_count = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;- const uint group_offset = simd_lane / scale_step_per_thread;-- // Base row = out_row (dense) or 0 (gather); rows indexed relative to it.- const uint base_row = gathered ? 0u : out_row;-- const device uint8_t* weights = reinterpret_cast<const device uint8_t*>(b);- weights += base_row * weights_row_stride + simd_lane * packs_per_thread * bytes_per_pack;-- RowState row_state(scales, zero_points, biases, base_row, group_count, group_offset);-- const device AT* input = a + batch_idx * in_vec_size + simd_lane * values_per_thread;- thread U input_values[values_per_thread];-- uint k = 0;- for (; k + block_size <= in_vec_size; k += block_size) {- U input_sum = load_vector<AT, U, values_per_thread, BITS>(input, input_values, signed_codes);-- RowParams row_params;- row_state.load(row_params, gather_indices, gathered, batch_idx, out_vec_size, out_row);- METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- const uint addr_row = gathered ? gather_indices[batch_idx * out_vec_size + out_row + row] : row;- const device uint8_t* weight_row = weights + addr_row * weights_row_stride;- result[row] += qdot<U, values_per_thread, BITS>(- weight_row,- input_values,- row_params.scale[row],- row_params.offset[row],- input_sum,- signed_codes- );- }-- weights += block_size * bytes_per_pack / pack_factor;- row_state.advance(block_size / GROUP_SIZE);- input += block_size;- }-- if constexpr (!INPUT_ALIGNED) {- const uint thread_offset = simd_lane * values_per_thread;- const int remaining = (k + thread_offset < in_vec_size) ? min(static_cast<int>(in_vec_size - k - thread_offset),- static_cast<int>(values_per_thread))- : 0;- if (remaining > 0) {- U input_sum = load_vector_safe<AT, U, values_per_thread>(input, input_values, remaining);-- RowParams row_params;- row_state.load(row_params, gather_indices, gathered, batch_idx, out_vec_size, out_row);- METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- const uint addr_row = gathered ? gather_indices[batch_idx * out_vec_size + out_row + row] : row;- const device uint8_t* weight_row = weights + addr_row * weights_row_stride;- result[row] += qdot_safe<U, values_per_thread, BITS>(- weight_row,- input_values,- row_params.scale[row],- row_params.offset[row],- input_sum,- remaining,- signed_codes- );- }- }- }- }-};--} // namespace gemm-} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_row_state.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_row_state.hdeleted file mode 100644index e62b01d62..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quantized_row_state.h+++ /dev/null@@ -1,143 +0,0 @@-#pragma once--#include "../../../generated/gemm.h"-#include "../../common/defines.h"-#include "../../common/quant_pack.h"--namespace uzu {-namespace gemm {--template <typename U, uint RESULTS_PER_SIMDGROUP>-struct QuantizedAffineRowParams {- U scale[RESULTS_PER_SIMDGROUP];- U offset[RESULTS_PER_SIMDGROUP];-};--template <typename T, typename U>-struct QuantizedGroupRows {- const device T* values;- uint group_stride;-- QuantizedGroupRows(const device T* values_base, uint base_row, uint group_count, uint group_offset)- : values(values_base + base_row * group_count + group_offset), group_stride(group_count) {}-- METAL_FUNC U value(uint row) const { return static_cast<U>(values[row * group_stride]); }-- void advance(uint groups) { values += groups; }-};--template <typename BT, typename U, GemmBPrologueKind B_PROLOGUE, uint BITS>-struct QuantizedOffsetState;--template <typename BT, typename U, uint BITS>-struct QuantizedOffsetState<BT, U, GemmBPrologueKind::ScaleBiasDequant, BITS> {- QuantizedGroupRows<BT, U> biases;-- QuantizedOffsetState(- const device uint8_t*,- const device BT* biases_base,- uint base_row,- uint group_count,- uint group_offset- )- : biases(biases_base, base_row, group_count, group_offset) {}-- METAL_FUNC U value(uint row, U) const { return biases.value(row); }-- void advance(uint groups) { biases.advance(groups); }-};--template <typename BT, typename U, uint BITS>-struct QuantizedOffsetState<BT, U, GemmBPrologueKind::ScaleZeroPointDequant, BITS> {- const device uint8_t* zero_points;- uint zero_point_stride;- uint zero_point_shift;-- QuantizedOffsetState(- const device uint8_t* zero_points_base,- const device BT*,- uint base_row,- uint group_count,- uint group_offset- ) {- if constexpr (BITS == 4) {- zero_point_stride = (group_count + 1) / 2;- zero_points = zero_points_base + base_row * zero_point_stride + group_offset / 2;- zero_point_shift = (group_offset & 1) * 4u;- } else {- zero_point_stride = group_count;- zero_points = zero_points_base + base_row * zero_point_stride + group_offset;- }- }-- METAL_FUNC U value(uint row, U scale) const {- uint8_t zero_point_value = zero_points[row * zero_point_stride];- if constexpr (BITS == 4) {- zero_point_value = (zero_point_value >> zero_point_shift) & 0x0F;- }- return -scale * static_cast<U>(zero_point_value);- }-- void advance(uint groups) {- constexpr uint zero_points_per_byte = get_pack_factor<BITS, 8>();- zero_points += groups / zero_points_per_byte;- }-};--template <typename BT, typename U, uint BITS>-struct QuantizedOffsetState<BT, U, GemmBPrologueKind::ScaleSymmetricDequant, BITS> {- QuantizedOffsetState(const device uint8_t*, const device BT*, uint, uint, uint) {}-- METAL_FUNC U value(uint, U scale) const {- constexpr U midpoint = U(1u << (BITS - 1));- return -scale * midpoint;- }-};--template <typename BT, typename U, GemmBPrologueKind B_PROLOGUE, uint BITS, uint RESULTS_PER_SIMDGROUP>-struct QuantizedRowState {- static_assert(BITS == 4 || BITS == 8, "QMV supports 4- and 8-bit only");-- using Params = QuantizedAffineRowParams<U, RESULTS_PER_SIMDGROUP>;-- QuantizedGroupRows<BT, U> scale_rows;- QuantizedOffsetState<BT, U, B_PROLOGUE, BITS> offset_state;-- QuantizedRowState(- const device BT* scales_base,- const device uint8_t* zero_points_base,- const device BT* biases_base,- uint base_row,- uint group_count,- uint group_offset- )- : scale_rows(scales_base, base_row, group_count, group_offset),- offset_state(zero_points_base, biases_base, base_row, group_count, group_offset) {}-- // Row index: dense = local row, gather = absolute gather index.- void load(- thread Params& params,- const device uint* gather_indices,- bool gathered,- uint batch_idx,- uint out_vec_size,- uint out_row- ) const {- METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- const uint addr_row = gathered ? gather_indices[batch_idx * out_vec_size + out_row + row] : row;- const U scale = scale_rows.value(addr_row);- params.scale[row] = scale;- params.offset[row] = offset_state.value(addr_row, scale);- }- }-- void advance(uint groups) {- scale_rows.advance(groups);- if constexpr (B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant)- offset_state.advance(groups);- }-};--} // namespace gemm-} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/reduce.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/reduce.hindex 30c79fd14..6620775b3 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/reduce.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/reduce.h@@ -1,41 +1,64 @@#pragma once+#include "tile.h"+namespace uzu {namespace gemm {-template <typename U, uint K_SPLIT, uint NUM_SIMDGROUPS, uint RESULTS_PER_SIMDGROUP>+template <typename Tile, bool FULL_TILE>struct Reduce {+ using U = float;+static METAL_FUNC void run(- thread U (&result)[RESULTS_PER_SIMDGROUP],+ thread U (&result)[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE],threadgroup U* shared_results,- uint simd_group,- uint simd_lane,- uint row_group,- uint k_slice+ const thread OutputTile<Tile, FULL_TILE>& tile) {- METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- result[row] = simd_sum(result[row]);- }-- if constexpr (K_SPLIT > 1) {- if (simd_lane == 0) {- METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] = result[row];- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);- if (k_slice == 0 && simd_lane == 0) {- METAL_PRAGMA_UNROLL- for (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- U acc = 0;+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {+ constexpr uint I = decltype(input_index)::value;+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ if constexpr (Tile::REDUCTION_LANES == METAL_SIMD_SIZE) {+ result[I][R] = simd_sum(result[I][R]);+ } else {+ U reduced = result[I][R];METAL_PRAGMA_UNROLL- for (uint s = 0; s < K_SPLIT; s++) {- acc += shared_results[(row_group * K_SPLIT + s) * RESULTS_PER_SIMDGROUP + row];+ for (uint offset = Tile::REDUCTION_LANES / 2; offset > 0; offset /= 2) {+ reduced += simd_shuffle_xor(reduced, offset);}- result[row] = acc;+ result[I][R] = reduced;}+ });+ });++ if constexpr (Tile::K_SPLIT > 1) {+ if (tile.reduction_lane == 0) {+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {+ constexpr uint I = decltype(input_index)::value;+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ const uint row = tile.local_row + R;+ shared_results+ [I * Tile::OUTPUT_ROWS * Tile::K_SPLIT + tile.simd_group * Tile::SIMDGROUP_OUTPUT_ROWS ++ (row % Tile::SIMDGROUP_OUTPUT_ROWS)] = result[I][R];+ });+ });+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tile.k_slice == 0 && tile.reduction_lane == 0) {+ Tile::for_each_input_row([&](auto input_index) UZU_ALWAYS_INLINE {+ constexpr uint I = decltype(input_index)::value;+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ U sum = 0;+ for (uint slice = 0; slice < Tile::K_SPLIT; slice++) {+ const uint group = tile.row_group * Tile::K_SPLIT + slice;+ const uint row = group * Tile::SIMDGROUP_OUTPUT_ROWS + (tile.local_row % Tile::SIMDGROUP_OUTPUT_ROWS) + R;+ sum += shared_results[I * Tile::OUTPUT_ROWS * Tile::K_SPLIT + row];+ }+ result[I][R] = sum;+ });+ });}}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/tile.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/tile.hnew file mode 100644index 000000000..d980352e9--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/tile.h@@ -0,0 +1,101 @@+#pragma once++#include <metal_stdlib>++#include "../../../common/integral_constant.h"+#include "../../common/defines.h"++namespace uzu {+namespace gemm {++struct QuantPosition {+ uint group;+ uint slice;++ METAL_FUNC bool valid(uint groups) const thread { return group < groups; }+};++template <+ uint INPUT_ROW_TILE_,+ uint OUTPUT_ROW_TILE_,+ uint REDUCTION_LANES_,+ uint GROUP_LANES_,+ uint NUM_SIMDGROUPS_,+ uint K_SPLIT_>+struct GemvTile {+ // Compile-time geometry.+ UZU_CONST uint INPUT_ROWS = INPUT_ROW_TILE_;+ UZU_CONST uint OUTPUT_ROWS = OUTPUT_ROW_TILE_;+ UZU_CONST uint REDUCTION_LANES = REDUCTION_LANES_;+ UZU_CONST uint GROUP_LANES = GROUP_LANES_;+ UZU_CONST uint NUM_SIMDGROUPS = NUM_SIMDGROUPS_;+ UZU_CONST uint K_SPLIT = K_SPLIT_;+ UZU_CONST uint ROW_BLOCKS = METAL_SIMD_SIZE / REDUCTION_LANES;+ UZU_CONST uint SIMDGROUP_OUTPUT_ROWS = OUTPUT_ROWS / (NUM_SIMDGROUPS / K_SPLIT);+ UZU_CONST uint ROWS_PER_LANE = SIMDGROUP_OUTPUT_ROWS / ROW_BLOCKS;+ UZU_CONST uint GROUPS_PER_STEP = REDUCTION_LANES / GROUP_LANES;++ static_assert(INPUT_ROWS > 0 && OUTPUT_ROWS > 0, "GEMV tiles must be non-empty");+ static_assert(+ REDUCTION_LANES == 4 || REDUCTION_LANES == 8 || REDUCTION_LANES == 16 || REDUCTION_LANES == 32,+ "reduction lanes must divide one simdgroup"+ );+ static_assert(METAL_SIMD_SIZE % REDUCTION_LANES == 0, "reduction lanes must divide one simdgroup");+ static_assert(GROUP_LANES > 0 && REDUCTION_LANES % GROUP_LANES == 0, "group lanes must divide reduction lanes");+ static_assert(NUM_SIMDGROUPS > 0 && NUM_SIMDGROUPS % K_SPLIT == 0, "K split must divide simdgroups");+ static_assert(OUTPUT_ROWS % (NUM_SIMDGROUPS / K_SPLIT) == 0, "output rows must divide row groups");+ static_assert(SIMDGROUP_OUTPUT_ROWS % ROW_BLOCKS == 0, "output rows must divide reduction blocks");++ template <typename Fn>+ static METAL_FUNC void for_each_input_row(Fn fn) {+ const_for_loop<0, INPUT_ROWS, 1>(fn);+ }++ template <typename Fn>+ static METAL_FUNC void for_each_output_row(Fn fn) {+ const_for_loop<0, ROWS_PER_LANE, 1>(fn);+ }+};++template <typename Tile, bool FULL_TILE>+struct OutputTile {+ uint simd_group;+ uint simd_lane;+ uint reduction_lane;+ uint row_group;+ uint k_slice;+ uint input_row;+ uint tile_row;+ uint local_row;+ uint row0;+ bool writer;+ bool clamped;++ static METAL_FUNC OutputTile+ make(uint output_tile_idx, uint input_tile_idx, uint simd_group, uint simd_lane, uint out_vec_size) {+ const uint row_group = simd_group / Tile::K_SPLIT;+ const uint k_slice = simd_group % Tile::K_SPLIT;+ const uint row_block = simd_lane / Tile::REDUCTION_LANES;+ const uint local_row = row_group * Tile::SIMDGROUP_OUTPUT_ROWS + row_block * Tile::ROWS_PER_LANE;+ const uint unclamped = output_tile_idx * Tile::OUTPUT_ROWS + local_row;+ const uint last = out_vec_size > Tile::ROWS_PER_LANE ? out_vec_size - Tile::ROWS_PER_LANE : 0;+ OutputTile tile;+ tile.simd_group = simd_group;+ tile.simd_lane = simd_lane;+ tile.reduction_lane = simd_lane % Tile::REDUCTION_LANES;+ tile.row_group = row_group;+ tile.k_slice = k_slice;+ tile.input_row = input_tile_idx * Tile::INPUT_ROWS;+ tile.tile_row = output_tile_idx * Tile::OUTPUT_ROWS;+ tile.local_row = local_row;+ tile.row0 = FULL_TILE ? unclamped : min(unclamped, last);+ tile.writer = Tile::K_SPLIT == 1 || k_slice == 0;+ tile.clamped = !FULL_TILE && unclamped > last;+ return tile;+ }++ static METAL_FUNC bool row_in_range(uint row, uint out_vec_size) { return FULL_TILE || row < out_vec_size; }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metalindex 08de3feeb..940dbd67c 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -1,11 +1,15 @@#include "../../common/dsl.h"+#include "../../common/integral_constant.h"#include "../../generated/gemm.h"-#include "common/b_source.h"+#include "common/arguments.h"#include "common/epilogue.h"-#include "common/output_tile.h"+#include "common/full_precision_b_source.h"+#include "common/tile.h"+#include "common/quant_b_source.h"#include "common/reduce.h"using namespace metal;+using namespace uzu;using namespace uzu::gemm;template <@@ -17,7 +21,10 @@ template <uint BITS,uint K_SPLIT,bool INPUT_ALIGNED,- uint RESULTS_PER_SIMDGROUP,+ uint INPUT_ROW_TILE,+ uint OUTPUT_ROW_TILE,+ uint REDUCTION_LANES,+ uint GROUP_LANES,uint NUM_SIMDGROUPS>VARIANTS(AT, bfloat, float)VARIANTS(BT, bfloat, float)@@ -33,21 +40,75 @@ VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)VARIANTS(BITS, 0, 4, 8)VARIANTS(K_SPLIT, 1, 2, 4, 8)VARIANTS(INPUT_ALIGNED, false, true)-VARIANTS(RESULTS_PER_SIMDGROUP, 1, 2, 4, 8)+VARIANTS(INPUT_ROW_TILE, 1, 2, 3, 4, 5, 6, 7, 8)+VARIANTS(OUTPUT_ROW_TILE, 1, 2, 4, 8, 16, 32, 64)+VARIANTS(REDUCTION_LANES, 8, 16, 32)+VARIANTS(GROUP_LANES, 1, 2, 4, 8, 16)VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)+CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")-CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || K_SPLIT == 1)-CONSTRAINT(K_SPLIT <= NUM_SIMDGROUPS)-// Only selector-reachable tiles are instantiated (fleet-tuned tables): fp-// always runs 8 simdgroups with 1 or 4 rows each; non-default quantized-// tiles exist for bf16 IO only. Widen locally when sweeping new configs.-CONSTRAINT(BITS != 0 || NUM_SIMDGROUPS == 8)-CONSTRAINT(BITS != 0 || RESULTS_PER_SIMDGROUP == 1 || RESULTS_PER_SIMDGROUP == 4)+CONSTRAINT(BITS == 0 || K_SPLIT == 1)+CONSTRAINT(BITS != 0 || (INPUT_ROW_TILE == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && GROUP_LANES == 1))+CONSTRAINT(INPUT_ROW_TILE != 1 || REDUCTION_LANES == 32)+CONSTRAINT(+ INPUT_ROW_TILE == 1 ||+ (K_SPLIT == 1 && INPUT_ALIGNED && AT == "bfloat" && DT == "bfloat" && GROUP_LANES == 1))++#define GEMV_TILE(OUTPUT_ROWS, LANES, SIMD_GROUPS) \+ (OUTPUT_ROW_TILE == OUTPUT_ROWS && REDUCTION_LANES == LANES && NUM_SIMDGROUPS == SIMD_GROUPS)+#define INPUT_ROWS(FIRST, LAST) (INPUT_ROW_TILE >= FIRST && INPUT_ROW_TILE <= LAST)++CONSTRAINT(+ INPUT_ROW_TILE == 1 ||+ (((B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8) ||+ (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant && BITS == 4)) &&+ (GROUP_SIZE == 32 || GROUP_SIZE == 64) &&+ ((GEMV_TILE(16, 8, 2) && INPUT_ROWS(2, 7)) ||+ (GEMV_TILE(16, 16, 4) && INPUT_ROWS(2, 6) &&+ ((GROUP_SIZE == 32 && (BITS == 4 || INPUT_ROW_TILE >= 3)) ||+ (BITS == 4 && GROUP_SIZE == 64 && INPUT_ROW_TILE >= 3))) ||+ (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant && GEMV_TILE(8, 8, 2) &&+ ((GROUP_SIZE == 32 && (INPUT_ROW_TILE == 4 || INPUT_ROW_TILE == 8)) ||+ (GROUP_SIZE == 64 && INPUT_ROW_TILE == 2))))))++// Keep only selector-reachable geometry families.CONSTRAINT(- BITS == 0 || (NUM_SIMDGROUPS == 8 && RESULTS_PER_SIMDGROUP == 4) ||- (AT == "bfloat" && DT == "bfloat"))+ BITS == 0 ||+ (NUM_SIMDGROUPS == 2 &&+ (OUTPUT_ROW_TILE == 2 || OUTPUT_ROW_TILE == 4 || OUTPUT_ROW_TILE == 8 ||+ (INPUT_ROW_TILE > 1 && OUTPUT_ROW_TILE == 16))) ||+ (NUM_SIMDGROUPS == 4 && (OUTPUT_ROW_TILE == 8 || OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32)) ||+ (NUM_SIMDGROUPS == 8 &&+ (OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32 ||+ (BITS == 4 && INPUT_ALIGNED && OUTPUT_ROW_TILE == 64))))+CONSTRAINT(+ BITS == 0 || (AT == "bfloat" && DT == "bfloat") ||+ (NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32))+CONSTRAINT(BITS != 8 || (NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32) || INPUT_ROW_TILE > 1)+CONSTRAINT(INPUT_ALIGNED || K_SPLIT == 1)+CONSTRAINT(K_SPLIT <= NUM_SIMDGROUPS && NUM_SIMDGROUPS % K_SPLIT == 0)+CONSTRAINT(OUTPUT_ROW_TILE % (NUM_SIMDGROUPS / K_SPLIT) == 0)+CONSTRAINT(OUTPUT_ROW_TILE / (NUM_SIMDGROUPS / K_SPLIT) % (32 / REDUCTION_LANES) == 0)+CONSTRAINT(+ (BITS == 0 && (OUTPUT_ROW_TILE == (NUM_SIMDGROUPS / K_SPLIT) ||+ OUTPUT_ROW_TILE == 4 * (NUM_SIMDGROUPS / K_SPLIT))) ||+ (BITS != 0 &&+ (OUTPUT_ROW_TILE == NUM_SIMDGROUPS || OUTPUT_ROW_TILE == 2 * NUM_SIMDGROUPS ||+ OUTPUT_ROW_TILE == 4 * NUM_SIMDGROUPS || OUTPUT_ROW_TILE == 8 * NUM_SIMDGROUPS)))+CONSTRAINT(REDUCTION_LANES % GROUP_LANES == 0)+CONSTRAINT(BITS == 0 || GROUP_SIZE % GROUP_LANES == 0)+CONSTRAINT(BITS != 4 || (GROUP_SIZE / GROUP_LANES) % 16 == 0)+CONSTRAINT(BITS != 8 || (GROUP_SIZE / GROUP_LANES) % 8 == 0)+CONSTRAINT(BITS != 0 || REDUCTION_LANES == 32)+CONSTRAINT(+ BITS == 0 || INPUT_ROW_TILE > 1 || (BITS == 4 &&+ ((GROUP_SIZE == 16 && GROUP_LANES == 1) || (GROUP_SIZE == 32 && GROUP_LANES == 2) ||+ (GROUP_SIZE == 64 && GROUP_LANES == 4) || (GROUP_SIZE == 128 && GROUP_LANES == 8))) ||+ (BITS == 8 &&+ ((GROUP_SIZE == 16 && GROUP_LANES == 2) || (GROUP_SIZE == 32 && GROUP_LANES == 4) ||+ (GROUP_SIZE == 64 && GROUP_LANES == 8) || (GROUP_SIZE == 128 && GROUP_LANES == 16))))KERNEL(Gemv)(const device uint32_t* b,const device BT* scales@@ -73,60 +134,36 @@ KERNEL(Gemv)(const GemmDTransform output_transform SPECIALIZE,const bool gathered SPECIALIZE,const bool signed_codes SPECIALIZE,- threadgroup float shared_results[NUM_SIMDGROUPS * RESULTS_PER_SIMDGROUP],- const uint batch_idx GROUPS(batch_size),- const uint out_block_idx GROUPS(group_count_x),+ const bool full_tile SPECIALIZE,+ threadgroup float shared_results[INPUT_ROW_TILE * OUTPUT_ROW_TILE * K_SPLIT],+ const uint input_tile_idx GROUPS(batch_size.div_ceil(INPUT_ROW_TILE)),+ const uint output_tile_idx GROUPS(group_count_x),const uint simd_lane THREADS(32),const uint simd_group THREADS(NUM_SIMDGROUPS)) {- typedef float U;- thread U result[RESULTS_PER_SIMDGROUP] = {0};-- OutputTile<K_SPLIT, NUM_SIMDGROUPS, RESULTS_PER_SIMDGROUP> tile =- OutputTile<K_SPLIT, NUM_SIMDGROUPS, RESULTS_PER_SIMDGROUP>::make(out_block_idx, simd_group, out_vec_size);- d += batch_idx * out_vec_size + tile.out_row;-- BSource<BT, AT, U, B_PROLOGUE, GROUP_SIZE, BITS, K_SPLIT, RESULTS_PER_SIMDGROUP, INPUT_ALIGNED>::accumulate(- result,- b,- scales,- zero_points,- biases,- a,- gather_indices,- gathered,- in_vec_size,- out_vec_size,- tile.out_row,- batch_idx,- simd_lane,- tile.k_slice,- signed_codes- );+ using Ops = GemvOperands<AT, BT, DT>;+ const Ops ops = {b, scales, zero_points, biases, a, d, output_bias, hadamard_factors, gather_indices};+ const GemvParams params =+ {in_vec_size, out_vec_size, batch_size, ab_scale, soft_cap, output_transform, gathered, signed_codes};+ dispatch_bool(full_tile, [&](auto full_tile_constant) {+ constexpr bool FullTile = decltype(full_tile_constant)::value;+ using Tile = GemvTile<INPUT_ROW_TILE, OUTPUT_ROW_TILE, REDUCTION_LANES, GROUP_LANES, NUM_SIMDGROUPS, K_SPLIT>;+ const OutputTile<Tile, FullTile> tile =+ OutputTile<Tile, FullTile>::make(output_tile_idx, input_tile_idx, simd_group, simd_lane, out_vec_size);+ thread float result[Tile::INPUT_ROWS][Tile::ROWS_PER_LANE] = {{0}};- Reduce<U, K_SPLIT, NUM_SIMDGROUPS, RESULTS_PER_SIMDGROUP>::run(- result,- shared_results,- simd_group,- simd_lane,- tile.row_group,- tile.k_slice- );+ if constexpr (BITS == 0) {+ FullPrecisionBSource<Tile, AT, BT, DT, INPUT_ALIGNED, FullTile>::accumulate(result, ops, params, tile);+ } else {+ QuantBSource<Tile, AT, BT, DT, B_PROLOGUE, GROUP_SIZE, BITS, INPUT_ALIGNED, FullTile>::accumulate(+ result,+ ops,+ params,+ tile+ );+ }- Epilogue<BT, DT, U, RESULTS_PER_SIMDGROUP>::store(- result,- d,- output_bias,- hadamard_factors,- shared_results,- ab_scale,- soft_cap,- output_transform,- tile.out_row,- out_vec_size,- out_block_idx,- simd_group,- simd_lane,- tile.writer- );+ Reduce<Tile, FullTile>::run(result, shared_results, tile);+ Epilogue<Tile, AT, BT, DT, FullTile>::store(result, ops, params, tile, shared_results);+ });}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex f9326cf43..56ceb4e6a 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -1,56 +1,97 @@-use std::{- collections::{HashMap, hash_map::Entry},- sync::OnceLock,-};-use super::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK};use crate::{backends::{common::{- Allocation, BufferArg, Encoder,gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE,gemm::{GemmBPrologueKind, GemmDTransform},},- kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulShape},+ kernel::matmul::MatmulShape,},- metal::{Metal, context::MetalContext, device_profile::DeviceProfile, kernel::GemvMetalKernel},+ metal::{context::MetalContext, device_profile::DeviceProfile, error::MetalError, kernel::GemvMetalKernel},},data_type::DataType,};-const DEFAULT_GEMV_MAX_BATCH: u32 = 8;-static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();--fn max_gemv_batch_threshold() -> u32 {- *GEMV_MAX_BATCH.get_or_init(|| {- // TODO: remove magic env var- std::env::var("UZU_GEMV_MAX_BATCH").ok().and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_GEMV_MAX_BATCH)- })-}+const GEMV_MAX_BATCH: u32 = 8;#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub(crate) struct GemvSpecialization {+pub struct GemvSpecialization {b_prologue: GemmBPrologueKind,group_size: u32,bits: u32,output_transform: GemmDTransform,input_aligned: bool,k_split: u32,- results_per_simdgroup: u32,+ output_row_tile: u32,num_simdgroups: u32,+ input_row_tile: u32,+ reduction_lanes: u32,+ group_lanes: u32,gathered: bool,signed_codes: bool,+ full_tile: bool,}impl GemvSpecialization {- pub(crate) fn select_shape(+ #[cfg(test)]+ pub fn tile(self) -> policy::GemvTile {+ policy::GemvTile {+ num_simdgroups: self.num_simdgroups,+ k_split: self.k_split,+ results_per_simdgroup: self.output_row_tile / (self.num_simdgroups / self.k_split),+ input_row_tile: self.input_row_tile,+ reduction_lanes: self.reduction_lanes,+ group_lanes: self.group_lanes,+ }+ }++ pub fn select_shape(shape: &MatmulShape,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,device_profile: DeviceProfile,- ) -> Option<GemvSpecialization> {+ ) -> Option<Self> {+ let is_quant = shape.is_quant();+ let bits = shape.b_bits.unwrap_or(0);+ let bf16_io = input_data_type == DataType::BF16 && output_data_type == DataType::BF16;+ let tile = if is_quant && shape.gathered {+ policy::gathered_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n)+ } else if is_quant {+ policy::quantized_tile(+ device_profile,+ bits,+ shape.b_group_size.unwrap_or(0),+ shape.m,+ shape.n,+ shape.k,+ shape.d_transform,+ bf16_io,+ )+ } else {+ let mixed_precision = weights_data_type == DataType::F32+ && (input_data_type != DataType::F32 || output_data_type != DataType::F32);+ if mixed_precision || shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {+ return None;+ }+ let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);+ if shape.d_transform.contains(GemmDTransform::RHT) {+ Some(policy::DEFAULT_TILE)+ } else {+ Some(policy::fp_tile(shape.m, shape.n, shape.k, input_aligned, device_profile))+ }+ };+ Self::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile?)+ }++ pub fn select_tile(+ shape: &MatmulShape,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+ tile: policy::GemvTile,+ ) -> Option<Self> {if !shape.b_transpose || !shape.a_full_precision {return None;}@@ -63,24 +104,20 @@ impl GemvSpecialization {if bad_leading_dimension {return None;}- if shape.d_transform.contains(GemmDTransform::ACCUMULATE) && !shape.n.is_multiple_of(32) {- return None;- }if shape.d_transform.contains(GemmDTransform::RHT) && !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE) {return None;}- if shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > max_gemv_batch_threshold() {+ if shape.d_transform.contains(GemmDTransform::ACCUMULATE) && !shape.n.is_multiple_of(32) {return None;}+ let bits = shape.b_bits.unwrap_or(0);if !is_quant {let mixed_precision = weights_data_type == DataType::F32&& (input_data_type != DataType::F32 || output_data_type != DataType::F32);- if mixed_precision {+ if mixed_precision || shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {return None;}}-- let bits = shape.b_bits.unwrap_or(0);let block_size = if !is_quant {FP_K_BLOCK} else if bits == 4 {@@ -89,49 +126,90 @@ impl GemvSpecialization {256};let input_aligned = shape.k.is_multiple_of(block_size);- let has_rht = shape.d_transform.contains(GemmDTransform::RHT);- let bf16_io = input_data_type == DataType::BF16 && output_data_type == DataType::BF16;- let tile = if is_quant && bf16_io {- policy::quant_tile(shape.m, shape.n, shape.k, bits, has_rht, device_profile)- } else if is_quant || has_rht {- // Non-bf16 quant IO and fp+RHT keep the default tile (the only- // one instantiated for those modes).- policy::DEFAULT_TILE- } else {- policy::fp_tile(shape.m, shape.n, shape.k, input_aligned, device_profile)- };- Some(Self {+ // Gathered quantized rows cannot share one input tile.+ if is_quant && shape.gathered && tile.input_row_tile > 1 {+ return None;+ }+ let specialization = Self {b_prologue: shape.b_prologue,group_size: shape.b_group_size.unwrap_or(0),bits,output_transform: shape.d_transform,input_aligned,k_split: tile.k_split,- results_per_simdgroup: tile.results_per_simdgroup,+ output_row_tile: tile.output_row_tile(),num_simdgroups: tile.num_simdgroups,+ input_row_tile: tile.input_row_tile,+ reduction_lanes: tile.reduction_lanes,+ group_lanes: tile.group_lanes,gathered: shape.gathered,signed_codes: shape.signed_codes,- })+ full_tile: full_tile(shape, tile),+ };+ Some(specialization)+ }++ pub fn output_row_tile(&self) -> u32 {+ self.output_row_tile+ }++ fn create_pipeline(+ &self,+ context: &MetalContext,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+ ) -> Result<GemvMetalKernel, MetalError> {+ GemvMetalKernel::new(+ context,+ input_data_type,+ weights_data_type,+ output_data_type,+ self.b_prologue,+ self.group_size,+ self.bits,+ self.k_split,+ self.input_aligned,+ self.input_row_tile,+ self.output_row_tile(),+ self.reduction_lanes,+ self.group_lanes,+ self.num_simdgroups,+ self.output_transform,+ self.gathered,+ self.signed_codes,+ self.full_tile,+ )}}-fn rows_per_threadgroup(- k_split: u32,- results_per_simdgroup: u32,- num_simdgroups: u32,-) -> u32 {- (num_simdgroups / k_split) * results_per_simdgroup+fn full_tile(+ shape: &MatmulShape,+ tile: policy::GemvTile,+) -> bool {+ shape.m.is_multiple_of(tile.input_row_tile) && shape.n.is_multiple_of(tile.output_row_tile())}-pub(crate) struct GemvDispatch {+use std::collections::{HashMap, hash_map::Entry};++use crate::backends::{+ common::{+ BufferArg, Encoder,+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError},+ },+ metal::Metal,+};++/// GEMV pipelines compiled on first use.+pub struct GemvKernel {weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,pipelines: HashMap<GemvSpecialization, GemvMetalKernel>,}-impl GemvDispatch {- pub(crate) fn new(+impl GemvKernel {+ pub fn new(weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,@@ -152,29 +230,15 @@ impl GemvDispatch {match self.pipelines.entry(specialization) {Entry::Occupied(entry) => Ok(entry.into_mut()),Entry::Vacant(entry) => {- let kernel = GemvMetalKernel::new(- context,- self.input_data_type,- self.weights_data_type,- self.output_data_type,- specialization.b_prologue,- specialization.group_size,- specialization.bits,- specialization.k_split,- specialization.input_aligned,- specialization.results_per_simdgroup,- specialization.num_simdgroups,- specialization.output_transform,- specialization.gathered,- specialization.signed_codes,- )- .map_err(MatmulError::BackendError)?;+ let kernel = specialization+ .create_pipeline(context, self.weights_data_type, self.input_data_type, self.output_data_type)+ .map_err(MatmulError::BackendError)?;Ok(entry.insert(kernel))},}}- pub(crate) fn encode<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(+ pub fn encode<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(&mut self,arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,specialization: GemvSpecialization,@@ -206,88 +270,80 @@ impl GemvDispatch {});};- let group_count_x = n.div_ceil(rows_per_threadgroup(- specialization.k_split,- specialization.results_per_simdgroup,- specialization.num_simdgroups,- ));+ // Preserve each weight buffer's residency range.+ let (scales, zero_points, biases) = match &b {+ MatmulB::FullPrecision {+ ..+ } => (None, None, None),+ MatmulB::ScaleBiasDequant {+ scales,+ biases,+ ..+ } => (Some(*scales), None, Some(*biases)),+ MatmulB::ScaleZeroPointDequant {+ scales,+ zero_points,+ ..+ } => (Some(*scales), Some(*zero_points), None),+ MatmulB::ScaleSymmetricDequant {+ scales,+ ..+ } => (Some(*scales), None, None),+ };+ let output_group_count = n.div_ceil(specialization.output_row_tile());let context = encoder.context();let pipeline = self.get_or_create(context, specialization)?;-match b {MatmulB::FullPrecision {b: weights,- } => {- pipeline.encode(- weights,- None::<&Allocation<Metal>>,- None::<&Allocation<Metal>>,- None::<&Allocation<Metal>>,- (a, a_offset),- &mut *d,- output_bias,- rht_factors,- gather_indices,- k,- n,- m,- ab_scale,- group_count_x,- soft_cap,- encoder,- );- },- quant_b @ (MatmulB::ScaleBiasDequant {+ } => pipeline.encode(+ weights,+ scales,+ zero_points,+ biases,+ (a, a_offset),+ &mut *d,+ output_bias,+ rht_factors,+ gather_indices,+ k,+ n,+ m,+ ab_scale,+ output_group_count,+ soft_cap,+ encoder,+ ),+ MatmulB::ScaleBiasDequant {+ b: weights,..}| MatmulB::ScaleZeroPointDequant {+ b: weights,..}| MatmulB::ScaleSymmetricDequant {+ b: weights,..- }) => {- let (weights, scales, zero_points, biases) = match quant_b {- MatmulB::ScaleBiasDequant {- b: w,- scales,- biases,- ..- } => (w, scales, None, Some(biases)),- MatmulB::ScaleZeroPointDequant {- b: w,- scales,- zero_points,- ..- } => (w, scales, Some(zero_points), None),- MatmulB::ScaleSymmetricDequant {- b: w,- scales,- ..- } => (w, scales, None, None),- MatmulB::FullPrecision {- ..- } => unreachable!(),- };- pipeline.encode(- weights,- Some(scales),- zero_points,- biases,- (a, a_offset),- &mut *d,- output_bias,- rht_factors,- gather_indices,- k,- n,- m,- ab_scale,- group_count_x,- soft_cap,- encoder,- );- },+ } => pipeline.encode(+ weights,+ scales,+ zero_points,+ biases,+ (a, a_offset),+ &mut *d,+ output_bias,+ rht_factors,+ gather_indices,+ k,+ n,+ m,+ ab_scale,+ output_group_count,+ soft_cap,+ encoder,+ ),}Ok(())diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/mod.rsindex 7f9bc76ef..8a71486a7 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/mod.rs@@ -1,4 +1,5 @@mod kernel;mod policy;-pub(crate) use kernel::{GemvDispatch, GemvSpecialization};+pub(super) use kernel::{GemvKernel, GemvSpecialization};+pub(super) use policy::GemvTile;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rsindex 43d33ba89..c410c67ed 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs@@ -1,24 +1,33 @@-use crate::backends::metal::device_profile::{DeviceGeneration, DeviceProfile, DeviceSize};+use crate::backends::metal::device_profile::{DeviceProfile, DeviceSize, GpuFamily};++mod quantized;++pub(super) use quantized::{gathered_tile, select as quantized_tile};// Full-precision GEMV accumulates four K values per SIMD lane, so one full// vectorized K block is 4 * 32 lanes.-pub(crate) const FP_K_BLOCK: u32 = 128;-pub(crate) const DEFAULT_RESULTS_PER_SIMDGROUP: u32 = 4;-pub(crate) const DEFAULT_NUM_SIMDGROUPS: u32 = 8;+pub(super) const FP_K_BLOCK: u32 = 128;+pub(super) const DEFAULT_RESULTS_PER_SIMDGROUP: u32 = 4;+pub(super) const DEFAULT_NUM_SIMDGROUPS: u32 = 8;-/// Tile specialization selected for one GEMV dispatch.#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub(crate) struct GemvTile {+pub struct GemvTile {/// SIMD groups launched in one threadgroup.- pub num_simdgroups: u32,+ pub(super) num_simdgroups: u32,/// Number of split-K slices reduced by one threadgroup.- pub k_split: u32,+ pub(super) k_split: u32,/// Output rows computed by each SIMD group.- pub results_per_simdgroup: u32,+ pub(super) results_per_simdgroup: u32,+ /// Input rows processed by one threadgroup.+ pub(super) input_row_tile: u32,+ /// SIMD lanes cooperating over K for one output-row block.+ pub(super) reduction_lanes: u32,+ /// SIMD lanes cooperating on one quantization group.+ pub(super) group_lanes: u32,}const SMALL_G13_HUGE_N: u32 = 32768;-const SMALL_G13_WIDE_ROW_N: u32 = 6144;+const SMALL_G13_BROAD_ROW_N: u32 = 6144;const DEEP_K: u32 = 8192;const FP_LARGE_SPLIT_K_MIN_DEPTH: u32 = 4 * FP_K_BLOCK;const FP_K_DEPTH_N_MAX: u32 = 4095;@@ -34,37 +43,50 @@ const fn tile(num_simdgroups,k_split,results_per_simdgroup,+ input_row_tile: 1,+ reduction_lanes: 32,+ group_lanes: 1,}}-const fn qtile(- num_simdgroups: u32,- results_per_simdgroup: u32,-) -> GemvTile {- // Q4 table sweeps only covered KS1; quant split-K is a separate future grid.- tile(num_simdgroups, 1, results_per_simdgroup)-}+pub(super) const DEFAULT_TILE: GemvTile = tile(DEFAULT_NUM_SIMDGROUPS, 1, DEFAULT_RESULTS_PER_SIMDGROUP);++impl GemvTile {+ pub const fn quantized(+ num_simdgroups: u32,+ results_per_simdgroup: u32,+ input_row_tile: u32,+ reduction_lanes: u32,+ group_lanes: u32,+ ) -> Self {+ Self {+ num_simdgroups,+ k_split: 1,+ results_per_simdgroup,+ input_row_tile,+ reduction_lanes,+ group_lanes,+ }+ }-pub(crate) const DEFAULT_TILE: GemvTile = qtile(DEFAULT_NUM_SIMDGROUPS, DEFAULT_RESULTS_PER_SIMDGROUP);-// Qxy = qtile(num_simdgroups=x, results_per_simdgroup=y), with KS1.-const Q21: GemvTile = qtile(2, 1);-const Q22: GemvTile = qtile(2, 2);-const Q24: GemvTile = qtile(2, 4);-const Q42: GemvTile = qtile(4, 2);-const Q44: GemvTile = qtile(4, 4);-const Q48: GemvTile = qtile(4, 8);-const Q82: GemvTile = qtile(8, 2);-const QUANT_N_BUCKET_MAXES: [u32; 6] = [512, 2048, 4096, 8192, 16384, 32768];-const QUANT_K_BUCKET_MAXES: [u32; 3] = [512, 2048, 8192];-const QUANT_RHT_TUNED_N_MIN_EXCLUSIVE: u32 = 2048;-const QUANT_RHT_TUNED_N_MAX: u32 = 4096;-const QUANT_RHT_TUNED_K_MIN: u32 = 2048;--fn table_bucket_index(- value: u32,- bucket_maxes: &[u32],-) -> usize {- bucket_maxes.partition_point(|&max| value > max)+ pub const fn quantized_output_tile(+ num_simdgroups: u32,+ output_row_tile: u32,+ input_row_tile: u32,+ reduction_lanes: u32,+ group_lanes: u32,+ ) -> Self {+ assert!(output_row_tile.is_multiple_of(num_simdgroups));+ Self::quantized(num_simdgroups, output_row_tile / num_simdgroups, input_row_tile, reduction_lanes, group_lanes)+ }++ pub(super) const fn output_row_tile(self) -> u32 {+ (self.num_simdgroups / self.k_split) * self.results_per_simdgroup+ }++ pub(super) const fn rows_per_lane(self) -> u32 {+ self.results_per_simdgroup / (32 / self.reduction_lanes)+ }}fn cap_k_split_to_complete_fp_k_blocks(@@ -119,7 +141,7 @@ fn preferred_fp_k_split(/// Selects the full-precision GEMV tile. `m` is the input-vector count,/// `n` is the output row count, and `k` is the reduction depth.-pub(crate) fn fp_tile(+pub(super) fn fp_tile(m: u32,n: u32,k: u32,@@ -127,12 +149,12 @@ pub(crate) fn fp_tile(profile: DeviceProfile,) -> GemvTile {let size = profile.size();- let is_small_g13 = size == DeviceSize::Small && profile.generation() == DeviceGeneration::Legacy;- // FP sweeps covered SG2/SG4/SG8; SG changes did not produce portable- // confirmed wins, so shipped FP policy keeps SG8 and tunes KS/R only.+ let gpu_family = profile.gpu_family();+ let is_small_legacy = size == DeviceSize::Small && gpu_family == GpuFamily::Legacy;+ // SG8 is the portable full-precision geometry.let should_disable_k_split = !input_aligned|| (m == 1 && size == DeviceSize::Large && k < FP_LARGE_SPLIT_K_MIN_DEPTH)- || (m == 1 && is_small_g13 && n >= SMALL_G13_HUGE_N);+ || (m == 1 && is_small_legacy && n >= SMALL_G13_HUGE_N);let k_split = if should_disable_k_split {1@@ -140,9 +162,7 @@ pub(crate) fn fp_tile(cap_k_split_to_complete_fp_k_blocks(k, preferred_fp_k_split(m, n, k))};- // R1 won most single-row FP sweeps; Large devices only switch back to R4- // for deep-K rows, while legacy wide rows keep R4.- let results_per_simdgroup = if is_small_g13 && m == 1 && n >= SMALL_G13_WIDE_ROW_N {+ let results_per_simdgroup = if is_small_legacy && m == 1 && n >= SMALL_G13_BROAD_ROW_N {DEFAULT_RESULTS_PER_SIMDGROUP} else if m == 1 && (k <= DEEP_K || size != DeviceSize::Large) {1@@ -153,77 +173,6 @@ pub(crate) fn fp_tile(tile(DEFAULT_NUM_SIMDGROUPS, k_split, results_per_simdgroup)}-/// Selects the quantized GEMV tile. `m` is the input-vector count, `n` is the-/// output row count, `k` is the reduction depth, and `bits` is the quant width.-pub(crate) fn quant_tile(- m: u32,- n: u32,- k: u32,- bits: u32,- has_rht: bool,- profile: DeviceProfile,-) -> GemvTile {- let size = profile.size();- let generation = profile.generation();- // These tables are fitted for batch-1 Q4 only; Q8/future widths keep the- // deterministic default until they have their own cold sweep.- if m != 1 || bits != 4 {- return DEFAULT_TILE;- }- if has_rht {- // This special case mirrors quant bucket edges: n in (2048, 4096]- // and k at or above the 2048 boundary.- return if size == DeviceSize::Large- && n > QUANT_RHT_TUNED_N_MIN_EXCLUSIVE- && n <= QUANT_RHT_TUNED_N_MAX- && k >= QUANT_RHT_TUNED_K_MIN- {- qtile(4, 8)- } else {- DEFAULT_TILE- };- }-- let k_bucket = table_bucket_index(k, &QUANT_K_BUCKET_MAXES);- let n_bucket = table_bucket_index(n, &QUANT_N_BUCKET_MAXES);- // Q4 BF16 decode choices from June 2026 gemv_fine_tune sweeps; omitted- // cells keep SG8_KS1_R4. Other quant widths keep DEFAULT_TILE until swept.- let selected = match (size, generation, k_bucket, n_bucket) {- (DeviceSize::Large, _, 0, 1) => Q42,- (DeviceSize::Large, _, 1, 0) => Q21,- (DeviceSize::Large, _, 1, 1..=3) => Q22,- (DeviceSize::Large, _, 1, 4) => Q21,- (DeviceSize::Large, _, 1, 5) => Q24,- (DeviceSize::Large, _, 2, 1) => Q42,- (DeviceSize::Large, _, 3, 1) => Q22,-- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 0, 1) => Q44,- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 1, 0) => Q42,- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 1, 1) => Q22,- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 1, 2) => Q42,- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 1, 4) => Q22,- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 1, 5) => Q42,- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 2, 1) => Q42,- (DeviceSize::Small, DeviceGeneration::Apple9 | DeviceGeneration::M5Plus, 3, 1) => Q22,-- (DeviceSize::Small, DeviceGeneration::Apple8, 0, 1) => Q44,- (DeviceSize::Small, DeviceGeneration::Apple8, 1, _) | (DeviceSize::Small, DeviceGeneration::Apple8, 2, 1) => {- Q82- },-- (DeviceSize::Small, DeviceGeneration::Legacy, 0, 1) => Q48,- (DeviceSize::Small, DeviceGeneration::Legacy, 1, 0..=3) => Q82,-- _ => DEFAULT_TILE,- };- if n < selected.results_per_simdgroup {- // Coarse N buckets can include n < R; keep the default R4 tile for tiny rows.- DEFAULT_TILE- } else {- selected- }-}-#[cfg(test)]#[path = "../../../../../../unit/backends/metal/kernel/matmul/gemv/policy_test.rs"]mod tests;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rsnew file mode 100644index 000000000..56bafba17--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs@@ -0,0 +1,148 @@+use super::{DEFAULT_RESULTS_PER_SIMDGROUP, GemvTile};+use crate::backends::{+ common::gpu_types::gemm::GemmDTransform,+ metal::device_profile::{DeviceProfile, DeviceSize, GpuFamily},+};++const QUANT_N_BUCKET_MAXES: [u32; 6] = [512, 2048, 4096, 8192, 16384, 32768];+const QUANT_K_BUCKET_MAXES: [u32; 3] = [512, 2048, 8192];+const QUANT_RHT_TUNED_N_MIN_EXCLUSIVE: u32 = 2048;+const QUANT_RHT_TUNED_N_MAX: u32 = 4096;+const QUANT_RHT_TUNED_K_MIN: u32 = 2048;++/// Lane-sliced tiles split each quantization group across reduction lanes.+/// A lane stages 8 bytes: 16 W4 values or 8 W8 values.+const fn lane_tile(+ num_simdgroups: u32,+ results_per_simdgroup: u32,+ bits: u32,+ group: u32,+) -> GemvTile {+ let pack_factor = match bits {+ 4 => 8,+ 8 => 4,+ _ => 1,+ };+ GemvTile::quantized(num_simdgroups, results_per_simdgroup, 1, 32, group / (2 * pack_factor))+}++const fn lane_default(+ bits: u32,+ group: u32,+) -> GemvTile {+ lane_tile(8, DEFAULT_RESULTS_PER_SIMDGROUP, bits, group)+}++pub fn gathered_tile(+ bits: u32,+ group: u32,+ m: u32,+ n: u32,+) -> Option<GemvTile> {+ if !matches!(bits, 4 | 8) || !matches!(group, 16 | 32 | 64 | 128) || !(1..=8).contains(&m) {+ return None;+ }+ let tile = lane_default(bits, group);+ (n >= tile.rows_per_lane()).then_some(tile)+}++fn table_bucket_index(+ value: u32,+ bucket_maxes: &[u32],+) -> usize {+ bucket_maxes.partition_point(|&max| value > max)+}++fn lane_policy(+ m: u32,+ n: u32,+ k: u32,+ bits: u32,+ group: u32,+ has_rht: bool,+ profile: DeviceProfile,+) -> GemvTile {+ let size = profile.size();+ let gpu_family = profile.gpu_family();+ if m != 1 || bits != 4 {+ return lane_default(bits, group);+ }+ if has_rht {+ return if size == DeviceSize::Large+ && n > QUANT_RHT_TUNED_N_MIN_EXCLUSIVE+ && n <= QUANT_RHT_TUNED_N_MAX+ && k >= QUANT_RHT_TUNED_K_MIN+ {+ lane_tile(4, 8, bits, group)+ } else {+ lane_default(bits, group)+ };+ }++ let k_bucket = table_bucket_index(k, &QUANT_K_BUCKET_MAXES);+ let n_bucket = table_bucket_index(n, &QUANT_N_BUCKET_MAXES);+ let (num_simdgroups, results_per_simdgroup) = match (size, gpu_family, k_bucket, n_bucket) {+ (DeviceSize::Large, _, 0, 1) => (4, 2),+ (DeviceSize::Large, _, 1, 0) => (2, 1),+ (DeviceSize::Large, _, 1, 1..=3) => (2, 2),+ (DeviceSize::Large, _, 1, 4) => (2, 1),+ (DeviceSize::Large, _, 1, 5) => (2, 4),+ (DeviceSize::Large, _, 2, 1) => (4, 2),+ (DeviceSize::Large, _, 3, 1) => (2, 2),+ (DeviceSize::Small, GpuFamily::Apple9, 0, 1) => (4, 4),+ (DeviceSize::Small, GpuFamily::Apple9, 1, 0) => (4, 2),+ (DeviceSize::Small, GpuFamily::Apple9, 1, 1) => (2, 2),+ (DeviceSize::Small, GpuFamily::Apple9, 1, 2) => (4, 2),+ (DeviceSize::Small, GpuFamily::Apple9, 1, 4) => (2, 2),+ (DeviceSize::Small, GpuFamily::Apple9, 1, 5) => (4, 2),+ (DeviceSize::Small, GpuFamily::Apple9, 2, 1) => (4, 2),+ (DeviceSize::Small, GpuFamily::Apple9, 3, 1) => (2, 2),+ (DeviceSize::Small, GpuFamily::Apple8, 0, 1) => (4, 4),+ (DeviceSize::Small, GpuFamily::Apple8, 1, _) | (DeviceSize::Small, GpuFamily::Apple8, 2, 1) => (8, 2),+ (DeviceSize::Small, GpuFamily::Legacy, 0, 1) => (4, 8),+ (DeviceSize::Small, GpuFamily::Legacy, 1, 0..=3) => (8, 2),+ _ => (8, 4),+ };+ let selected = lane_tile(num_simdgroups, results_per_simdgroup, bits, group);+ if n < selected.results_per_simdgroup {+ lane_default(bits, group)+ } else {+ selected+ }+}++pub fn select(+ profile: DeviceProfile,+ bits: u32,+ group: u32,+ m: u32,+ n: u32,+ k: u32,+ d_transform: GemmDTransform,+ bf16_io: bool,+) -> Option<GemvTile> {+ if !matches!(bits, 4 | 8) || !matches!(group, 16 | 32 | 64 | 128) || !(1..=8).contains(&m) {+ return None;+ }++ if !bf16_io {+ let tile = lane_default(bits, group);+ return (m <= 4 && n >= tile.rows_per_lane()).then_some(tile);+ }++ let has_rht = d_transform.contains(GemmDTransform::RHT);+ let tile = match m {+ 1..=4 => lane_policy(m, n, k, bits, group, has_rht, profile),+ 5..=8 if n <= 64 => lane_default(bits, group),+ _ => return None,+ };+ let rows = tile.output_row_tile();+ // Input rows are tiled independently; partial tiles clamp loads and stores.+ if n < tile.rows_per_lane() {+ return None;+ }+ if has_rht && (!rows.is_multiple_of(32) || !n.is_multiple_of(rows)) {+ return None;+ }+ Some(tile)+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rsindex e90e9346c..2139f6cca 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs@@ -1,10 +1,12 @@pub mod gemm;-pub mod gemv;+mod gemv;+mod qmv;pub use self::gemm::GemmKernel;use self::{gemm::{GemmPlan, GemmProblem},- gemv::{GemvDispatch, GemvSpecialization},+ gemv::{GemvKernel, GemvSpecialization},+ qmv::QmvRoute,};use crate::{backends::{@@ -22,7 +24,7 @@ use crate::{};pub struct MatmulMetalKernel {- gemv: GemvDispatch,+ gemv: GemvKernel,pub gemm: GemmKernel,weights_data_type: DataType,input_data_type: DataType,@@ -64,42 +66,55 @@ impl MatmulMetalKernel {matches!(plan.tiling, GemmTiling::Tile16x32x256_Simdgroups1x1 | GemmTiling::Tile16x128x256_Simdgroups1x4)}- fn select_dispatch(- &self,+ fn choose_dispatch(shape: &MatmulShape,- context: &MetalContext,+ profile: crate::backends::metal::device_profile::DeviceProfile,+ supports_mxu: bool,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,) -> MatmulDispatch {- let gemv = GemvSpecialization::select_shape(- shape,- self.weights_data_type,- self.input_data_type,- self.output_data_type,- context.device_profile(),- );- let problem = GemmProblem::new(- *shape,- self.weights_data_type,- self.output_data_type,- context.supports_mxu(),- context.device_profile(),- );+ let all_bf16 = weights_data_type == DataType::BF16+ && input_data_type == DataType::BF16+ && output_data_type == DataType::BF16;+ if let Some(route) = qmv::route(profile, shape, all_bf16) {+ return match route {+ QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(+ GemvSpecialization::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile)+ .expect("typed QMV route must contain a legal GEMV tile"),+ ),+ QmvRoute::MainGemm(plan) => MatmulDispatch::Gemm(plan),+ };+ }+ let gemv =+ GemvSpecialization::select_shape(shape, weights_data_type, input_data_type, output_data_type, profile);+ let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, profile);let plan = problem.select_plan();match gemv {None => MatmulDispatch::Gemm(plan),Some(_)- if Self::prefer_gemm_over_gemv(- *shape,- plan,- self.weights_data_type,- self.input_data_type,- self.output_data_type,- ) =>+ if Self::prefer_gemm_over_gemv(*shape, plan, weights_data_type, input_data_type, output_data_type) =>{MatmulDispatch::Gemm(plan)},Some(gemv) => MatmulDispatch::Gemv(gemv),}}++ fn select_dispatch(+ &self,+ shape: &MatmulShape,+ context: &MetalContext,+ ) -> MatmulDispatch {+ Self::choose_dispatch(+ shape,+ context.device_profile(),+ context.supports_mxu(),+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,+ )+ }}impl MatmulKernel for MatmulMetalKernel {@@ -118,7 +133,7 @@ impl MatmulKernel for MatmulMetalKernel {}let gemm = GemmKernel::new(context, weights_data_type, input_data_type, output_data_type)?;- let gemv = GemvDispatch::new(weights_data_type, input_data_type, output_data_type);+ let gemv = GemvKernel::new(weights_data_type, input_data_type, output_data_type);Ok(Self {gemv,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rsnew file mode 100644index 000000000..b75f223f9--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rs@@ -0,0 +1,12 @@+mod routes;++pub(super) use routes::route;++use super::{gemm::GemmPlan, gemv::GemvTile};++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) enum QmvRoute {+ Tuned(GemvTile),+ MainGemv(GemvTile),+ MainGemm(GemmPlan),+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rsnew file mode 100644index 000000000..2ace699b2--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs@@ -0,0 +1,415 @@+// Qwen3.6-27B W4/ZP and W8/Symmetric, G32/G64.+// Tuned on M1, M2, M2 Pro, M3 Max, M4, M4 Pro, M5 Max.+use super::{+ super::{+ gemm::{GemmEngine, GemmPlan},+ gemv::GemvTile,+ },+ QmvRoute,+};+use crate::backends::{+ common::{+ gpu_types::gemm::{GemmBPrologueKind, GemmTiling},+ kernel::matmul::MatmulShape,+ },+ metal::device_profile::{DeviceIdentity, DeviceProfile},+};++const DOWN: u8 = 1 << 0;+const GATE: u8 = 1 << 1;+const GATE_UP: u8 = 1 << 2;+const PROJECTION_IN: u8 = 1 << 3;+const PROJECTION_OUT: u8 = 1 << 4;+const QKV: u8 = 1 << 5;+const READOUT: u8 = 1 << 6;++#[derive(Clone, Copy)]+struct RouteRow {+ identity: DeviceIdentity,+ bits: u32,+ group: u32,+ m: u32,+ shapes: u8,+ route: QmvRoute,+}++const fn shape(+ n: u32,+ k: u32,+) -> u8 {+ match (n, k) {+ (5120, 17408) => DOWN,+ (6144, 5120) => GATE,+ (34816, 5120) => GATE_UP,+ (16480, 5120) => PROJECTION_IN,+ (5120, 6144) => PROJECTION_OUT,+ (8192, 5120) => QKV,+ (248320, 5120) => READOUT,+ _ => 0,+ }+}++#[rustfmt::skip]+macro_rules! tuned { ($input:literal, $output:literal, $lanes:literal, $group_lanes:literal, $simdgroups:literal) => { QmvRoute::Tuned(GemvTile::quantized_output_tile($simdgroups, $output, $input, $lanes, $group_lanes)) }; }+#[rustfmt::skip]+macro_rules! main_gemv { ($input:literal, $results_per_simdgroup:literal, $lanes:literal, $group_lanes:literal, $simdgroups:literal) => { QmvRoute::MainGemv(GemvTile::quantized($simdgroups, $results_per_simdgroup, $input, $lanes, $group_lanes)) }; }+#[rustfmt::skip]+macro_rules! main_gemm { ($engine:ident, $tiling:ident, $split:literal) => { QmvRoute::MainGemm(GemmPlan { engine: GemmEngine::$engine, tiling: GemmTiling::$tiling, split_k: $split }) }; }+#[rustfmt::skip]+macro_rules! qmv_format { (w4_zp_g32) => { (4, 32) }; (w4_zp_g64) => { (4, 64) }; (w8_sym_g32) => { (8, 32) }; (w8_sym_g64) => { (8, 64) }; }+#[rustfmt::skip]+macro_rules! row { ($identity:ident, $format:ident, $m:literal, $shapes:expr, $route:expr) => { RouteRow { identity: DeviceIdentity::$identity, bits: qmv_format!($format).0, group: qmv_format!($format).1, m: $m, shapes: $shapes, route: $route } }; }++#[rustfmt::skip]+const ROWS: &[RouteRow] = &[+ row!(M1, w4_zp_g32, 2, READOUT, main_gemv!(1, 4, 32, 2, 8)),+ row!(M1, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(2, 16, 8, 1, 2)),+ row!(M1, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M1, w4_zp_g32, 4, DOWN, main_gemv!(1, 4, 32, 2, 8)),+ row!(M1, w4_zp_g32, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M1, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M1, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),+ row!(M1, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M1, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M1, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M1, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 16, 1, 4)),+ row!(M1, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),+ row!(M1, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),+ row!(M1, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M1, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M1, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),+ row!(M1, w8_sym_g32, 3, PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M1, w8_sym_g32, 4, GATE_UP, main_gemv!(1, 4, 32, 4, 8)),+ row!(M1, w8_sym_g32, 4, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M1, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M1, w8_sym_g32, 6, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M1, w8_sym_g32, 6, DOWN, tuned!(6, 16, 16, 1, 4)),+ row!(M1, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M1, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M1, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M1, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M1, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M1, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M1, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M2, w4_zp_g32, 2, PROJECTION_IN, main_gemv!(1, 4, 32, 2, 8)),+ row!(M2, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M2, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M2, w4_zp_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M2, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M2, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M2, w4_zp_g32, 7, READOUT, tuned!(4, 8, 8, 1, 2)),+ row!(M2, w4_zp_g32, 7, GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(7, 16, 8, 1, 2)),+ row!(M2, w4_zp_g32, 7, DOWN | GATE, tuned!(8, 8, 8, 1, 2)),+ row!(M2, w4_zp_g64, 2, PROJECTION_IN, tuned!(2, 8, 8, 1, 2)),+ row!(M2, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M2, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M2, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M2, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M2, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M2, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M2, w8_sym_g32, 2, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M2, w8_sym_g32, 2, DOWN, tuned!(2, 16, 8, 1, 2)),+ row!(M2, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M2, w8_sym_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M2, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M2, w8_sym_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M2, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 2, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M2, w8_sym_g64, 2, DOWN, tuned!(2, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M2, w8_sym_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g32, 2, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),+ row!(M2Pro, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g32, 3, DOWN | GATE_UP | READOUT, main_gemv!(1, 4, 32, 2, 8)),+ row!(M2Pro, w4_zp_g32, 3, PROJECTION_IN, tuned!(3, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g32, 3, GATE | PROJECTION_OUT | QKV, tuned!(3, 16, 16, 1, 4)),+ row!(M2Pro, w4_zp_g32, 4, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 2, 8)),+ row!(M2Pro, w4_zp_g32, 4, PROJECTION_IN, tuned!(4, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g32, 4, GATE, tuned!(4, 16, 16, 1, 4)),+ row!(M2Pro, w4_zp_g32, 5, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),+ row!(M2Pro, w4_zp_g32, 5, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M2Pro, w4_zp_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M2Pro, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g32, 6, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),+ row!(M2Pro, w4_zp_g32, 6, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M2Pro, w4_zp_g32, 6, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M2Pro, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g32, 7, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),+ row!(M2Pro, w4_zp_g32, 7, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M2Pro, w4_zp_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M2Pro, w4_zp_g32, 7, PROJECTION_IN, tuned!(7, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g64, 3, PROJECTION_OUT, tuned!(3, 16, 16, 1, 4)),+ row!(M2Pro, w4_zp_g64, 4, GATE_UP | PROJECTION_IN | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV, tuned!(4, 16, 16, 1, 4)),+ row!(M2Pro, w4_zp_g64, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),+ row!(M2Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M2Pro, w8_sym_g32, 3, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M2Pro, w8_sym_g32, 3, PROJECTION_IN, tuned!(3, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g32, 3, GATE, tuned!(3, 16, 16, 1, 4)),+ row!(M2Pro, w8_sym_g32, 4, DOWN | GATE_UP | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M2Pro, w8_sym_g32, 4, PROJECTION_IN, tuned!(4, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g32, 4, GATE | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),+ row!(M2Pro, w8_sym_g32, 5, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),+ row!(M2Pro, w8_sym_g32, 5, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M2Pro, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M2Pro, w8_sym_g32, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g32, 6, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),+ row!(M2Pro, w8_sym_g32, 6, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M2Pro, w8_sym_g32, 6, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M2Pro, w8_sym_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),+ row!(M2Pro, w8_sym_g32, 7, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M2Pro, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M2Pro, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M2Pro, w8_sym_g64, 3, DOWN | GATE | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 8, 8)),+ row!(M2Pro, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g64, 4, DOWN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M2Pro, w8_sym_g64, 4, GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),+ row!(M3Max, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),+ row!(M3Max, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M3Max, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),+ row!(M3Max, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),+ row!(M3Max, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),+ row!(M3Max, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),+ row!(M3Max, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g64, 2, GATE_UP | PROJECTION_IN | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M3Max, w4_zp_g64, 2, DOWN | GATE | PROJECTION_OUT, tuned!(2, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g64, 3, DOWN | GATE | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g64, 3, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M3Max, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g64, 4, GATE_UP | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),+ row!(M3Max, w4_zp_g64, 5, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g64, 5, GATE_UP, tuned!(5, 16, 16, 1, 4)),+ row!(M3Max, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),+ row!(M3Max, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),+ row!(M3Max, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),+ row!(M3Max, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),+ row!(M3Max, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M3Max, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M3Max, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),+ row!(M3Max, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M3Max, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),+ row!(M3Max, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M3Max, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M3Max, w8_sym_g64, 3, GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M3Max, w8_sym_g64, 3, DOWN | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g64, 4, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),+ row!(M3Max, w8_sym_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),+ row!(M4, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M4, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),+ row!(M4, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),+ row!(M4, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M4, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),+ row!(M4, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M4, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),+ row!(M4, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M4, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),+ row!(M4, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),+ row!(M4, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M4, w4_zp_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),+ row!(M4, w4_zp_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M4, w4_zp_g64, 4, DOWN | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M4, w4_zp_g64, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 16, 1, 4)),+ row!(M4, w4_zp_g64, 5, DOWN | GATE_UP | PROJECTION_IN | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M4, w4_zp_g64, 5, GATE | PROJECTION_OUT | QKV, tuned!(5, 16, 16, 1, 4)),+ row!(M4, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),+ row!(M4, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M4, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),+ row!(M4, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M4, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),+ row!(M4, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),+ row!(M4, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),+ row!(M4, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M4, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M4, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),+ row!(M4, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),+ row!(M4, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M4, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),+ row!(M4, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M4, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M4, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M4, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M4, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M4, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),+ row!(M4Pro, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),+ row!(M4Pro, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M4Pro, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),+ row!(M4Pro, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),+ row!(M4Pro, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),+ row!(M4Pro, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),+ row!(M4Pro, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M4Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M4Pro, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g64, 4, PROJECTION_IN | QKV, tuned!(4, 16, 16, 1, 4)),+ row!(M4Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g64, 5, PROJECTION_IN | QKV, tuned!(5, 16, 16, 1, 4)),+ row!(M4Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),+ row!(M4Pro, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),+ row!(M4Pro, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),+ row!(M4Pro, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),+ row!(M4Pro, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M4Pro, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M4Pro, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),+ row!(M4Pro, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),+ row!(M4Pro, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),+ row!(M4Pro, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),+ row!(M4Pro, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 8, 8)),+ row!(M4Pro, w8_sym_g64, 2, READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M4Pro, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g32, 2, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(2, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g32, 2, GATE_UP | READOUT, tuned!(2, 16, 16, 1, 4)),+ row!(M5Max, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g32, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),+ row!(M5Max, w4_zp_g32, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),+ row!(M5Max, w4_zp_g32, 6, GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),+ row!(M5Max, w4_zp_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),+ row!(M5Max, w4_zp_g32, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),+ row!(M5Max, w4_zp_g32, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),+ row!(M5Max, w4_zp_g32, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),+ row!(M5Max, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g64, 6, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),+ row!(M5Max, w4_zp_g64, 6, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),+ row!(M5Max, w4_zp_g64, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),+ row!(M5Max, w4_zp_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),+ row!(M5Max, w4_zp_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),+ row!(M5Max, w4_zp_g64, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),+ row!(M5Max, w4_zp_g64, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),+ row!(M5Max, w4_zp_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),+ row!(M5Max, w8_sym_g32, 2, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 4, 8)),+ row!(M5Max, w8_sym_g32, 2, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g32, 4, PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),+ row!(M5Max, w8_sym_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),+ row!(M5Max, w8_sym_g32, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),+ row!(M5Max, w8_sym_g32, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),+ row!(M5Max, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),+ row!(M5Max, w8_sym_g64, 2, GATE | GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),+ row!(M5Max, w8_sym_g64, 2, DOWN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 4, GATE | PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 6, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),+ row!(M5Max, w8_sym_g64, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),+ row!(M5Max, w8_sym_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),+ row!(M5Max, w8_sym_g64, 6, GATE_UP | PROJECTION_IN | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),+ row!(M5Max, w8_sym_g64, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),+ row!(M5Max, w8_sym_g64, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),+ row!(M5Max, w8_sym_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),+];++fn qmv_format(shape: &MatmulShape) -> Option<(u32, u32)> {+ match (shape.b_prologue, shape.b_bits, shape.b_group_size) {+ (GemmBPrologueKind::ScaleZeroPointDequant, Some(4), Some(group @ (32 | 64))) => Some((4, group)),+ (GemmBPrologueKind::ScaleSymmetricDequant, Some(8), Some(group @ (32 | 64))) => Some((8, group)),+ _ => None,+ }+}++pub fn route(+ device: DeviceProfile,+ shape: &MatmulShape,+ all_bf16: bool,+) -> Option<QmvRoute> {+ if !all_bf16+ || shape.m < 2+ || shape.m > 7+ || !shape.a_full_precision+ || !shape.b_transpose+ || shape.b_leading_dimension.is_some()+ || shape.gathered+ {+ return None;+ }+ let (bits, group) = qmv_format(shape)?;+ let mask = self::shape(shape.n, shape.k);+ if mask == 0 {+ return None;+ }+ let matches =+ |row: &&RouteRow| row.bits == bits && row.group == group && row.m == shape.m && row.shapes & mask != 0;+ let route =+ ROWS.iter().filter(matches).find(|row| row.identity == device.identity()).map(|row| row.route).or_else(|| {+ let same_family = |row: &&RouteRow| device.gpu_family().contains(row.identity);+ let mut routes = ROWS.iter().filter(matches).filter(same_family).map(|row| row.route);+ let route = routes.next()?;+ routes.all(|candidate| candidate == route).then_some(route)+ });+ route.filter(|route| {+ !matches!(+ route,+ QmvRoute::MainGemm(GemmPlan {+ engine: GemmEngine::Mxu,+ ..+ })+ ) || device.supports_mxu()+ })+}++#[cfg(test)]+#[path = "../../../../../../unit/backends/metal/kernel/matmul/qmv/routes_test.rs"]+mod tests;diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rsindex c6483b9c1..d03f39fce 100644--- a/crates/uzu-engine/src/backends/metal/mod.rs+++ b/crates/uzu-engine/src/backends/metal/mod.rs@@ -20,8 +20,4 @@ const _: () = {pub use backend::Metal;pub use context::MetalContext;#[cfg(test)]-pub use device_profile::{DeviceGeneration, DeviceProfile, DeviceSize};-#[cfg(test)]pub use kernel::matmul::gemm::GemmEngine;-#[cfg(test)]-pub(crate) use kernel::matmul::gemv::{GemvDispatch, GemvSpecialization};diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rsindex 2c9d41e0c..ae7a6de50 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs@@ -14,10 +14,10 @@ use crate::{kernel::{ActivationTransform, Kernels,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulShape},+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},- metal::{DeviceProfile, GemmEngine, GemvDispatch, GemvSpecialization, Metal, MetalContext},+ metal::{GemmEngine, Metal, MetalContext},},data_type::DataType,tests::{@@ -33,7 +33,7 @@ type MetalMatmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel;enum BenchPath {A8GemmMxu,Bf16GemmMxu,- Bf16Gemv,+ Bf16Routed,}impl BenchPath {@@ -41,7 +41,7 @@ impl BenchPath {match self {BenchPath::A8GemmMxu => "a8_gemm_mxu",BenchPath::Bf16GemmMxu => "abf16_gemm_mxu",- BenchPath::Bf16Gemv => "abf16_gemv",+ BenchPath::Bf16Routed => "abf16_routed",}}}@@ -91,16 +91,15 @@ impl BenchmarkData {let rht_factors = alloc_allocation_with_data::<Metal, i32>(context, &rht);let groups = k / group_size;- let a_elements = m as usize * k as usize;Self {unsigned_weights,signed_weights,weight_scales,activations,rht_factors,- a_working: alloc_allocation::<Metal, bf16>(context, a_elements),- a_int8: alloc_allocation::<Metal, i8>(context, a_elements),- a_scales: alloc_allocation::<Metal, f32>(context, m as usize * groups as usize),+ a_working: alloc_allocation::<Metal, bf16>(context, (m * k) as usize),+ a_int8: alloc_allocation::<Metal, i8>(context, (m * k) as usize),+ a_scales: alloc_allocation::<Metal, f32>(context, (m * groups) as usize),m,k,n,@@ -149,8 +148,6 @@ fn encode_step(prepare: &ActivationTransform<Metal>,hadamard: &ActivationTransform<Metal>,matmul: &mut MetalMatmul,- gemv: &mut GemvDispatch,- device_profile: DeviceProfile,encoder: &mut Encoder<Metal>,) {match path {@@ -196,19 +193,11 @@ fn encode_step(let args = data.bf16_arguments(output);matmul.gemm.encode_with_engine(args, GemmEngine::Mxu, encoder).expect("bf16 gemm mxu encode");},- BenchPath::Bf16Gemv => {+ BenchPath::Bf16Routed => {encoder.encode_copy(&data.activations, .., &mut data.a_working, ..);hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, data.m, data.k, encoder);let args = data.bf16_arguments(output);- let spec = GemvSpecialization::select_shape(- &MatmulShape::from_arguments(&args),- DataType::BF16,- DataType::BF16,- DataType::BF16,- device_profile,- )- .expect("bf16 gemv specialization");- gemv.encode(args, spec, encoder).expect("bf16 gemv encode");+ matmul.encode(args, encoder).expect("routed bf16 matmul encode");},}}@@ -216,15 +205,12 @@ fn encode_step(fn bench_bits(c: &mut Criterion,context: &MetalContext,- device_profile: DeviceProfile,prepare: &ActivationTransform<Metal>,hadamard: &ActivationTransform<Metal>,bits: u32,) {let mut matmul = <MetalMatmul as MatmulKernel>::new(context, DataType::BF16, DataType::BF16, DataType::BF16).expect("matmul kernel");- let mut gemv = GemvDispatch::new(DataType::BF16, DataType::BF16, DataType::BF16);-let mut group = c.benchmark_group(format!("{}/Kernel/A8W/w{bits}", type_short_name::<Metal>()));group.sample_size(10);group.warm_up_time(Duration::from_millis(100));@@ -241,22 +227,10 @@ fn bench_bits(HADAMARD_TRANSFORM_BLOCK_SIZE,0xA8_00 ^ u64::from(bits) ^ k as u64 ^ n as u64,);- let mut output = alloc_allocation::<Metal, bf16>(context, m as usize * n as usize);+ let mut output = alloc_allocation::<Metal, bf16>(context, (m * n) as usize);let shape_label = format!("{layer}_m{m}_k{k}_n{n}");- let gemv_eligible = GemvSpecialization::select_shape(- &MatmulShape::from_arguments(&data.bf16_arguments(&mut output)),- DataType::BF16,- DataType::BF16,- DataType::BF16,- device_profile,- )- .is_some();-- let mut paths = vec![BenchPath::A8GemmMxu, BenchPath::Bf16GemmMxu];- if gemv_eligible {- paths.push(BenchPath::Bf16Gemv);- }+ let paths = [BenchPath::A8GemmMxu, BenchPath::Bf16GemmMxu, BenchPath::Bf16Routed];group.throughput(Throughput::Elements((m * k * n) as u64));for path in paths {@@ -264,17 +238,7 @@ fn bench_bits(let benchmark_path =format!("{}/Kernel/A8W/w{bits}/{}/{shape_label}", type_short_name::<Metal>(), path.label());iter_encode_loop_named::<Metal, _>(context, bench, &benchmark_path, |encoder| {- encode_step(- path,- &mut data,- &mut output,- prepare,- hadamard,- &mut matmul,- &mut gemv,- device_profile,- encoder,- );+ encode_step(path, &mut data, &mut output, prepare, hadamard, &mut matmul, encoder);});});}@@ -288,12 +252,10 @@ fn bench_a8w(c: &mut Criterion) {if !context.supports_mxu() {return;}- let device_profile = context.device_profile();-let prepare = ActivationTransform::<Metal>::quantize(&context, DataType::BF16, 128, None).expect("prepare kernel");let hadamard = ActivationTransform::<Metal>::input_rht(&context, DataType::BF16, true).expect("hadamard kernel");for bits in [8u32, 4u32] {- bench_bits(c, &context, device_profile, &prepare, &hadamard, bits);+ bench_bits(c, &context, &prepare, &hadamard, bits);}}diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rsindex 40e268fd9..6f8197bfd 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs@@ -253,6 +253,13 @@ fn parity_bf16_gemv_qmv_fused_scale_bias() {#[case::gs64_4bit(1, 96, 64, 64, 4, QuantizationMethod::ScaleBias)]#[case::gs64_4bit_zp(2, 96, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs128_8bit(1, 192, 64, 128, 8, QuantizationMethod::ScaleBias)]+// Whole quantization groups with an unaligned K block.+#[case::gs64_4bit_k1152_zp(1, 1152, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]+#[case::gs64_8bit_k1152_zp(1, 1152, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]+// Same, plus a final partial quantization group (1056 = 16 * 64 + 32).+#[case::gs64_4bit_k1056_partial_group(1, 1056, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]+#[case::gs64_4bit_k514_unaligned_rows(2, 514, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]+#[case::gs64_8bit_k257_unaligned_rows(2, 257, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]fn parity_gemv_partial_group_bf16(#[case] m: u32,#[case] k: u32,diff --git a/crates/uzu-engine/unit/backends/metal/device_profile_test.rs b/crates/uzu-engine/unit/backends/metal/device_profile_test.rsindex 5f59a9b15..ab2cc7a8a 100644--- a/crates/uzu-engine/unit/backends/metal/device_profile_test.rs+++ b/crates/uzu-engine/unit/backends/metal/device_profile_test.rs@@ -4,17 +4,68 @@ use super::*;#[uzu_test]fn device_profile_detection() {- assert_eq!(classify_device(40, true, true, false), DeviceProfile::new(40, DeviceGeneration::Apple9));- assert_eq!(classify_device(40, true, true, true), DeviceProfile::new(40, DeviceGeneration::M5Plus));- assert_eq!(classify_device(20, true, true, false), DeviceProfile::new(20, DeviceGeneration::Apple9));- assert_eq!(classify_device(10, true, false, false), DeviceProfile::new(10, DeviceGeneration::Apple8));- assert_eq!(classify_device(8, false, false, false), DeviceProfile::new(8, DeviceGeneration::Legacy));+ let cases = [+ ("Apple M4 Max", 40, false, DeviceIdentity::M4Max, GpuFamily::Apple9, DeviceSize::Large),+ ("Apple M5 Max", 40, true, DeviceIdentity::M5Max, GpuFamily::M5Plus, DeviceSize::Large),+ ("Apple M3 Pro", 20, false, DeviceIdentity::M3Pro, GpuFamily::Apple9, DeviceSize::Small),+ ("Apple M2", 10, false, DeviceIdentity::M2, GpuFamily::Apple8, DeviceSize::Small),+ ("Apple M1", 8, false, DeviceIdentity::M1, GpuFamily::Legacy, DeviceSize::Small),+ ];+ for (name, cores, mxu, identity, family, size) in cases {+ let profile = classify_device(name, cores, mxu);+ assert_eq!(profile.identity(), identity);+ assert_eq!(profile.gpu_family(), family);+ assert_eq!(profile.size(), size);+ assert_eq!(profile.supports_mxu(), mxu);+ }}#[uzu_test]-fn size_derives_from_core_count() {- // M1 Max ships in two bins that straddle the cutoff, on identical silicon.- assert_eq!(classify_device(24, false, false, false).size(), DeviceSize::Small);- assert_eq!(classify_device(32, false, false, false).size(), DeviceSize::Large);- assert_eq!(classify_device(8, false, false, false).gpu_core_count(), 8);+fn family_detection_does_not_use_mxu_as_generation_proxy() {+ let m5_without_mxu = classify_device("Apple M5", 12, false);+ let m4_with_mxu = classify_device("Apple M4", 10, true);++ assert_eq!(m5_without_mxu.gpu_family(), GpuFamily::M5Plus);+ assert!(!m5_without_mxu.supports_mxu());+ assert_eq!(m4_with_mxu.gpu_family(), GpuFamily::Apple9);+ assert!(m4_with_mxu.supports_mxu());+}++#[uzu_test]+fn device_identities_remain_distinct() {+ let cases = [+ ("Apple M1", DeviceIdentity::M1),+ ("Apple M1 Pro", DeviceIdentity::M1Pro),+ ("Apple M1 Max", DeviceIdentity::M1Max),+ ("Apple M1 Ultra", DeviceIdentity::M1Ultra),+ ("Apple M2", DeviceIdentity::M2),+ ("Apple M2 Pro", DeviceIdentity::M2Pro),+ ("Apple M2 Max", DeviceIdentity::M2Max),+ ("Apple M2 Ultra", DeviceIdentity::M2Ultra),+ ("Apple M3", DeviceIdentity::M3),+ ("Apple M3 Pro", DeviceIdentity::M3Pro),+ ("Apple M3 Max", DeviceIdentity::M3Max),+ ("Apple M4", DeviceIdentity::M4),+ ("Apple M4 Pro", DeviceIdentity::M4Pro),+ ("Apple M4 Max", DeviceIdentity::M4Max),+ ("Apple M5", DeviceIdentity::M5),+ ("Apple M5 Pro", DeviceIdentity::M5Pro),+ ("Apple M5 Max", DeviceIdentity::M5Max),+ ];+ for (name, expected) in cases {+ assert_eq!(classify_device(name, 16, false).identity(), expected);+ }+}++#[uzu_test]+fn untuned_devices_keep_their_identity_and_family() {+ let m1_ultra = classify_device("Apple M1 Ultra", 64, false);+ assert_eq!(m1_ultra.identity(), DeviceIdentity::M1Ultra);+ assert_eq!(m1_ultra.gpu_family(), GpuFamily::Legacy);+}++#[uzu_test]+fn core_count_is_reduced_to_device_size() {+ assert_eq!(classify_device("Apple M1 Max", 24, false).size(), DeviceSize::Small);+ assert_eq!(classify_device("Apple M1 Max", 32, false).size(), DeviceSize::Large);}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex c1a50399d..018f92f72 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -4,13 +4,13 @@ use super::{super::specialization::GemmSpecialization, *};use crate::backends::{common::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmDTransform},metal::{- device_profile::{DeviceGeneration, DeviceProfile},+ device_profile::{DeviceIdentity, DeviceProfile, DeviceSize},kernel::matmul::MatmulMetalKernel,},};-const LEGACY_PROFILE: DeviceProfile = DeviceProfile::new(8, DeviceGeneration::Legacy);-const APPLE9_PROFILE: DeviceProfile = DeviceProfile::new(10, DeviceGeneration::Apple9);+const LEGACY_PROFILE: DeviceProfile = DeviceProfile::new(DeviceIdentity::M1, DeviceSize::Small, false);+const APPLE9_PROFILE: DeviceProfile = DeviceProfile::new(DeviceIdentity::M4, DeviceSize::Small, false);fn shape(m: u32,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rsindex 9118473c9..df438a596 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs@@ -1,59 +1,198 @@use uzu_engine_macros::uzu_test;use super::*;-use crate::backends::metal::device_profile::DeviceGeneration;+use crate::{+ backends::{+ common::{+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},+ kernel::matmul::MatmulShape,+ },+ metal::device_profile::{DeviceIdentity, DeviceProfile, DeviceSize},+ },+ data_type::DataType,+};-const LARGE: DeviceProfile = DeviceProfile::new(40, DeviceGeneration::Apple9);-const SMALL_APPLE9: DeviceProfile = DeviceProfile::new(10, DeviceGeneration::Apple9);-const SMALL_APPLE8: DeviceProfile = DeviceProfile::new(10, DeviceGeneration::Apple8);-const SMALL_LEGACY: DeviceProfile = DeviceProfile::new(8, DeviceGeneration::Legacy);-const SMALL_M5: DeviceProfile = DeviceProfile::new(10, DeviceGeneration::M5Plus);-/// 32-core M1 Max: Legacy generation on a Large die, so the G13 rules must not fire.-const LARGE_LEGACY: DeviceProfile = DeviceProfile::new(32, DeviceGeneration::Legacy);+fn profile(+ size: DeviceSize,+ identity: DeviceIdentity,+) -> DeviceProfile {+ DeviceProfile::new(+ identity,+ size,+ matches!(identity, DeviceIdentity::M5 | DeviceIdentity::M5Pro | DeviceIdentity::M5Max),+ )+}++fn qtile(+ num_simdgroups: u32,+ results_per_simdgroup: u32,+) -> GemvTile {+ GemvTile::quantized(num_simdgroups, results_per_simdgroup, 1, 32, 2)+}#[uzu_test]fn fp_policy_cases() {- #[rustfmt::skip]let cases = [- (LARGE, 1, 12288, 1536, true, tile(8, 8, 1)),- (LARGE, 1, 12288, 1536, false, tile(8, 1, 1)),- (LARGE, 1, 1536, 256, true, tile(8, 1, 1)),- (SMALL_APPLE9, 1, 1536, 256, true, tile(8, 2, 1)),- (LARGE, 8, 12288, 1536, true, tile(8, 1, 4)),- (LARGE, 1, 1536, 12288, true, tile(8, 8, 4)),- (SMALL_LEGACY, 1, 262144, 1536, true, tile(8, 1, 4)),- (LARGE_LEGACY, 1, 262144, 1536, true, tile(8, 8, 1)),- (LARGE_LEGACY, 1, 6144, 1536, true, tile(8, 8, 1)),+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 12288, 1536, true, tile(8, 8, 1)),+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 12288, 1536, false, tile(8, 1, 1)),+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 1536, 256, true, tile(8, 1, 1)),+ (profile(DeviceSize::Small, DeviceIdentity::M3), 1, 1536, 256, true, tile(8, 2, 1)),+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 8, 12288, 1536, true, tile(8, 1, 4)),+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 1536, 12288, true, tile(8, 8, 4)),+ (profile(DeviceSize::Small, DeviceIdentity::M1), 1, 262144, 1536, true, tile(8, 1, 4)),];for (profile, m, n, k, aligned, expected) in cases {- assert_eq!(fp_tile(m, n, k, aligned, profile), expected, "profile={profile:?} m={m} n={n} k={k}");+ assert_eq!(fp_tile(m, n, k, aligned, profile), expected);}}#[uzu_test]fn quant_policy_cases() {- #[rustfmt::skip]let cases = [- (LARGE, 1, 256, 1536, 4, false, qtile(2, 1)),- (LARGE, 1, 262144, 1536, 4, false, DEFAULT_TILE),- (SMALL_APPLE9, 1, 1536, 256, 4, false, qtile(4, 4)),- (SMALL_APPLE8, 1, 2048, 1536, 4, false, qtile(8, 2)),- (SMALL_LEGACY, 1, 256, 1536, 4, false, qtile(8, 2)),- (SMALL_LEGACY, 1, 1536, 256, 4, false, qtile(4, 8)),- (LARGE, 2, 2048, 1536, 4, false, DEFAULT_TILE),- (LARGE, 1, 2048, 1536, 8, false, DEFAULT_TILE),- (LARGE, 1, 2560, 9216, 4, true, qtile(4, 8)),- // Small M5 dies classified as SmallApple9 before the split; keep those rows.- (SMALL_M5, 1, 1536, 256, 4, false, qtile(4, 4)),- (SMALL_M5, 1, 256, 1536, 4, false, qtile(4, 2)),+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 256, 1536, 4, false, qtile(2, 1)),+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 262144, 1536, 4, false, qtile(8, 4)),+ (profile(DeviceSize::Small, DeviceIdentity::M3), 1, 1536, 256, 4, false, qtile(4, 4)),+ (profile(DeviceSize::Small, DeviceIdentity::M2), 1, 2048, 1536, 4, false, qtile(8, 2)),+ (profile(DeviceSize::Small, DeviceIdentity::M1), 1, 256, 1536, 4, false, qtile(8, 2)),+ (profile(DeviceSize::Small, DeviceIdentity::M1), 1, 1536, 256, 4, false, qtile(4, 8)),+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 2, 2048, 1536, 4, false, qtile(8, 4)),+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 4, 5120, 5120, 4, false, qtile(8, 4)),+ (+ profile(DeviceSize::Large, DeviceIdentity::M3Max),+ 1,+ 2048,+ 1536,+ 8,+ false,+ GemvTile::quantized(8, 4, 1, 32, 4),+ ),+ (profile(DeviceSize::Large, DeviceIdentity::M3Max), 1, 2560, 9216, 4, true, qtile(4, 8)),];for (profile, m, n, k, bits, has_rht, expected) in cases {- assert_eq!(- quant_tile(m, n, k, bits, has_rht, profile),- expected,- "profile={profile:?} m={m} n={n} k={k} bits={bits}"- );+ let actual = quantized_tile(+ profile,+ bits,+ 32,+ m,+ n,+ k,+ if has_rht {+ GemmDTransform::RHT+ } else {+ GemmDTransform::empty()+ },+ true,+ )+ .expect("quantized route");+ assert_eq!(actual, expected, "m={m} n={n} k={k} bits={bits} profile={profile:?}");}}++fn quantized(+ profile: DeviceProfile,+ bits: u32,+ group: u32,+ m: u32,+ n: u32,+ k: u32,+ d_transform: GemmDTransform,+) -> Option<GemvTile> {+ quantized_tile(profile, bits, group, m, n, k, d_transform, true)+}++#[uzu_test]+fn quantized_policy_edges() {+ for (profile, m, group, expected) in [+ (profile(DeviceSize::Large, DeviceIdentity::M5Max), 2, 32, GemvTile::quantized(8, 4, 1, 32, 4)),+ (profile(DeviceSize::Large, DeviceIdentity::M5Max), 3, 32, GemvTile::quantized(8, 4, 1, 32, 4)),+ (profile(DeviceSize::Small, DeviceIdentity::M2), 4, 64, GemvTile::quantized(8, 4, 1, 32, 8)),+ ] {+ assert_eq!(quantized(profile, 8, group, m, 4096, 4096, GemmDTransform::RHT), Some(expected));+ }+ let profile = profile(DeviceSize::Large, DeviceIdentity::M5Max);+ let none = GemmDTransform::empty();+ assert_eq!(quantized(profile, 4, 64, 4, 8192, 4100, none), Some(GemvTile::quantized(8, 4, 1, 32, 4)));+ assert!(quantized(profile, 4, 32, 4, 8, 4096, none).is_some());+ assert!(quantized(profile, 4, 32, 1, 8192, 4096, none).is_some());+ assert_eq!(quantized(profile, 4, 32, 9, 8192, 4096, none), None);+ assert_eq!(quantized_tile(profile, 4, 32, 4, 8192, 4096, none, true), Some(GemvTile::quantized(8, 4, 1, 32, 2)));+}++#[uzu_test]+fn untuned_quantized_io_uses_only_the_generated_fallback() {+ let profile = profile(DeviceSize::Large, DeviceIdentity::M5Max);+ assert_eq!(quantized_tile(profile, 4, 32, 4, 3, 4096, GemmDTransform::empty(), false), None);+ let tile =+ quantized_tile(profile, 4, 32, 4, 4096, 4096, GemmDTransform::empty(), false).expect("mixed-IO fallback");+ assert_eq!(tile, GemvTile::quantized(8, 4, 1, 32, 2));+ assert_eq!(quantized_tile(profile, 4, 32, 5, 4096, 4096, GemmDTransform::empty(), false), None);+}++fn quant_shape(+ m: u32,+ n: u32,+ d_transform: GemmDTransform,+) -> MatmulShape {+ MatmulShape {+ m,+ n,+ k: 4096,+ b_transpose: true,+ b_leading_dimension: None,+ b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,+ b_bits: Some(4),+ b_group_size: Some(32),+ signed_codes: false,+ a_full_precision: true,+ gathered: false,+ d_transform,+ }+}++#[uzu_test]+fn block_unaligned_quantized_k_stays_on_gemv() {+ let profile = profile(DeviceSize::Large, DeviceIdentity::M5Max);+ let shape = MatmulShape {+ m: 1,+ n: 64,+ k: 1152,+ b_transpose: true,+ b_leading_dimension: None,+ b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,+ b_bits: Some(4),+ b_group_size: Some(64),+ signed_codes: false,+ a_full_precision: true,+ gathered: false,+ d_transform: GemmDTransform::empty(),+ };+ assert!(+ super::super::kernel::GemvSpecialization::select_shape(+ &shape,+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ profile,+ )+ .is_some()+ );+}++#[uzu_test]+fn specialization_preserves_quantized_route_and_accumulate_tail() {+ let profile = profile(DeviceSize::Large, DeviceIdentity::M5Max);+ let select = |n, transform| {+ super::super::kernel::GemvSpecialization::select_shape(+ &quant_shape(4, n, transform),+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ profile,+ )+ };+ let clean = select(8192, GemmDTransform::empty()).expect("quantized specialization");+ assert!(clean.output_row_tile() > DEFAULT_RESULTS_PER_SIMDGROUP);+ assert_eq!(select(8192 + clean.output_row_tile() / 2, GemmDTransform::ACCUMULATE), None);+}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rsnew file mode 100644index 000000000..7e016371c--- /dev/null+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs@@ -0,0 +1,233 @@+use std::fmt::Write;++use uzu_engine_macros::uzu_test;+use xxhash_rust::xxh3::xxh3_64;++use super::*;+use crate::{+ backends::{+ common::{+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},+ kernel::matmul::MatmulShape,+ },+ metal::{+ device_profile::{DeviceIdentity, DeviceProfile, DeviceSize},+ kernel::matmul::{MatmulDispatch, MatmulMetalKernel, gemm::GemmProblem, gemv::GemvSpecialization},+ },+ },+ data_type::DataType,+};++const FROZEN_PLAN_FINGERPRINT: u64 = 5_839_212_743_558_880_136;++const DEVICES: [(&str, DeviceIdentity); 7] = [+ ("m1", DeviceIdentity::M1),+ ("m2", DeviceIdentity::M2),+ ("m2-pro", DeviceIdentity::M2Pro),+ ("m3-max", DeviceIdentity::M3Max),+ ("m4", DeviceIdentity::M4),+ ("m4-pro", DeviceIdentity::M4Pro),+ ("m5-max", DeviceIdentity::M5Max),+];+const FORMATS: [(&str, u32, u32, GemmBPrologueKind); 4] = [+ ("W4/ZP G32", 4, 32, GemmBPrologueKind::ScaleZeroPointDequant),+ ("W4/ZP G64", 4, 64, GemmBPrologueKind::ScaleZeroPointDequant),+ ("W8/Symmetric G32", 8, 32, GemmBPrologueKind::ScaleSymmetricDequant),+ ("W8/Symmetric G64", 8, 64, GemmBPrologueKind::ScaleSymmetricDequant),+];+const SHAPES: [(&str, u32, u32); 7] = [+ ("down", 5120, 17408),+ ("gate", 6144, 5120),+ ("gate-up", 34816, 5120),+ ("projection-in", 16480, 5120),+ ("projection-out", 5120, 6144),+ ("qkv", 8192, 5120),+ ("readout", 248320, 5120),+];++fn problem(+ m: u32,+ n: u32,+ k: u32,+ bits: u32,+ group: u32,+ prologue: GemmBPrologueKind,+) -> MatmulShape {+ MatmulShape {+ m,+ n,+ k,+ b_transpose: true,+ b_leading_dimension: None,+ b_prologue: prologue,+ b_bits: Some(bits),+ b_group_size: Some(group),+ signed_codes: false,+ a_full_precision: true,+ gathered: false,+ d_transform: GemmDTransform::empty(),+ }+}++#[uzu_test]+fn table_is_complete_and_fingerprint_is_stable() {+ let mut canonical = Vec::new();+ let mut matched_rows = vec![false; ROWS.len()];+ let mut tuned = 0;+ let mut main_gemv = 0;+ let mut main_gemm = 0;+ for &(device_name, identity) in &DEVICES {+ for &(format_name, bits, group, prologue) in &FORMATS {+ for m in 2..=7 {+ for &(shape_name, n, k) in &SHAPES {+ let mask = shape(n, k);+ let matches: Vec<_> = ROWS+ .iter()+ .enumerate()+ .filter(|row| {+ row.1.identity == identity+ && row.1.bits == bits+ && row.1.group == group+ && row.1.m == m+ && row.1.shapes & mask != 0+ })+ .collect();+ assert_eq!(matches.len(), 1, "route coverage for {device_name} {format_name} M={m} {shape_name}");+ let (row_index, row) = matches[0];+ matched_rows[row_index] = true;+ let selected = row.route;+ match selected {+ QmvRoute::Tuned(_) => tuned += 1,+ QmvRoute::MainGemv(_) => main_gemv += 1,+ QmvRoute::MainGemm(_) => main_gemm += 1,+ }+ canonical.push(format!("{device_name}|{format_name}|{shape_name}|{m}|{n}|{k}|{selected:?}"));+ let profile = device_profile(identity);+ let problem = problem(m, n, k, bits, group, prologue);+ assert_eq!(route(profile, &problem, true), Some(selected));+ let runtime = MatmulMetalKernel::choose_dispatch(+ &problem,+ profile,+ profile.supports_mxu(),+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ );+ if let QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) = selected {+ let specialization = GemvSpecialization::select_tile(+ &problem,+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ tile,+ )+ .expect("stored GEMV tile must be legal");+ assert_eq!(specialization.tile(), tile);+ assert!(matches!(runtime, MatmulDispatch::Gemv(actual) if actual == specialization));+ } else if let QmvRoute::MainGemm(plan) = selected {+ let problem =+ GemmProblem::new(problem, DataType::BF16, DataType::BF16, profile.supports_mxu(), profile);+ assert!(problem.plan_is_legal(plan), "stored GEMM plan must be legal");+ assert!(matches!(runtime, MatmulDispatch::Gemm(actual) if actual == plan));+ }+ }+ }+ }+ }+ assert!(matched_rows.into_iter().all(|matched| matched), "route table contains an orphaned row");+ canonical.sort();+ assert_eq!(canonical.len(), 1176);+ let mut expanded = String::new();+ for line in canonical {+ writeln!(&mut expanded, "{line}").unwrap();+ }+ assert_eq!(tuned, 864);+ assert_eq!(main_gemv, 221);+ assert_eq!(main_gemm, 91);+ assert_eq!(xxh3_64(expanded.trim_end().as_bytes()), FROZEN_PLAN_FINGERPRINT);+}++fn device_profile(identity: DeviceIdentity) -> DeviceProfile {+ let large = matches!(identity, DeviceIdentity::M3Max | DeviceIdentity::M5Max);+ DeviceProfile::new(+ identity,+ if large {+ DeviceSize::Large+ } else {+ DeviceSize::Small+ },+ matches!(identity, DeviceIdentity::M5Max),+ )+}++#[uzu_test]+fn exact_lookup_rejects_non_matrix_inputs() {+ let profile = device_profile(DeviceIdentity::M1);+ let p = problem(2, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);+ assert!(route(profile, &p, true).is_some());+ for mutate in [+ |p: &mut MatmulShape| p.m = 1,+ |p: &mut MatmulShape| p.n = 1,+ |p: &mut MatmulShape| p.b_bits = Some(8),+ |p: &mut MatmulShape| p.gathered = true,+ ] {+ let mut rejected = p;+ mutate(&mut rejected);+ assert!(route(profile, &rejected, true).is_none());+ }+ assert!(route(profile, &p, false).is_none());++ let mut rht = p;+ rht.d_transform = GemmDTransform::RHT;+ rht.signed_codes = true;+ let selected = route(profile, &rht, true).expect("RHT must preserve the exact route");+ let QmvRoute::Tuned(tile) = selected else {+ panic!("test anchor must use a tuned tile");+ };+ assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_some());+ rht.n -= 1;+ assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_none());++ let m5_without_mxu = DeviceProfile::new(DeviceIdentity::M5Max, DeviceSize::Large, false);+ let p = problem(7, 6144, 5120, 8, 64, GemmBPrologueKind::ScaleSymmetricDequant);+ assert!(matches!(route(device_profile(DeviceIdentity::M5Max), &p, true), Some(QmvRoute::MainGemm(_))));+ assert_eq!(route(m5_without_mxu, &p, true), None);+}++#[uzu_test]+fn normal_routing_handles_inputs_outside_the_frozen_matrix() {+ let profile = device_profile(DeviceIdentity::M1);+ for (m, n, k) in [(1, 5120, 17408), (2, 4096, 5120)] {+ let problem = problem(m, n, k, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);+ assert_eq!(route(profile, &problem, true), None);+ let specialization =+ GemvSpecialization::select_shape(&problem, DataType::BF16, DataType::BF16, DataType::BF16, profile)+ .expect("normal M1 policy should select GEMV for this anchor");+ assert!(matches!(+ MatmulMetalKernel::choose_dispatch(+ &problem,+ profile,+ false,+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ ),+ MatmulDispatch::Gemv(actual) if actual == specialization+ ));+ }+}++#[uzu_test]+fn family_lookup_requires_one_unanimous_route() {+ let m1_max = DeviceProfile::new(DeviceIdentity::M1Max, DeviceSize::Large, false);+ let m1_route = problem(4, 8192, 5120, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);+ assert_eq!(route(m1_max, &m1_route, true), route(device_profile(DeviceIdentity::M1), &m1_route, true));++ let family = DeviceProfile::new(DeviceIdentity::M2Max, DeviceSize::Small, false);+ let unanimous = problem(6, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);+ assert_eq!(route(family, &unanimous, true), route(device_profile(DeviceIdentity::M2), &unanimous, true));++ let disagreement = problem(3, 5120, 6144, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);+ assert_eq!(route(family, &disagreement, true), None);+ assert!(route(device_profile(DeviceIdentity::M2), &disagreement, true).is_some());+}