A high-performance inference engine for AI models
Faster pre-M5 prefill GEMM: f32 dequant staged as half, batch-only tile rule (#1061)
CCYehea52b58792fe
diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rsindex 35c8a2873..e04858e76 100644--- a/crates/uzu-engine/src/backends/metal/context.rs+++ b/crates/uzu-engine/src/backends/metal/context.rs@@ -9,8 +9,8 @@ use std::{use metal::{MTL4CommandAllocator, MTL4CommandQueue, MTL4CommandQueueDescriptor, MTLCaptureTarget, MTLComputePipelineState,- MTLDevice, MTLDeviceExt, MTLEvent, MTLEventExt, MTLFunctionConstantValues, MTLGPUFamily, MTLLibrary,- MTLResidencySet, MTLResidencySetDescriptor, MTLSparsePageSize,+ MTLDevice, MTLDeviceExt, MTLEvent, MTLEventExt, MTLFunctionConstantValues, MTLLibrary, MTLResidencySet,+ MTLResidencySetDescriptor, MTLSparsePageSize,};use objc2::{rc::Retained, runtime::ProtocolObject};use parking_lot::Mutex;@@ -42,7 +42,6 @@ pub(super) struct SparseState {pub struct MetalContext {pub device: Retained<ProtocolObject<dyn MTLDevice>>,pub gpu_core_count: u32,- pub apple_gpu_family: MTLGPUFamily,pub supports_mxu: bool,pub device_name: String,pub(super) residency_set: Arc<Mutex<Retained<ProtocolObject<dyn MTLResidencySet>>>>,@@ -124,7 +123,6 @@ impl Context for MetalContext {let device_name = device.name();let gpu_core_count = device.gpu_core_count();- let apple_gpu_family = device.newest_supported_apple_gpu_family();let supports_mxu = device.supports_mxu();let peak_memory_usage = Arc::new(AtomicUsize::new(0));@@ -162,7 +160,6 @@ impl Context for MetalContext {Ok(Arc::new_cyclic(|weak_self: &Weak<Self>| Self {device,gpu_core_count,- apple_gpu_family,supports_mxu,device_name,residency_set: Arc::new(Mutex::new(residency_set)),diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rsindex 4c51fa007..8b536d3e4 100644--- a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs@@ -23,7 +23,6 @@ pub enum GemmBLayout {pub enum GemmTiling {Tile8x32x32_Simdgroups1x1,Tile64x32x32_Simdgroups2x2,- Tile64x64x16_Simdgroups2x2,Tile64x64x32_Simdgroups2x2,Tile32x32x32_Simdgroups2x2,Tile16x32x256_Simdgroups1x1,@@ -48,7 +47,6 @@ impl GemmTiling {match self {Self::Tile8x32x32_Simdgroups1x1 => 8,Self::Tile64x32x32_Simdgroups2x2 => 64,- Self::Tile64x64x16_Simdgroups2x2 => 64,Self::Tile64x64x32_Simdgroups2x2 => 64,Self::Tile32x32x32_Simdgroups2x2 => 32,Self::Tile16x32x256_Simdgroups1x1 => 16,@@ -66,7 +64,6 @@ impl GemmTiling {match self {Self::Tile8x32x32_Simdgroups1x1 => 32,Self::Tile64x32x32_Simdgroups2x2 => 32,- Self::Tile64x64x16_Simdgroups2x2 => 64,Self::Tile64x64x32_Simdgroups2x2 => 64,Self::Tile32x32x32_Simdgroups2x2 => 32,Self::Tile16x32x256_Simdgroups1x1 => 32,@@ -84,7 +81,6 @@ impl GemmTiling {match self {Self::Tile8x32x32_Simdgroups1x1 => 32,Self::Tile64x32x32_Simdgroups2x2 => 32,- Self::Tile64x64x16_Simdgroups2x2 => 16,Self::Tile64x64x32_Simdgroups2x2 => 32,Self::Tile32x32x32_Simdgroups2x2 => 32,Self::Tile16x32x256_Simdgroups1x1 => 256,@@ -103,7 +99,6 @@ impl GemmTiling {match self {Self::Tile8x32x32_Simdgroups1x1 => 1,Self::Tile64x32x32_Simdgroups2x2 => 2,- Self::Tile64x64x16_Simdgroups2x2 => 2,Self::Tile64x64x32_Simdgroups2x2 => 2,Self::Tile32x32x32_Simdgroups2x2 => 2,Self::Tile16x32x256_Simdgroups1x1 => 1,@@ -121,7 +116,6 @@ impl GemmTiling {match self {Self::Tile8x32x32_Simdgroups1x1 => 1,Self::Tile64x32x32_Simdgroups2x2 => 2,- Self::Tile64x64x16_Simdgroups2x2 => 2,Self::Tile64x64x32_Simdgroups2x2 => 2,Self::Tile32x32x32_Simdgroups2x2 => 2,Self::Tile16x32x256_Simdgroups1x1 => 1,diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.hindex 34d1c9dda..e46959984 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h@@ -40,18 +40,17 @@ enum class GemmBLayout : uint32_t {enum class GemmTiling : uint32_t {Tile8x32x32_Simdgroups1x1 = 0,Tile64x32x32_Simdgroups2x2 = 1,- Tile64x64x16_Simdgroups2x2 = 2,- Tile64x64x32_Simdgroups2x2 = 3,- Tile32x32x32_Simdgroups2x2 = 4,- Tile16x32x256_Simdgroups1x1 = 5,- Tile16x32x256_Simdgroups1x1x4 = 6,- Tile16x128x256_Simdgroups1x4 = 7,- Tile32x64x256_Simdgroups2x2 = 8,- Tile64x32x256_Simdgroups4x1 = 9,- Tile64x64x256_Simdgroups2x2 = 10,- Tile128x128x256_Simdgroups4x4 = 11,- Tile8x32x64_Simdgroups1x2x4 = 12,- Tile8x32x128_Simdgroups1x2x4 = 13,+ Tile64x64x32_Simdgroups2x2 = 2,+ Tile32x32x32_Simdgroups2x2 = 3,+ Tile16x32x256_Simdgroups1x1 = 4,+ Tile16x32x256_Simdgroups1x1x4 = 5,+ Tile16x128x256_Simdgroups1x4 = 6,+ Tile32x64x256_Simdgroups2x2 = 7,+ Tile64x32x256_Simdgroups4x1 = 8,+ Tile64x64x256_Simdgroups2x2 = 9,+ Tile128x128x256_Simdgroups4x4 = 10,+ Tile8x32x64_Simdgroups1x2x4 = 11,+ Tile8x32x128_Simdgroups1x2x4 = 12,};static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.hindex ffe4ce189..15492da5f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h@@ -84,37 +84,34 @@ METAL_FUNC float magic_code_float(uint code) {METAL_FUNC float magic_dequant_bias(float scale) { return -12582912.0f * scale; }+// fma(code, scale, bias) in f32, rounded once to U, stored in pairs.template <typename U, int N, int bits>-inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local, const bool signed_codes) {+inline void dequantize(+ const device uint8_t* w,+ float scale,+ float bias,+ threadgroup U* w_local,+ const bool signed_codes+) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");-+ static_assert(bits == 8 || N % 2 == 0, "int4 values are unpacked in pairs");+ threadgroup vec<U, 2>* pairs = reinterpret_cast<threadgroup vec<U, 2>*>(w_local);if constexpr (bits == 4) {- U s0 = scale;- U s1 = scale / static_cast<U>(16.0f);- // Keeping literal masks in each arm lets the compiler vectorize the unpack.- if (signed_codes) {- for (int i = 0; i < (N / 2); i++) {- const uint8_t word = w[i] ^ uint8_t(0x88u);- w_local[2 * i] = s0 * (word & 0x0f) + bias;- w_local[2 * i + 1] = s1 * (word & 0xf0) + bias;- }- } else {- for (int i = 0; i < (N / 2); i++) {- w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;- w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;- }+ const uint sign_flip = signed_codes ? 0x88u : 0u;+ for (int i = 0; i < (N / 2); i++) {+ const uint code = uint(w[i]) ^ sign_flip;+ pairs[i] = vec<U, 2>(vec<float, 2>(fma(float(code & 0x0fu), scale, bias), fma(float(code >> 4), scale, bias)));+ }+ } else {+ const uint sign_flip = signed_codes ? 0x80u : 0u;+ for (int i = 0; i < (N / 2); i++) {+ pairs[i] = vec<U, 2>(vec<float, 2>(+ fma(float(uint(w[2 * i]) ^ sign_flip), scale, bias),+ fma(float(uint(w[2 * i + 1]) ^ sign_flip), scale, bias)+ ));}- } else if constexpr (bits == 8) {- if (signed_codes) {- const device int8_t* signed_weights = reinterpret_cast<const device int8_t*>(w);- const U adjusted_bias = bias + scale * U(128);- for (int i = 0; i < N; i++) {- w_local[i] = scale * U(signed_weights[i]) + adjusted_bias;- }- } else {- for (int i = 0; i < N; i++) {- w_local[i] = scale * U(w[i]) + bias;- }+ if constexpr (N % 2 == 1) {+ w_local[N - 1] = U(fma(float(uint(w[N - 1]) ^ sign_flip), scale, bias));}}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/threadgroup_tile.hindex 9a3a0876e..10ca9612f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/threadgroup_tile.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/threadgroup_tile.h@@ -132,15 +132,17 @@ struct ThreadgroupTile {c_fragment.map([&](AccumulatorType value) { return op.apply(value); });}- METAL_FUNC void apply_bias(const device BT* bias) {- const device BT* bias_ptr = bias + simdgroup_col_offset;+ template <typename BiasT>+ METAL_FUNC void apply_bias(const device BiasT* bias) {+ const device BiasT* bias_ptr = bias + simdgroup_col_offset;for_each_output([&](ushort, ushort col_offset, ushort k, thread AccumulatorType& v) {v += static_cast<AccumulatorType>(bias_ptr[col_offset + k]);});}- METAL_FUNC void apply_bias_safe(const device BT* bias, short2 tile_dimensions) {- const device BT* bias_ptr = bias + simdgroup_col_offset;+ template <typename BiasT>+ METAL_FUNC void apply_bias_safe(const device BiasT* bias, short2 tile_dimensions) {+ const device BiasT* bias_ptr = bias + simdgroup_col_offset;tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);for_each_output([&](ushort row_offset, ushort col_offset, ushort k, thread AccumulatorType& v) {if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x)diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.hindex 89528b0de..7c70a5186 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h@@ -8,7 +8,6 @@ namespace gemm {constexpr uint gemm_tiling_block_m(GemmTiling t) {return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 8: t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 64- : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 64: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 16@@ -26,7 +25,6 @@ constexpr uint gemm_tiling_block_m(GemmTiling t) {constexpr uint gemm_tiling_block_n(GemmTiling t) {return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 32: t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 32- : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 64: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 32@@ -44,7 +42,6 @@ constexpr uint gemm_tiling_block_n(GemmTiling t) {constexpr uint gemm_tiling_block_k(GemmTiling t) {return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 32: t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 32- : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 16: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 32: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 256@@ -62,7 +59,6 @@ constexpr uint gemm_tiling_block_k(GemmTiling t) {constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 1: t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 2- : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 2: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 1@@ -80,7 +76,6 @@ constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 1: t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 2- : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 2: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 1diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 567890ed9..0e300983c 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -34,6 +34,7 @@ struct MxuMmaCore {using Right = RightOperand;using LeftElementType = typename Left::ElementType;using RightElementType = typename Right::ElementType;+ using RightStageType = RightElementType;using LeftStorage = operands::LeftStorage<Left>;using RightStorage = operands::RightStorage<Right>;using FragmentOps = uzu::matmul::MxuFragmentOps<>;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.hindex 9fe71f078..ff5ebaa18 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h@@ -19,7 +19,8 @@ template <short DESTINATION_LEADING_DIMENSION,short THREADGROUP_SIZE,short GROUP_SIZE,- short BITS>+ short BITS,+ typename StageT = T>struct QuantizedBlockLoaderScaleBias {static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");@@ -33,6 +34,10 @@ struct QuantizedBlockLoaderScaleBias {: (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;UZU_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;UZU_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;+ static_assert(+ DESTINATION_LEADING_DIMENSION % 2 == 0 && (READS_PER_THREAD * PACK_FACTOR) % 2 == 0,+ "paired stores need 4-byte alignment"+ );const int src_leading_dim;const int tile_stride;@@ -43,7 +48,7 @@ struct QuantizedBlockLoaderScaleBias {const short tile_row_index;const short tile_col_index;- threadgroup T* dst;+ threadgroup StageT* dst;const device uint8_t* src;const device T* scales;const device T* biases;@@ -65,7 +70,8 @@ struct QuantizedBlockLoaderScaleBias {group_step_counter(0), group_stride(params_group_stride_), thread_index(simd_group_id * 32 + simd_lane_id),tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),- dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index * PACK_FACTOR),+ dst(reinterpret_cast<threadgroup StageT*>(dst_) + tile_row_index * DESTINATION_LEADING_DIMENSION ++ tile_col_index * PACK_FACTOR),src(src_ + tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR + tile_col_index * BYTES_PER_PACK),scales(scales_ + tile_row_index * params_output_stride_),biases(biases_ + tile_row_index * params_output_stride_), signed_codes(signed_codes_) {}@@ -77,11 +83,7 @@ struct QuantizedBlockLoaderScaleBias {}}- T scale = *scales;- T bias = *biases;- for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, PACK_FACTOR, BITS>(src + i * BYTES_PER_PACK, scale, bias, dst + i * PACK_FACTOR, signed_codes);- }+ dequantize<StageT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, float(*scales), float(*biases), dst, signed_codes);}void load_safe(short2 src_tile_dim) const {@@ -93,28 +95,34 @@ struct QuantizedBlockLoaderScaleBias {if (tile_row_index >= src_tile_dim.y) {for (int i = 0; i < READS_PER_THREAD * PACK_FACTOR; i++) {- dst[i] = T(0);+ dst[i] = StageT(0);}return;}const int valid_cols = src_tile_dim.x;const int valid_packs = (valid_cols + PACK_FACTOR - 1) / PACK_FACTOR;- T scale = *scales;- T bias = *biases;+ const float scale = float(*scales);+ const float bias = float(*biases);for (int i = 0; i < READS_PER_THREAD; i++) {const int pack_index = tile_col_index + i;if (pack_index < valid_packs) {- dequantize<T, PACK_FACTOR, BITS>(src + i * BYTES_PER_PACK, scale, bias, dst + i * PACK_FACTOR, signed_codes);+ dequantize<StageT, PACK_FACTOR, BITS>(+ src + i * BYTES_PER_PACK,+ scale,+ bias,+ dst + i * PACK_FACTOR,+ signed_codes+ );if (pack_index == valid_packs - 1) {const int remaining = valid_cols - pack_index * PACK_FACTOR;for (int lane = remaining; lane < PACK_FACTOR; ++lane) {- dst[i * PACK_FACTOR + lane] = T(0);+ dst[i * PACK_FACTOR + lane] = StageT(0);}}} else {for (int lane = 0; lane < PACK_FACTOR; ++lane) {- dst[i * PACK_FACTOR + lane] = T(0);+ dst[i * PACK_FACTOR + lane] = StageT(0);}}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.hindex 0cdd8d38e..87f7fe985 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h@@ -20,7 +20,8 @@ template <short THREADGROUP_SIZE,short GROUP_SIZE,short BITS,- bool SCALE_SYMMETRIC = false>+ bool SCALE_SYMMETRIC = false,+ typename StageT = T>struct QuantizedBlockLoaderScaleZeroPoint {static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");@@ -34,6 +35,10 @@ struct QuantizedBlockLoaderScaleZeroPoint {: (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;UZU_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;UZU_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;+ static_assert(+ DESTINATION_LEADING_DIMENSION % 2 == 0 && (READS_PER_THREAD * PACK_FACTOR) % 2 == 0,+ "paired stores need 4-byte alignment"+ );const int src_leading_dim;const int tile_stride;@@ -45,7 +50,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {const short tile_row_index;const short tile_col_index;- threadgroup T* dst;+ threadgroup StageT* dst;const device uint8_t* src;const device T* scales;const device uint8_t* zero_points_row_start;@@ -72,7 +77,8 @@ struct QuantizedBlockLoaderScaleZeroPoint {thread_index(simd_group_id * 32 + simd_lane_id),tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),- dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index * PACK_FACTOR),+ dst(reinterpret_cast<threadgroup StageT*>(dst_) + tile_row_index * DESTINATION_LEADING_DIMENSION ++ tile_col_index * PACK_FACTOR),src(src_ + tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR + tile_col_index * BYTES_PER_PACK),scales(scales_ + tile_row_index * params_output_stride_),zero_points_row_start(SCALE_SYMMETRIC ? nullptr : zero_points_row_start_),@@ -108,10 +114,9 @@ struct QuantizedBlockLoaderScaleZeroPoint {static_assert(SCALE_SYMMETRIC, "zero-point-free loader construction requires symmetric quantization");}- inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {+ inline void current_scale_bias(thread float& out_scale, thread float& out_bias) const {uint zero_point_value;- T scale_value;- scale_value = *scales;+ const float scale_value = float(*scales);if constexpr (SCALE_SYMMETRIC) {zero_point_value = symmetric_zero_point<ushort(BITS)>();} else {@@ -120,7 +125,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {zero_point_value = decode_zero_point<ushort(BITS)>(zero_points_row_start[byte_index], packed_index);}out_scale = scale_value;- out_bias = static_cast<T>(-scale_value * static_cast<T>(zero_point_value));+ out_bias = -scale_value * float(zero_point_value);}void load_unsafe() const {@@ -130,12 +135,10 @@ struct QuantizedBlockLoaderScaleZeroPoint {}}- T scale;- T bias;+ float scale;+ float bias;current_scale_bias(scale, bias);- for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, PACK_FACTOR, BITS>(src + i * BYTES_PER_PACK, scale, bias, dst + i * PACK_FACTOR, signed_codes);- }+ dequantize<StageT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, scale, bias, dst, signed_codes);}void load_safe(short2 src_tile_dim) const {@@ -147,29 +150,35 @@ struct QuantizedBlockLoaderScaleZeroPoint {if (tile_row_index >= src_tile_dim.y) {for (int i = 0; i < READS_PER_THREAD * PACK_FACTOR; i++) {- dst[i] = T(0);+ dst[i] = StageT(0);}return;}const int valid_cols = src_tile_dim.x;const int valid_packs = (valid_cols + PACK_FACTOR - 1) / PACK_FACTOR;- T scale;- T bias;+ float scale;+ float bias;current_scale_bias(scale, bias);for (int i = 0; i < READS_PER_THREAD; i++) {const int pack_index = tile_col_index + i;if (pack_index < valid_packs) {- dequantize<T, PACK_FACTOR, BITS>(src + i * BYTES_PER_PACK, scale, bias, dst + i * PACK_FACTOR, signed_codes);+ dequantize<StageT, PACK_FACTOR, BITS>(+ src + i * BYTES_PER_PACK,+ scale,+ bias,+ dst + i * PACK_FACTOR,+ signed_codes+ );if (pack_index == valid_packs - 1) {const int remaining = valid_cols - pack_index * PACK_FACTOR;for (int lane = remaining; lane < PACK_FACTOR; ++lane) {- dst[i * PACK_FACTOR + lane] = T(0);+ dst[i * PACK_FACTOR + lane] = StageT(0);}}} else {for (int lane = 0; lane < PACK_FACTOR; ++lane) {- dst[i * PACK_FACTOR + lane] = T(0);+ dst[i * PACK_FACTOR + lane] = StageT(0);}}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.hindex 4adb2da49..2fdcd57c4 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h@@ -45,7 +45,8 @@ static METAL_FUNC auto make_staged_loader(Core::SHARED_STRIDE_B,Core::THREADGROUP_THREADS,RightOperand::GROUP_SIZE,- RightOperand::BITS>;+ RightOperand::BITS,+ typename Core::RightStageType>;return Loader(values,scales,@@ -67,7 +68,8 @@ static METAL_FUNC auto make_staged_loader(Core::THREADGROUP_THREADS,RightOperand::GROUP_SIZE,RightOperand::BITS,- false>;+ false,+ typename Core::RightStageType>;return Loader(values,scales,@@ -96,7 +98,8 @@ static METAL_FUNC auto make_staged_loader(Core::THREADGROUP_THREADS,RightOperand::GROUP_SIZE,RightOperand::BITS,- true>;+ true,+ typename Core::RightStageType>;return Loader(values,scales,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex 1c5fb6e2e..889c931bc 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -34,6 +34,9 @@ struct SimdgroupMmaCore {using Right = RightOperand;using LeftElementType = typename Left::ElementType;using RightElementType = typename Right::ElementType;+ // Quantized weights are dequantized in f32 and staged as half, the fast pre-M5 MMA operand.+ using RightStageType = conditional_t<Right::QUANTIZED, half, RightElementType>;+ static_assert(sizeof(RightStageType) == sizeof(RightElementType), "staging memory is sized for the weight type");using LeftStorage = operands::LeftStorage<Left>;using RightStorage = operands::RightStorage<Right>;static_assert(!Left::QUANTIZED, "simdgroup MMA stages full-precision left operands only");@@ -65,7 +68,7 @@ struct SimdgroupMmaCore {THREADGROUP_THREADS>;using TileAccumulator = uzu::matmul::ThreadgroupTile<LeftElementType,- RightElementType,+ RightStageType,OutputElementType,THREADGROUP_BLOCK_M,THREADGROUP_BLOCK_N,@@ -82,7 +85,7 @@ struct SimdgroupMmaCore {template <uint GEMM_ALIGNMENT_RAW, typename BLoader>static METAL_FUNC void k_loop(threadgroup LeftElementType* a_shared,- threadgroup RightElementType* b_shared,+ threadgroup RightStageType* b_shared,const int aligned_k_iterations,thread ALoader& loader_a,thread BLoader& loader_b,@@ -244,7 +247,7 @@ struct SimdgroupMmaCore {constexpr uint gemm_alignment = gemm_alignment_mask.value;k_loop<gemm_alignment>(a_shared,- b_shared,+ reinterpret_cast<threadgroup RightStageType*>(b_shared),params->aligned_inner_iterations,loader_a,loader_b,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 5c7f457ff..fca6a18f4 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -54,7 +54,6 @@ VARIANTS(GEMM_TILING,GemmTiling::Tile8x32x32_Simdgroups1x1,GemmTiling::Tile64x32x32_Simdgroups2x2,- GemmTiling::Tile64x64x16_Simdgroups2x2,GemmTiling::Tile64x64x32_Simdgroups2x2,GemmTiling::Tile32x32x32_Simdgroups2x2,GemmTiling::Tile16x32x256_Simdgroups1x1,@@ -98,13 +97,9 @@ CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 4)CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision ||- (TRANSPOSE_B &&- GEMM_TILING != GemmTiling::Tile64x64x16_Simdgroups2x2 &&- GEMM_TILING != GemmTiling::Tile64x32x32_Simdgroups2x2))+ (TRANSPOSE_B && GEMM_TILING != GemmTiling::Tile64x32x32_Simdgroups2x2))CONSTRAINT(- B_PROLOGUE != GemmBPrologueKind::FullPrecision || USE_MXU ||- GEMM_TILING == GemmTiling::Tile64x64x16_Simdgroups2x2 ||- GEMM_TILING == GemmTiling::Tile64x32x32_Simdgroups2x2)+ B_PROLOGUE != GemmBPrologueKind::FullPrecision || USE_MXU || GEMM_TILING == GemmTiling::Tile64x32x32_Simdgroups2x2)CONSTRAINT(GEMM_TILING != GemmTiling::Tile16x32x256_Simdgroups1x1 ||(TRANSPOSE_B &&diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex c1da24799..7736a5f23 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -154,13 +154,7 @@ impl GemmKernel {shape: MatmulShape,context: &MetalContext,) -> GemmProblem {- GemmProblem::new(- shape,- self.weights_data_type,- self.output_data_type,- context.supports_mxu,- context.apple_gpu_family,- )+ GemmProblem::new(shape, self.weights_data_type, self.output_data_type, context.supports_mxu)}#[cfg(test)]diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rsindex 84bd19da1..7b5fe1b1e 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs@@ -4,8 +4,6 @@//! features to choices. Eligibility, fallbacks, and legality stay in//! `selection.rs`.-use metal::MTLGPUFamily;-use crate::backends::metal::gpu_types::gemm::GemmTiling;const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;@@ -21,10 +19,9 @@ const WIDE_N_MODERATE_K_RATIO: u32 = 6;const MXU_M_BUCKET_MAXES: [u32; 4] = [16, 63, 255, 511];const MXU_N_BUCKET_MAXES: [u32; 2] = [63, 127];-const SIMDGROUP_QUANT_SMALL_M_MAX: u32 = 32;-const SIMDGROUP_QUANT_NARROW_BLOCK_M: u32 = 8;-const SIMDGROUP_QUANT_LARGE_M_MIN: u32 = 64;-const SIMDGROUP_QUANT_WIDE_N_MIN: u32 = 6144;+// Two 8-row blocks: past this the 32-row tile has fewer, fuller blocks and wins on every chip.+const SIMDGROUP_QUANT_NARROW_TILE_M_MAX: u32 = 16;+const SIMDGROUP_QUANT_WIDE_BLOCK_M: u32 = 64;const SPLIT_K_TARGET_TILES_FP: u32 = 512;pub(super) const SPLIT_K_TARGET_TILES_A8: u32 = 256;@@ -99,42 +96,14 @@ pub(super) fn mxu_fp_tile(GemmTiling::Tile32x64x256_Simdgroups2x2}-pub(super) fn simdgroup_fp_tile(- m: u32,- n: u32,- k: u32,-) -> GemmTiling {- if 2_u32.saturating_mul(m.max(n)) > k {- GemmTiling::Tile64x64x16_Simdgroups2x2+pub(super) fn simdgroup_quant_tile(m: u32) -> GemmTiling {+ if m <= SIMDGROUP_QUANT_NARROW_TILE_M_MAX {+ GemmTiling::Tile8x32x32_Simdgroups1x1+ } else if (1..=SIMDGROUP_QUANT_WIDE_BLOCK_M / 2).contains(&(m % SIMDGROUP_QUANT_WIDE_BLOCK_M)) {+ // The last wide block would be at most half full: the narrow tile pads fewer rows.+ GemmTiling::Tile32x32x32_Simdgroups2x2} else {- GemmTiling::Tile64x32x32_Simdgroups2x2- }-}--/// A partial trailing M block costs a second pass over the weights. Older GPUs are bound by that;-/// Apple9 and newer would rather keep the narrow tile's parallelism.-fn prefers_wide_partial_m_tile(apple_gpu_family: MTLGPUFamily) -> bool {- apple_gpu_family <= MTLGPUFamily::Apple8-}--pub(super) fn simdgroup_quant_tile(- m: u32,- n: u32,- apple_gpu_family: MTLGPUFamily,-) -> GemmTiling {- if m < SIMDGROUP_QUANT_SMALL_M_MAX {- if !prefers_wide_partial_m_tile(apple_gpu_family)- || m <= SIMDGROUP_QUANT_NARROW_BLOCK_M- || m.is_multiple_of(SIMDGROUP_QUANT_NARROW_BLOCK_M)- {- GemmTiling::Tile8x32x32_Simdgroups1x1- } else {- GemmTiling::Tile32x32x32_Simdgroups2x2- }- } else if m >= SIMDGROUP_QUANT_LARGE_M_MIN && n >= SIMDGROUP_QUANT_WIDE_N_MIN && n.is_multiple_of(64) {GemmTiling::Tile64x64x32_Simdgroups2x2- } else {- GemmTiling::Tile32x32x32_Simdgroups2x2}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex 0c335dc3e..0e0958b1b 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -1,4 +1,3 @@-use metal::MTLGPUFamily;use thiserror::Error;use crate::{@@ -29,7 +28,6 @@ pub struct GemmProblem {weights_data_type: DataType,output_data_type: DataType,supports_mxu: bool,- apple_gpu_family: MTLGPUFamily,}#[derive(Debug, Clone, Copy, PartialEq, Error)]@@ -80,14 +78,12 @@ impl GemmProblem {weights_data_type: DataType,output_data_type: DataType,supports_mxu: bool,- apple_gpu_family: MTLGPUFamily,) -> Self {Self {shape,weights_data_type,output_data_type,supports_mxu,- apple_gpu_family,}}@@ -100,7 +96,7 @@ impl GemmProblem {} else {GemmEngine::Simdgroup};- self.finish_plan(engine, select_tiling(self.shape, engine, self.apple_gpu_family))+ self.finish_plan(engine, select_tiling(self.shape, engine))}pub fn small_batch_plan(@@ -131,7 +127,7 @@ impl GemmProblem {return Ok(plan);}self.validate_engine(engine)?;- Ok(self.finish_plan(engine, select_tiling(self.shape, engine, self.apple_gpu_family)))+ Ok(self.finish_plan(engine, select_tiling(self.shape, engine)))}pub(super) fn validate_engine(@@ -192,8 +188,6 @@ impl GemmProblem {}fn split_k_output_supported(&self) -> bool {- use crate::backends::common::gpu_types::gemm::GemmDTransform;-let mut output_transform = self.shape.d_transform;if self.shape.is_quant()&& output_transform.contains(GemmDTransform::RHT)@@ -206,7 +200,7 @@ impl GemmProblem {}}-pub(super) fn select_trellis_plan(+fn select_trellis_plan(m: u32,n: u32,k: u32,@@ -266,11 +260,10 @@ fn mxu_is_eligible(shape: MatmulShape) -> bool {fn select_tiling(shape: MatmulShape,engine: GemmEngine,- apple_gpu_family: MTLGPUFamily,) -> GemmTiling {match engine {- GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m, shape.n, apple_gpu_family),- GemmEngine::Simdgroup => policy::simdgroup_fp_tile(shape.m, shape.n, shape.k),+ GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m),+ GemmEngine::Simdgroup => GemmTiling::Tile64x32x32_Simdgroups2x2,GemmEngine::Mxu if shape.a_is_quantized() => match policy::mxu_int8_gemm_tile(shape.m, shape.n) {tilingif tiling.simdgroups_k() > 1diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rsindex 0f8cb2afe..408e9a328 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs@@ -1,5 +1,3 @@-use metal::MTLGPUFamily;-use crate::{backends::{common::{@@ -44,7 +42,7 @@ pub fn supports_integer_right_operand(shape: &MatmulShape) -> bool {}impl MatmulMetalKernel {- pub fn prefer_gemm_over_gemv(+ fn prefer_gemm_over_gemv(shape: MatmulShape,plan: GemmPlan,weights_data_type: DataType,@@ -84,13 +82,12 @@ impl MatmulMetalKernel {shape: &MatmulShape,gemm_b_layout: GemmBLayout,gpu_core_count: u32,- apple_gpu_family: MTLGPUFamily,supports_mxu: bool,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,) -> MatmulDispatch {- let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);+ let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu);if shape.b_is_trellis {return MatmulDispatch::Gemm(problem.select_plan());}@@ -138,7 +135,6 @@ impl MatmulMetalKernel {shape,gemm_b_layout,context.gpu_core_count,- context.apple_gpu_family,context.supports_mxu,self.weights_data_type,self.input_data_type,diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/device_extensions.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/device_extensions.rsindex 931420981..1c3a4d987 100644--- a/crates/uzu-engine/src/backends/metal/metal_extensions/device_extensions.rs+++ b/crates/uzu-engine/src/backends/metal/metal_extensions/device_extensions.rs@@ -1,7 +1,7 @@use core::mem::transmute;use std::ffi::CString;-use metal::{MTLDevice, MTLGPUFamily};+use metal::MTLDevice;use obfstr::obfstr;use objc2::{Message,@@ -9,8 +9,6 @@ use objc2::{runtime::{NSObjectProtocol, ProtocolObject, Sel},};-use super::GpuFamilyExt;-// Used to bypass objc2's debug-mode class validation for proxy objects like// `CaptureMTLDevice` which forward messages at the ObjC runtime level (via// `forwardingTargetForSelector:`) but don't have the methods in their own class@@ -69,15 +67,6 @@ pub trait DeviceExt: MTLDevice + Message + NSObjectProtocol + Sized {fn supports_placement_sparse_resources(&self) -> bool {optional_selector_value(self, obfstr!("supportsPlacementSparse"), false)}-- /// Newest Apple GPU family the device supports.- fn newest_supported_apple_gpu_family(&self) -> MTLGPUFamily {- MTLGPUFamily::all_cases()- .into_iter()- .rev()- .find(|family| self.supports_family(*family))- .expect("unsupported Metal device: uzu requires an Apple GPU")- }}impl DeviceExt for ProtocolObject<dyn MTLDevice> {}diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/gpu_family_extensions.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/gpu_family_extensions.rsdeleted file mode 100644index a376ad86c..000000000--- a/crates/uzu-engine/src/backends/metal/metal_extensions/gpu_family_extensions.rs+++ /dev/null@@ -1,23 +0,0 @@-use metal::MTLGPUFamily;--pub trait GpuFamilyExt: Sized {- /// All Apple GPU family cases, oldest to newest.- fn all_cases() -> [Self; 10];-}--impl GpuFamilyExt for MTLGPUFamily {- fn all_cases() -> [Self; 10] {- [- Self::Apple1,- Self::Apple2,- Self::Apple3,- Self::Apple4,- Self::Apple5,- Self::Apple6,- Self::Apple7,- Self::Apple8,- Self::Apple9,- Self::Apple10,- ]- }-}diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rsindex 88e550683..5b9cd8012 100644--- a/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rs+++ b/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rs@@ -1,13 +1,11 @@mod data_type;mod device_extensions;mod function_constant_values_extensions_set_value;-mod gpu_family_extensions;mod library_extensions_pipeline;mod sparse_page_size_extensions;pub use data_type::MetalDataTypeExt;pub use device_extensions::DeviceExt;pub use function_constant_values_extensions_set_value::FunctionConstantValuesSetValue;-pub use gpu_family_extensions::GpuFamilyExt;pub use library_extensions_pipeline::LibraryPipelineExtensions;pub use sparse_page_size_extensions::SparsePageSizeExt;diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex 913832e7e..19d9126b1 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -1,15 +1,10 @@use futures::executor::block_on;-use metal::MTLGPUFamily;use uzu_engine_macros::uzu_test;use super::*;use crate::{backends::{- common::{- Context,- gpu_types::gemm::GemmDTransform,- kernel::matmul::{MatmulKernel, QuantParamsLayout},- },+ common::{Context, kernel::matmul::MatmulKernel},metal::{context::MetalContext, kernel::matmul::MatmulMetalKernel},},tests::matmul::{a8_format, matmul_shape as shape, quantized_matmul_shape},@@ -27,73 +22,12 @@ fn problem(shape: MatmulShape,output_data_type: DataType,) -> GemmProblem {- GemmProblem::new(shape, DataType::BF16, output_data_type, true, MTLGPUFamily::Apple7)+ GemmProblem::new(shape, DataType::BF16, output_data_type, true)}#[uzu_test]-fn policy_boundaries_are_preserved() {- use GemmTiling::*;-- for (is_a_int8, m, n, expected) in [- (true, 16, 4096, Tile16x32x256_Simdgroups1x1),- (true, 17, 4096, Tile32x64x256_Simdgroups2x2),- (true, 64, 63, Tile64x32x256_Simdgroups4x1),- (true, 512, 4096, Tile128x128x256_Simdgroups4x4),- (false, 16, 4096, Tile32x64x256_Simdgroups2x2),- (false, 64, 4096, Tile64x64x256_Simdgroups2x2),- (false, 256, 4096, Tile128x128x256_Simdgroups4x4),- ] {- assert_eq!(policy::mxu_mn_tile(is_a_int8, m, n), expected);- }-- for (m, n, k, expected) in [- (15, 2560, 2560, Tile16x32x256_Simdgroups1x1),- (15, 4096, 8192, Tile16x128x256_Simdgroups1x4),- (15, 131_073, 4096, Tile16x32x256_Simdgroups1x1),- (64, 4096, 2048, Tile64x64x256_Simdgroups2x2),- (256, 4096, 2048, Tile128x128x256_Simdgroups4x4),- ] {- assert_eq!(policy::mxu_fp_tile(m, n, k), expected);- }-- for (m, n, family, expected) in [- (16, 4096, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),- (8, 4096, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),- (9, 4096, MTLGPUFamily::Apple8, Tile32x32x32_Simdgroups2x2),- (9, 4096, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),- (31, 4096, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),- (32, 4096, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),- (64, 6143, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),- (64, 6144, MTLGPUFamily::Apple9, Tile64x64x32_Simdgroups2x2),- ] {- assert_eq!(policy::simdgroup_quant_tile(m, n, family), expected);- }-}--#[uzu_test]-fn selection_fallbacks_and_split_k_are_preserved() {+fn illegal_layouts_fall_back() {use GemmEngine::*;- use GemmTiling::*;-- for (mut case, expect_split) in [(shape(4, 32, 1024), true), (shape(64, 128, 128), false)] {- case.d_transform = GemmDTransform::RHT | GemmDTransform::BIAS;- for engine in [Simdgroup, Mxu] {- let plan = problem(case, DataType::BF16)- .select_plan_for_engine(engine)- .expect("parity anchor supports the forced GEMM engine");- assert_eq!(plan.split_k > 1, expect_split, "forced {engine:?} RHT+bias parity anchor");- }- }-- let mut non_split_quant_rht_bias = shape(9, 64, 64);- non_split_quant_rht_bias.b_prologue = GemmBPrologueKind::ScaleZeroPointDequant;- non_split_quant_rht_bias.b_bits = Some(4);- non_split_quant_rht_bias.b_group_size = Some(64);- non_split_quant_rht_bias.params_layout = Some(QuantParamsLayout::GroupOutput);- non_split_quant_rht_bias.d_transform = GemmDTransform::RHT | GemmDTransform::BIAS;- let plan = problem(non_split_quant_rht_bias, DataType::BF16).select_plan();- assert_eq!(plan.engine, Simdgroup);- assert_eq!(plan.split_k, 1);let mut invalid_layout = quant(shape(64, 4096, 4096));invalid_layout.b_transpose = false;@@ -101,64 +35,13 @@ fn selection_fallbacks_and_split_k_are_preserved() {invalid_layout.a = a8_format();assert_eq!(problem(invalid_layout, DataType::BF16).select_plan().engine, Mxu);assert_eq!(problem(quant(shape(64, 4096, 4095)), DataType::BF16).select_plan().engine, Simdgroup);-- for (m, n, k, expected_tiling, expected_split_k) in [- (16, 4096, 4096, Tile16x32x256_Simdgroups1x1x4, 2),- (16, 1024, 4096, Tile16x32x256_Simdgroups1x1x4, 8),- (16, 34816, 4096, Tile16x32x256_Simdgroups1x1x4, 1),- (8, 4096, 384, Tile16x32x256_Simdgroups1x1, 1),- ] {- let mut a8 = quant(shape(m, n, k));- a8.a = a8_format();- let plan = problem(a8, DataType::BF16).select_plan();- assert_eq!(plan.tiling, expected_tiling);- assert_eq!(plan.split_k, expected_split_k);- }-- let mut biased = quant(shape(16, 4096, 4096));- biased.a = a8_format();- biased.d_transform = GemmDTransform::BIAS;- assert_eq!(- GemmProblem::new(biased, DataType::BF16, DataType::F32, true, MTLGPUFamily::Apple7).select_plan().split_k,- 1- );-- let mut zero = quant(shape(0, 1, 1));- zero.b_prologue = GemmBPrologueKind::ScaleZeroPointDequant;- zero.b_group_size = Some(u32::MAX);- assert_eq!(problem(zero, DataType::BF16).select_plan().split_k, 1);-}--#[uzu_test]-fn trellis_plan_matches_projection_cases() {- use GemmTiling::*;-- for (m, n, k, tiling, split_k) in [- (16, 128, 64, Tile16x32x256_Simdgroups1x1, 1),- (16, 4096, 2048, Tile32x64x256_Simdgroups2x2, 4),- (16, 5120, 5120, Tile32x64x256_Simdgroups2x2, 2),- (16, 5120, 5184, Tile16x32x256_Simdgroups1x1, 3),- (16, 4096, 1984, Tile16x32x256_Simdgroups1x1, 1),- (16, 32, 5120, Tile64x64x256_Simdgroups2x2, 80),- (17, 12_288, 2048, Tile64x64x256_Simdgroups2x2, 2),- (17, 80, 64, Tile64x64x256_Simdgroups2x2, 1),- (2048, 2048, 128, Tile128x128x256_Simdgroups4x4, 1),- (1, 80, 5120, Tile16x32x256_Simdgroups1x1, 80),- ] {- let mut trellis_shape = shape(m, n, k);- trellis_shape.a = ActivationFormat::Trellis;- trellis_shape.b_is_trellis = true;- let plan = problem(trellis_shape, DataType::BF16).select_plan();- assert_eq!((plan.engine, plan.tiling, plan.split_k), (GemmEngine::Mxu, tiling, split_k), "M {m} N {n} K {k}");- }}#[uzu_test]fn forced_engine_errors_are_preserved() {let huge = shape(u32::MAX, u32::MAX, u32::MAX);assert_eq!(- GemmProblem::new(huge, DataType::BF16, DataType::BF16, false, MTLGPUFamily::Apple7)- .select_plan_for_engine(GemmEngine::Mxu),+ GemmProblem::new(huge, DataType::BF16, DataType::BF16, false).select_plan_for_engine(GemmEngine::Mxu),Err(GemmPlanError::MxuUnavailable));@@ -170,24 +53,6 @@ fn forced_engine_errors_are_preserved() {);}-#[uzu_test]-fn gemv_gemm_route_boundaries_are_preserved() {- for (shape, data_type, prefer_gemm) in [- (shape(4, 4096, 8192), DataType::BF16, true),- (shape(4, 8192, 4096), DataType::BF16, false),- (shape(4, 4096, 8192), DataType::F32, false),- (shape(3, 4096, 8192), DataType::BF16, false),- (shape(5, 4096, 8192), DataType::BF16, false),- ] {- let plan = GemmProblem::new(shape, data_type, data_type, true, MTLGPUFamily::Apple7).select_plan();- assert_eq!(MatmulMetalKernel::prefer_gemm_over_gemv(shape, plan, data_type, data_type, data_type), prefer_gemm);- }- let mut gathered = shape(4, 4096, 8192);- gathered.gathered = true;- let plan = problem(gathered, DataType::BF16).select_plan();- assert!(!MatmulMetalKernel::prefer_gemm_over_gemv(gathered, plan, DataType::BF16, DataType::BF16, DataType::BF16,));-}-#[uzu_test]fn int8_activations_require_mxu_tiled_weights() {let context = block_on(MetalContext::new()).expect("Metal context");diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rsindex da07743a8..36d109f78 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs@@ -1,112 +1,16 @@use uzu_engine_macros::uzu_test;-use super::{DEFAULT_RESULTS_PER_SIMDGROUP, GemvTile, fp_tile, quantized_tile, tile};+use super::GemvTile;use crate::{- backends::{- common::kernel::matmul::{MatmulShape, QuantParamsLayout},- metal::gpu_types::gemm::GemmBLayout,- },- data_type::DataType,+ backends::{common::kernel::matmul::MatmulShape, metal::gpu_types::gemm::GemmBLayout},tests::matmul::quantized_matmul_shape as shape,};-const LARGE_GPU_CORE_COUNT: u32 = 30;--fn qtile(- num_simdgroups: u32,- results_per_simdgroup: u32,-) -> GemvTile {- GemvTile::quantized(num_simdgroups, results_per_simdgroup, 1, 32, 2)-}--#[uzu_test]-fn fp_policy_cases() {- let cases = [- (1, 12288, 1536, true, tile(8, 8, 1)),- (1, 12288, 1536, false, tile(8, 1, 1)),- (1, 1536, 256, true, tile(8, 2, 1)),- (8, 12288, 1536, true, tile(8, 1, 4)),- (1, 1536, 12288, true, tile(8, 8, 1)),- (1, 262144, 1536, true, tile(8, 8, 1)),- ];-- for (m, n, k, aligned, expected) in cases {- assert_eq!(fp_tile(m, n, k, aligned), expected, "m={m} n={n} k={k} aligned={aligned}");- }-}--#[uzu_test]-fn quant_policy_cases() {- let cases = [- (1, 256, 4, qtile(8, 4)),- (1, 262144, 4, qtile(8, 4)),- (2, 2048, 4, qtile(8, 4)),- (4, 5120, 4, qtile(8, 4)),- (1, 2048, 8, GemvTile::quantized(8, 4, 1, 32, 4)),- ];-- for (m, n, bits, expected) in cases {- let actual = quantized_tile(bits, 32, m, n, true).expect("quantized route");- assert_eq!(actual, expected, "m={m} n={n} bits={bits}");- }-}--#[uzu_test]-fn quantized_policy_edges() {- assert_eq!(quantized_tile(4, 64, 4, 8192, true), Some(GemvTile::quantized(8, 4, 1, 32, 4)));- assert!(quantized_tile(4, 32, 4, 8, true).is_some());- assert!(quantized_tile(4, 32, 1, 8192, true).is_some());- assert_eq!(quantized_tile(4, 32, 9, 8192, true), None);- assert_eq!(quantized_tile(4, 32, 6, 8192, true), None);- assert_eq!(quantized_tile(4, 32, 6, 64, true), Some(GemvTile::quantized(8, 4, 1, 32, 2)));- assert_eq!(quantized_tile(4, 32, 1, 8192, false), None);- assert_eq!(quantized_tile(4, 32, 4, 8192, true), Some(GemvTile::quantized(8, 4, 1, 32, 2)));-}--#[uzu_test]-fn block_unaligned_quantized_k_stays_on_gemv() {- let shape = MatmulShape {- params_layout: Some(QuantParamsLayout::OutputGroup),- ..shape(1, 64, 1152, 4, 64)- };- assert!(- super::super::kernel::GemvSpecialization::select_shape(- &shape,- GemmBLayout::RowMajor,- DataType::BF16,- DataType::BF16,- DataType::BF16,- LARGE_GPU_CORE_COUNT,- )- .is_some()- );-}--#[uzu_test]-fn specialization_preserves_quantized_route() {- let select = |n| {- super::super::kernel::GemvSpecialization::select_shape(- &MatmulShape {- params_layout: Some(QuantParamsLayout::OutputGroup),- ..shape(4, n, 4096, 4, 32)- },- GemmBLayout::RowMajor,- DataType::BF16,- DataType::BF16,- DataType::BF16,- LARGE_GPU_CORE_COUNT,- )- };- let clean = select(8192).expect("quantized specialization");- assert!(clean.output_row_tile() > DEFAULT_RESULTS_PER_SIMDGROUP);- assert!(select(8192 + clean.output_row_tile() / 2).is_some());-}-#[uzu_test]fn gathered_group_major_is_a_gemv_route() {- let mut shape = shape(1, 8192, 4096, 4, 32);+ // Gathered readout has no GEMM implementation, so group-major gathered rows must stay on GEMV.+ let mut shape: MatmulShape = shape(1, 8192, 4096, 4, 32);shape.gathered = true;- assert!(- super::super::kernel::GemvSpecialization::select_tile(&shape, GemmBLayout::RowMajor, qtile(8, 4),).is_some()- );+ let tile = GemvTile::quantized(8, 4, 1, 32, 2);+ assert!(super::super::kernel::GemvSpecialization::select_tile(&shape, GemmBLayout::RowMajor, tile).is_some());}