Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Faster pre-M5 prefill GEMM: f32 dequant staged as half, batch-only tile rule (#1061)

CCYehea52b58792fe
diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rs
index 35c8a2873..e04858e76 100644
--- a/crates/uzu-engine/src/backends/metal/context.rs
+++ b/crates/uzu-engine/src/backends/metal/context.rs
@@ -9,8 +9,8 @@ use std::{
use metal::{
MTL4CommandAllocator, MTL4CommandQueue, MTL4CommandQueueDescriptor, MTLCaptureTarget, MTLComputePipelineState,
- MTLDevice, MTLDeviceExt, MTLEvent, MTLEventExt, MTLFunctionConstantValues, MTLGPUFamily, MTLLibrary,
- MTLResidencySet, MTLResidencySetDescriptor, MTLSparsePageSize,
+ MTLDevice, MTLDeviceExt, MTLEvent, MTLEventExt, MTLFunctionConstantValues, MTLLibrary, MTLResidencySet,
+ MTLResidencySetDescriptor, MTLSparsePageSize,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
use parking_lot::Mutex;
@@ -42,7 +42,6 @@ pub(super) struct SparseState {
pub struct MetalContext {
pub device: Retained<ProtocolObject<dyn MTLDevice>>,
pub gpu_core_count: u32,
- pub apple_gpu_family: MTLGPUFamily,
pub supports_mxu: bool,
pub device_name: String,
pub(super) residency_set: Arc<Mutex<Retained<ProtocolObject<dyn MTLResidencySet>>>>,
@@ -124,7 +123,6 @@ impl Context for MetalContext {
let device_name = device.name();
let gpu_core_count = device.gpu_core_count();
- let apple_gpu_family = device.newest_supported_apple_gpu_family();
let supports_mxu = device.supports_mxu();
let peak_memory_usage = Arc::new(AtomicUsize::new(0));
@@ -162,7 +160,6 @@ impl Context for MetalContext {
Ok(Arc::new_cyclic(|weak_self: &Weak<Self>| Self {
device,
gpu_core_count,
- apple_gpu_family,
supports_mxu,
device_name,
residency_set: Arc::new(Mutex::new(residency_set)),
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
index 4c51fa007..8b536d3e4 100644
--- a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
@@ -23,7 +23,6 @@ pub enum GemmBLayout {
pub enum GemmTiling {
Tile8x32x32_Simdgroups1x1,
Tile64x32x32_Simdgroups2x2,
- Tile64x64x16_Simdgroups2x2,
Tile64x64x32_Simdgroups2x2,
Tile32x32x32_Simdgroups2x2,
Tile16x32x256_Simdgroups1x1,
@@ -48,7 +47,6 @@ impl GemmTiling {
match self {
Self::Tile8x32x32_Simdgroups1x1 => 8,
Self::Tile64x32x32_Simdgroups2x2 => 64,
- Self::Tile64x64x16_Simdgroups2x2 => 64,
Self::Tile64x64x32_Simdgroups2x2 => 64,
Self::Tile32x32x32_Simdgroups2x2 => 32,
Self::Tile16x32x256_Simdgroups1x1 => 16,
@@ -66,7 +64,6 @@ impl GemmTiling {
match self {
Self::Tile8x32x32_Simdgroups1x1 => 32,
Self::Tile64x32x32_Simdgroups2x2 => 32,
- Self::Tile64x64x16_Simdgroups2x2 => 64,
Self::Tile64x64x32_Simdgroups2x2 => 64,
Self::Tile32x32x32_Simdgroups2x2 => 32,
Self::Tile16x32x256_Simdgroups1x1 => 32,
@@ -84,7 +81,6 @@ impl GemmTiling {
match self {
Self::Tile8x32x32_Simdgroups1x1 => 32,
Self::Tile64x32x32_Simdgroups2x2 => 32,
- Self::Tile64x64x16_Simdgroups2x2 => 16,
Self::Tile64x64x32_Simdgroups2x2 => 32,
Self::Tile32x32x32_Simdgroups2x2 => 32,
Self::Tile16x32x256_Simdgroups1x1 => 256,
@@ -103,7 +99,6 @@ impl GemmTiling {
match self {
Self::Tile8x32x32_Simdgroups1x1 => 1,
Self::Tile64x32x32_Simdgroups2x2 => 2,
- Self::Tile64x64x16_Simdgroups2x2 => 2,
Self::Tile64x64x32_Simdgroups2x2 => 2,
Self::Tile32x32x32_Simdgroups2x2 => 2,
Self::Tile16x32x256_Simdgroups1x1 => 1,
@@ -121,7 +116,6 @@ impl GemmTiling {
match self {
Self::Tile8x32x32_Simdgroups1x1 => 1,
Self::Tile64x32x32_Simdgroups2x2 => 2,
- Self::Tile64x64x16_Simdgroups2x2 => 2,
Self::Tile64x64x32_Simdgroups2x2 => 2,
Self::Tile32x32x32_Simdgroups2x2 => 2,
Self::Tile16x32x256_Simdgroups1x1 => 1,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
index 34d1c9dda..e46959984 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
@@ -40,18 +40,17 @@ enum class GemmBLayout : uint32_t {
enum class GemmTiling : uint32_t {
Tile8x32x32_Simdgroups1x1 = 0,
Tile64x32x32_Simdgroups2x2 = 1,
- Tile64x64x16_Simdgroups2x2 = 2,
- Tile64x64x32_Simdgroups2x2 = 3,
- Tile32x32x32_Simdgroups2x2 = 4,
- Tile16x32x256_Simdgroups1x1 = 5,
- Tile16x32x256_Simdgroups1x1x4 = 6,
- Tile16x128x256_Simdgroups1x4 = 7,
- Tile32x64x256_Simdgroups2x2 = 8,
- Tile64x32x256_Simdgroups4x1 = 9,
- Tile64x64x256_Simdgroups2x2 = 10,
- Tile128x128x256_Simdgroups4x4 = 11,
- Tile8x32x64_Simdgroups1x2x4 = 12,
- Tile8x32x128_Simdgroups1x2x4 = 13,
+ Tile64x64x32_Simdgroups2x2 = 2,
+ Tile32x32x32_Simdgroups2x2 = 3,
+ Tile16x32x256_Simdgroups1x1 = 4,
+ Tile16x32x256_Simdgroups1x1x4 = 5,
+ Tile16x128x256_Simdgroups1x4 = 6,
+ Tile32x64x256_Simdgroups2x2 = 7,
+ Tile64x32x256_Simdgroups4x1 = 8,
+ Tile64x64x256_Simdgroups2x2 = 9,
+ Tile128x128x256_Simdgroups4x4 = 10,
+ Tile8x32x64_Simdgroups1x2x4 = 11,
+ Tile8x32x128_Simdgroups1x2x4 = 12,
};
static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
index ffe4ce189..15492da5f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
@@ -84,37 +84,34 @@ METAL_FUNC float magic_code_float(uint code) {
METAL_FUNC float magic_dequant_bias(float scale) { return -12582912.0f * scale; }
+// fma(code, scale, bias) in f32, rounded once to U, stored in pairs.
template <typename U, int N, int bits>
-inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local, const bool signed_codes) {
+inline void dequantize(
+ const device uint8_t* w,
+ float scale,
+ float bias,
+ threadgroup U* w_local,
+ const bool signed_codes
+) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
-
+ static_assert(bits == 8 || N % 2 == 0, "int4 values are unpacked in pairs");
+ threadgroup vec<U, 2>* pairs = reinterpret_cast<threadgroup vec<U, 2>*>(w_local);
if constexpr (bits == 4) {
- U s0 = scale;
- U s1 = scale / static_cast<U>(16.0f);
- // Keeping literal masks in each arm lets the compiler vectorize the unpack.
- if (signed_codes) {
- for (int i = 0; i < (N / 2); i++) {
- const uint8_t word = w[i] ^ uint8_t(0x88u);
- w_local[2 * i] = s0 * (word & 0x0f) + bias;
- w_local[2 * i + 1] = s1 * (word & 0xf0) + bias;
- }
- } else {
- for (int i = 0; i < (N / 2); i++) {
- w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;
- w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;
- }
+ const uint sign_flip = signed_codes ? 0x88u : 0u;
+ for (int i = 0; i < (N / 2); i++) {
+ const uint code = uint(w[i]) ^ sign_flip;
+ pairs[i] = vec<U, 2>(vec<float, 2>(fma(float(code & 0x0fu), scale, bias), fma(float(code >> 4), scale, bias)));
+ }
+ } else {
+ const uint sign_flip = signed_codes ? 0x80u : 0u;
+ for (int i = 0; i < (N / 2); i++) {
+ pairs[i] = vec<U, 2>(vec<float, 2>(
+ fma(float(uint(w[2 * i]) ^ sign_flip), scale, bias),
+ fma(float(uint(w[2 * i + 1]) ^ sign_flip), scale, bias)
+ ));
}
- } else if constexpr (bits == 8) {
- if (signed_codes) {
- const device int8_t* signed_weights = reinterpret_cast<const device int8_t*>(w);
- const U adjusted_bias = bias + scale * U(128);
- for (int i = 0; i < N; i++) {
- w_local[i] = scale * U(signed_weights[i]) + adjusted_bias;
- }
- } else {
- for (int i = 0; i < N; i++) {
- w_local[i] = scale * U(w[i]) + bias;
- }
+ if constexpr (N % 2 == 1) {
+ w_local[N - 1] = U(fma(float(uint(w[N - 1]) ^ sign_flip), scale, bias));
}
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
index 9a3a0876e..10ca9612f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
@@ -132,15 +132,17 @@ struct ThreadgroupTile {
c_fragment.map([&](AccumulatorType value) { return op.apply(value); });
}
- METAL_FUNC void apply_bias(const device BT* bias) {
- const device BT* bias_ptr = bias + simdgroup_col_offset;
+ template <typename BiasT>
+ METAL_FUNC void apply_bias(const device BiasT* bias) {
+ const device BiasT* bias_ptr = bias + simdgroup_col_offset;
for_each_output([&](ushort, ushort col_offset, ushort k, thread AccumulatorType& v) {
v += static_cast<AccumulatorType>(bias_ptr[col_offset + k]);
});
}
- METAL_FUNC void apply_bias_safe(const device BT* bias, short2 tile_dimensions) {
- const device BT* bias_ptr = bias + simdgroup_col_offset;
+ template <typename BiasT>
+ METAL_FUNC void apply_bias_safe(const device BiasT* bias, short2 tile_dimensions) {
+ const device BiasT* bias_ptr = bias + simdgroup_col_offset;
tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
for_each_output([&](ushort row_offset, ushort col_offset, ushort k, thread AccumulatorType& v) {
if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x)
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
index 89528b0de..7c70a5186 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
@@ -8,7 +8,6 @@ namespace gemm {
constexpr uint gemm_tiling_block_m(GemmTiling t) {
return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 8
: t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 64
- : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 64
: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64
: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32
: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 16
@@ -26,7 +25,6 @@ constexpr uint gemm_tiling_block_m(GemmTiling t) {
constexpr uint gemm_tiling_block_n(GemmTiling t) {
return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 32
: t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 32
- : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 64
: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64
: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32
: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 32
@@ -44,7 +42,6 @@ constexpr uint gemm_tiling_block_n(GemmTiling t) {
constexpr uint gemm_tiling_block_k(GemmTiling t) {
return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 32
: t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 32
- : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 16
: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 32
: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32
: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 256
@@ -62,7 +59,6 @@ constexpr uint gemm_tiling_block_k(GemmTiling t) {
constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {
return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 1
: t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 2
- : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 2
: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2
: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2
: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 1
@@ -80,7 +76,6 @@ constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {
constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {
return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 1
: t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 2
- : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 2
: t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2
: t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2
: t == GemmTiling::Tile16x32x256_Simdgroups1x1 ? 1
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 567890ed9..0e300983c 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -34,6 +34,7 @@ struct MxuMmaCore {
using Right = RightOperand;
using LeftElementType = typename Left::ElementType;
using RightElementType = typename Right::ElementType;
+ using RightStageType = RightElementType;
using LeftStorage = operands::LeftStorage<Left>;
using RightStorage = operands::RightStorage<Right>;
using FragmentOps = uzu::matmul::MxuFragmentOps<>;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
index 9fe71f078..ff5ebaa18 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
@@ -19,7 +19,8 @@ template <
short DESTINATION_LEADING_DIMENSION,
short THREADGROUP_SIZE,
short GROUP_SIZE,
- short BITS>
+ short BITS,
+ typename StageT = T>
struct QuantizedBlockLoaderScaleBias {
static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");
static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");
@@ -33,6 +34,10 @@ struct QuantizedBlockLoaderScaleBias {
: (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;
UZU_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;
UZU_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
+ static_assert(
+ DESTINATION_LEADING_DIMENSION % 2 == 0 && (READS_PER_THREAD * PACK_FACTOR) % 2 == 0,
+ "paired stores need 4-byte alignment"
+ );
const int src_leading_dim;
const int tile_stride;
@@ -43,7 +48,7 @@ struct QuantizedBlockLoaderScaleBias {
const short tile_row_index;
const short tile_col_index;
- threadgroup T* dst;
+ threadgroup StageT* dst;
const device uint8_t* src;
const device T* scales;
const device T* biases;
@@ -65,7 +70,8 @@ struct QuantizedBlockLoaderScaleBias {
group_step_counter(0), group_stride(params_group_stride_), thread_index(simd_group_id * 32 + simd_lane_id),
tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),
tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),
- dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index * PACK_FACTOR),
+ dst(reinterpret_cast<threadgroup StageT*>(dst_) + tile_row_index * DESTINATION_LEADING_DIMENSION +
+ tile_col_index * PACK_FACTOR),
src(src_ + tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR + tile_col_index * BYTES_PER_PACK),
scales(scales_ + tile_row_index * params_output_stride_),
biases(biases_ + tile_row_index * params_output_stride_), signed_codes(signed_codes_) {}
@@ -77,11 +83,7 @@ struct QuantizedBlockLoaderScaleBias {
}
}
- T scale = *scales;
- T bias = *biases;
- for (int i = 0; i < READS_PER_THREAD; i++) {
- dequantize<T, PACK_FACTOR, BITS>(src + i * BYTES_PER_PACK, scale, bias, dst + i * PACK_FACTOR, signed_codes);
- }
+ dequantize<StageT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, float(*scales), float(*biases), dst, signed_codes);
}
void load_safe(short2 src_tile_dim) const {
@@ -93,28 +95,34 @@ struct QuantizedBlockLoaderScaleBias {
if (tile_row_index >= src_tile_dim.y) {
for (int i = 0; i < READS_PER_THREAD * PACK_FACTOR; i++) {
- dst[i] = T(0);
+ dst[i] = StageT(0);
}
return;
}
const int valid_cols = src_tile_dim.x;
const int valid_packs = (valid_cols + PACK_FACTOR - 1) / PACK_FACTOR;
- T scale = *scales;
- T bias = *biases;
+ const float scale = float(*scales);
+ const float bias = float(*biases);
for (int i = 0; i < READS_PER_THREAD; i++) {
const int pack_index = tile_col_index + i;
if (pack_index < valid_packs) {
- dequantize<T, PACK_FACTOR, BITS>(src + i * BYTES_PER_PACK, scale, bias, dst + i * PACK_FACTOR, signed_codes);
+ dequantize<StageT, PACK_FACTOR, BITS>(
+ src + i * BYTES_PER_PACK,
+ scale,
+ bias,
+ dst + i * PACK_FACTOR,
+ signed_codes
+ );
if (pack_index == valid_packs - 1) {
const int remaining = valid_cols - pack_index * PACK_FACTOR;
for (int lane = remaining; lane < PACK_FACTOR; ++lane) {
- dst[i * PACK_FACTOR + lane] = T(0);
+ dst[i * PACK_FACTOR + lane] = StageT(0);
}
}
} else {
for (int lane = 0; lane < PACK_FACTOR; ++lane) {
- dst[i * PACK_FACTOR + lane] = T(0);
+ dst[i * PACK_FACTOR + lane] = StageT(0);
}
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
index 0cdd8d38e..87f7fe985 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
@@ -20,7 +20,8 @@ template <
short THREADGROUP_SIZE,
short GROUP_SIZE,
short BITS,
- bool SCALE_SYMMETRIC = false>
+ bool SCALE_SYMMETRIC = false,
+ typename StageT = T>
struct QuantizedBlockLoaderScaleZeroPoint {
static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");
static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");
@@ -34,6 +35,10 @@ struct QuantizedBlockLoaderScaleZeroPoint {
: (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;
UZU_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;
UZU_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
+ static_assert(
+ DESTINATION_LEADING_DIMENSION % 2 == 0 && (READS_PER_THREAD * PACK_FACTOR) % 2 == 0,
+ "paired stores need 4-byte alignment"
+ );
const int src_leading_dim;
const int tile_stride;
@@ -45,7 +50,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {
const short tile_row_index;
const short tile_col_index;
- threadgroup T* dst;
+ threadgroup StageT* dst;
const device uint8_t* src;
const device T* scales;
const device uint8_t* zero_points_row_start;
@@ -72,7 +77,8 @@ struct QuantizedBlockLoaderScaleZeroPoint {
thread_index(simd_group_id * 32 + simd_lane_id),
tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),
tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),
- dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index * PACK_FACTOR),
+ dst(reinterpret_cast<threadgroup StageT*>(dst_) + tile_row_index * DESTINATION_LEADING_DIMENSION +
+ tile_col_index * PACK_FACTOR),
src(src_ + tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR + tile_col_index * BYTES_PER_PACK),
scales(scales_ + tile_row_index * params_output_stride_),
zero_points_row_start(SCALE_SYMMETRIC ? nullptr : zero_points_row_start_),
@@ -108,10 +114,9 @@ struct QuantizedBlockLoaderScaleZeroPoint {
static_assert(SCALE_SYMMETRIC, "zero-point-free loader construction requires symmetric quantization");
}
- inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {
+ inline void current_scale_bias(thread float& out_scale, thread float& out_bias) const {
uint zero_point_value;
- T scale_value;
- scale_value = *scales;
+ const float scale_value = float(*scales);
if constexpr (SCALE_SYMMETRIC) {
zero_point_value = symmetric_zero_point<ushort(BITS)>();
} else {
@@ -120,7 +125,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {
zero_point_value = decode_zero_point<ushort(BITS)>(zero_points_row_start[byte_index], packed_index);
}
out_scale = scale_value;
- out_bias = static_cast<T>(-scale_value * static_cast<T>(zero_point_value));
+ out_bias = -scale_value * float(zero_point_value);
}
void load_unsafe() const {
@@ -130,12 +135,10 @@ struct QuantizedBlockLoaderScaleZeroPoint {
}
}
- T scale;
- T bias;
+ float scale;
+ float bias;
current_scale_bias(scale, bias);
- for (int i = 0; i < READS_PER_THREAD; i++) {
- dequantize<T, PACK_FACTOR, BITS>(src + i * BYTES_PER_PACK, scale, bias, dst + i * PACK_FACTOR, signed_codes);
- }
+ dequantize<StageT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, scale, bias, dst, signed_codes);
}
void load_safe(short2 src_tile_dim) const {
@@ -147,29 +150,35 @@ struct QuantizedBlockLoaderScaleZeroPoint {
if (tile_row_index >= src_tile_dim.y) {
for (int i = 0; i < READS_PER_THREAD * PACK_FACTOR; i++) {
- dst[i] = T(0);
+ dst[i] = StageT(0);
}
return;
}
const int valid_cols = src_tile_dim.x;
const int valid_packs = (valid_cols + PACK_FACTOR - 1) / PACK_FACTOR;
- T scale;
- T bias;
+ float scale;
+ float bias;
current_scale_bias(scale, bias);
for (int i = 0; i < READS_PER_THREAD; i++) {
const int pack_index = tile_col_index + i;
if (pack_index < valid_packs) {
- dequantize<T, PACK_FACTOR, BITS>(src + i * BYTES_PER_PACK, scale, bias, dst + i * PACK_FACTOR, signed_codes);
+ dequantize<StageT, PACK_FACTOR, BITS>(
+ src + i * BYTES_PER_PACK,
+ scale,
+ bias,
+ dst + i * PACK_FACTOR,
+ signed_codes
+ );
if (pack_index == valid_packs - 1) {
const int remaining = valid_cols - pack_index * PACK_FACTOR;
for (int lane = remaining; lane < PACK_FACTOR; ++lane) {
- dst[i * PACK_FACTOR + lane] = T(0);
+ dst[i * PACK_FACTOR + lane] = StageT(0);
}
}
} else {
for (int lane = 0; lane < PACK_FACTOR; ++lane) {
- dst[i * PACK_FACTOR + lane] = T(0);
+ dst[i * PACK_FACTOR + lane] = StageT(0);
}
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
index 4adb2da49..2fdcd57c4 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
@@ -45,7 +45,8 @@ static METAL_FUNC auto make_staged_loader(
Core::SHARED_STRIDE_B,
Core::THREADGROUP_THREADS,
RightOperand::GROUP_SIZE,
- RightOperand::BITS>;
+ RightOperand::BITS,
+ typename Core::RightStageType>;
return Loader(
values,
scales,
@@ -67,7 +68,8 @@ static METAL_FUNC auto make_staged_loader(
Core::THREADGROUP_THREADS,
RightOperand::GROUP_SIZE,
RightOperand::BITS,
- false>;
+ false,
+ typename Core::RightStageType>;
return Loader(
values,
scales,
@@ -96,7 +98,8 @@ static METAL_FUNC auto make_staged_loader(
Core::THREADGROUP_THREADS,
RightOperand::GROUP_SIZE,
RightOperand::BITS,
- true>;
+ true,
+ typename Core::RightStageType>;
return Loader(
values,
scales,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index 1c5fb6e2e..889c931bc 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -34,6 +34,9 @@ struct SimdgroupMmaCore {
using Right = RightOperand;
using LeftElementType = typename Left::ElementType;
using RightElementType = typename Right::ElementType;
+ // Quantized weights are dequantized in f32 and staged as half, the fast pre-M5 MMA operand.
+ using RightStageType = conditional_t<Right::QUANTIZED, half, RightElementType>;
+ static_assert(sizeof(RightStageType) == sizeof(RightElementType), "staging memory is sized for the weight type");
using LeftStorage = operands::LeftStorage<Left>;
using RightStorage = operands::RightStorage<Right>;
static_assert(!Left::QUANTIZED, "simdgroup MMA stages full-precision left operands only");
@@ -65,7 +68,7 @@ struct SimdgroupMmaCore {
THREADGROUP_THREADS>;
using TileAccumulator = uzu::matmul::ThreadgroupTile<
LeftElementType,
- RightElementType,
+ RightStageType,
OutputElementType,
THREADGROUP_BLOCK_M,
THREADGROUP_BLOCK_N,
@@ -82,7 +85,7 @@ struct SimdgroupMmaCore {
template <uint GEMM_ALIGNMENT_RAW, typename BLoader>
static METAL_FUNC void k_loop(
threadgroup LeftElementType* a_shared,
- threadgroup RightElementType* b_shared,
+ threadgroup RightStageType* b_shared,
const int aligned_k_iterations,
thread ALoader& loader_a,
thread BLoader& loader_b,
@@ -244,7 +247,7 @@ struct SimdgroupMmaCore {
constexpr uint gemm_alignment = gemm_alignment_mask.value;
k_loop<gemm_alignment>(
a_shared,
- b_shared,
+ reinterpret_cast<threadgroup RightStageType*>(b_shared),
params->aligned_inner_iterations,
loader_a,
loader_b,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 5c7f457ff..fca6a18f4 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -54,7 +54,6 @@ VARIANTS(
GEMM_TILING,
GemmTiling::Tile8x32x32_Simdgroups1x1,
GemmTiling::Tile64x32x32_Simdgroups2x2,
- GemmTiling::Tile64x64x16_Simdgroups2x2,
GemmTiling::Tile64x64x32_Simdgroups2x2,
GemmTiling::Tile32x32x32_Simdgroups2x2,
GemmTiling::Tile16x32x256_Simdgroups1x1,
@@ -98,13 +97,9 @@ CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)
CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 4)
CONSTRAINT(
B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
- (TRANSPOSE_B &&
- GEMM_TILING != GemmTiling::Tile64x64x16_Simdgroups2x2 &&
- GEMM_TILING != GemmTiling::Tile64x32x32_Simdgroups2x2))
+ (TRANSPOSE_B && GEMM_TILING != GemmTiling::Tile64x32x32_Simdgroups2x2))
CONSTRAINT(
- B_PROLOGUE != GemmBPrologueKind::FullPrecision || USE_MXU ||
- GEMM_TILING == GemmTiling::Tile64x64x16_Simdgroups2x2 ||
- GEMM_TILING == GemmTiling::Tile64x32x32_Simdgroups2x2)
+ B_PROLOGUE != GemmBPrologueKind::FullPrecision || USE_MXU || GEMM_TILING == GemmTiling::Tile64x32x32_Simdgroups2x2)
CONSTRAINT(
GEMM_TILING != GemmTiling::Tile16x32x256_Simdgroups1x1 ||
(TRANSPOSE_B &&
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index c1da24799..7736a5f23 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -154,13 +154,7 @@ impl GemmKernel {
shape: MatmulShape,
context: &MetalContext,
) -> GemmProblem {
- GemmProblem::new(
- shape,
- self.weights_data_type,
- self.output_data_type,
- context.supports_mxu,
- context.apple_gpu_family,
- )
+ GemmProblem::new(shape, self.weights_data_type, self.output_data_type, context.supports_mxu)
}
#[cfg(test)]
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
index 84bd19da1..7b5fe1b1e 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
@@ -4,8 +4,6 @@
//! features to choices. Eligibility, fallbacks, and legality stay in
//! `selection.rs`.
-use metal::MTLGPUFamily;
-
use crate::backends::metal::gpu_types::gemm::GemmTiling;
const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;
@@ -21,10 +19,9 @@ const WIDE_N_MODERATE_K_RATIO: u32 = 6;
const MXU_M_BUCKET_MAXES: [u32; 4] = [16, 63, 255, 511];
const MXU_N_BUCKET_MAXES: [u32; 2] = [63, 127];
-const SIMDGROUP_QUANT_SMALL_M_MAX: u32 = 32;
-const SIMDGROUP_QUANT_NARROW_BLOCK_M: u32 = 8;
-const SIMDGROUP_QUANT_LARGE_M_MIN: u32 = 64;
-const SIMDGROUP_QUANT_WIDE_N_MIN: u32 = 6144;
+// Two 8-row blocks: past this the 32-row tile has fewer, fuller blocks and wins on every chip.
+const SIMDGROUP_QUANT_NARROW_TILE_M_MAX: u32 = 16;
+const SIMDGROUP_QUANT_WIDE_BLOCK_M: u32 = 64;
const SPLIT_K_TARGET_TILES_FP: u32 = 512;
pub(super) const SPLIT_K_TARGET_TILES_A8: u32 = 256;
@@ -99,42 +96,14 @@ pub(super) fn mxu_fp_tile(
GemmTiling::Tile32x64x256_Simdgroups2x2
}
-pub(super) fn simdgroup_fp_tile(
- m: u32,
- n: u32,
- k: u32,
-) -> GemmTiling {
- if 2_u32.saturating_mul(m.max(n)) > k {
- GemmTiling::Tile64x64x16_Simdgroups2x2
+pub(super) fn simdgroup_quant_tile(m: u32) -> GemmTiling {
+ if m <= SIMDGROUP_QUANT_NARROW_TILE_M_MAX {
+ GemmTiling::Tile8x32x32_Simdgroups1x1
+ } else if (1..=SIMDGROUP_QUANT_WIDE_BLOCK_M / 2).contains(&(m % SIMDGROUP_QUANT_WIDE_BLOCK_M)) {
+ // The last wide block would be at most half full: the narrow tile pads fewer rows.
+ GemmTiling::Tile32x32x32_Simdgroups2x2
} else {
- GemmTiling::Tile64x32x32_Simdgroups2x2
- }
-}
-
-/// A partial trailing M block costs a second pass over the weights. Older GPUs are bound by that;
-/// Apple9 and newer would rather keep the narrow tile's parallelism.
-fn prefers_wide_partial_m_tile(apple_gpu_family: MTLGPUFamily) -> bool {
- apple_gpu_family <= MTLGPUFamily::Apple8
-}
-
-pub(super) fn simdgroup_quant_tile(
- m: u32,
- n: u32,
- apple_gpu_family: MTLGPUFamily,
-) -> GemmTiling {
- if m < SIMDGROUP_QUANT_SMALL_M_MAX {
- if !prefers_wide_partial_m_tile(apple_gpu_family)
- || m <= SIMDGROUP_QUANT_NARROW_BLOCK_M
- || m.is_multiple_of(SIMDGROUP_QUANT_NARROW_BLOCK_M)
- {
- GemmTiling::Tile8x32x32_Simdgroups1x1
- } else {
- GemmTiling::Tile32x32x32_Simdgroups2x2
- }
- } else if m >= SIMDGROUP_QUANT_LARGE_M_MIN && n >= SIMDGROUP_QUANT_WIDE_N_MIN && n.is_multiple_of(64) {
GemmTiling::Tile64x64x32_Simdgroups2x2
- } else {
- GemmTiling::Tile32x32x32_Simdgroups2x2
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index 0c335dc3e..0e0958b1b 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -1,4 +1,3 @@
-use metal::MTLGPUFamily;
use thiserror::Error;
use crate::{
@@ -29,7 +28,6 @@ pub struct GemmProblem {
weights_data_type: DataType,
output_data_type: DataType,
supports_mxu: bool,
- apple_gpu_family: MTLGPUFamily,
}
#[derive(Debug, Clone, Copy, PartialEq, Error)]
@@ -80,14 +78,12 @@ impl GemmProblem {
weights_data_type: DataType,
output_data_type: DataType,
supports_mxu: bool,
- apple_gpu_family: MTLGPUFamily,
) -> Self {
Self {
shape,
weights_data_type,
output_data_type,
supports_mxu,
- apple_gpu_family,
}
}
@@ -100,7 +96,7 @@ impl GemmProblem {
} else {
GemmEngine::Simdgroup
};
- self.finish_plan(engine, select_tiling(self.shape, engine, self.apple_gpu_family))
+ self.finish_plan(engine, select_tiling(self.shape, engine))
}
pub fn small_batch_plan(
@@ -131,7 +127,7 @@ impl GemmProblem {
return Ok(plan);
}
self.validate_engine(engine)?;
- Ok(self.finish_plan(engine, select_tiling(self.shape, engine, self.apple_gpu_family)))
+ Ok(self.finish_plan(engine, select_tiling(self.shape, engine)))
}
pub(super) fn validate_engine(
@@ -192,8 +188,6 @@ impl GemmProblem {
}
fn split_k_output_supported(&self) -> bool {
- use crate::backends::common::gpu_types::gemm::GemmDTransform;
-
let mut output_transform = self.shape.d_transform;
if self.shape.is_quant()
&& output_transform.contains(GemmDTransform::RHT)
@@ -206,7 +200,7 @@ impl GemmProblem {
}
}
-pub(super) fn select_trellis_plan(
+fn select_trellis_plan(
m: u32,
n: u32,
k: u32,
@@ -266,11 +260,10 @@ fn mxu_is_eligible(shape: MatmulShape) -> bool {
fn select_tiling(
shape: MatmulShape,
engine: GemmEngine,
- apple_gpu_family: MTLGPUFamily,
) -> GemmTiling {
match engine {
- GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m, shape.n, apple_gpu_family),
- GemmEngine::Simdgroup => policy::simdgroup_fp_tile(shape.m, shape.n, shape.k),
+ GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m),
+ GemmEngine::Simdgroup => GemmTiling::Tile64x32x32_Simdgroups2x2,
GemmEngine::Mxu if shape.a_is_quantized() => match policy::mxu_int8_gemm_tile(shape.m, shape.n) {
tiling
if tiling.simdgroups_k() > 1
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
index 0f8cb2afe..408e9a328 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
@@ -1,5 +1,3 @@
-use metal::MTLGPUFamily;
-
use crate::{
backends::{
common::{
@@ -44,7 +42,7 @@ pub fn supports_integer_right_operand(shape: &MatmulShape) -> bool {
}
impl MatmulMetalKernel {
- pub fn prefer_gemm_over_gemv(
+ fn prefer_gemm_over_gemv(
shape: MatmulShape,
plan: GemmPlan,
weights_data_type: DataType,
@@ -84,13 +82,12 @@ impl MatmulMetalKernel {
shape: &MatmulShape,
gemm_b_layout: GemmBLayout,
gpu_core_count: u32,
- apple_gpu_family: MTLGPUFamily,
supports_mxu: bool,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
) -> MatmulDispatch {
- let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);
+ let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu);
if shape.b_is_trellis {
return MatmulDispatch::Gemm(problem.select_plan());
}
@@ -138,7 +135,6 @@ impl MatmulMetalKernel {
shape,
gemm_b_layout,
context.gpu_core_count,
- context.apple_gpu_family,
context.supports_mxu,
self.weights_data_type,
self.input_data_type,
diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/device_extensions.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/device_extensions.rs
index 931420981..1c3a4d987 100644
--- a/crates/uzu-engine/src/backends/metal/metal_extensions/device_extensions.rs
+++ b/crates/uzu-engine/src/backends/metal/metal_extensions/device_extensions.rs
@@ -1,7 +1,7 @@
use core::mem::transmute;
use std::ffi::CString;
-use metal::{MTLDevice, MTLGPUFamily};
+use metal::MTLDevice;
use obfstr::obfstr;
use objc2::{
Message,
@@ -9,8 +9,6 @@ use objc2::{
runtime::{NSObjectProtocol, ProtocolObject, Sel},
};
-use super::GpuFamilyExt;
-
// Used to bypass objc2's debug-mode class validation for proxy objects like
// `CaptureMTLDevice` which forward messages at the ObjC runtime level (via
// `forwardingTargetForSelector:`) but don't have the methods in their own class
@@ -69,15 +67,6 @@ pub trait DeviceExt: MTLDevice + Message + NSObjectProtocol + Sized {
fn supports_placement_sparse_resources(&self) -> bool {
optional_selector_value(self, obfstr!("supportsPlacementSparse"), false)
}
-
- /// Newest Apple GPU family the device supports.
- fn newest_supported_apple_gpu_family(&self) -> MTLGPUFamily {
- MTLGPUFamily::all_cases()
- .into_iter()
- .rev()
- .find(|family| self.supports_family(*family))
- .expect("unsupported Metal device: uzu requires an Apple GPU")
- }
}
impl DeviceExt for ProtocolObject<dyn MTLDevice> {}
diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/gpu_family_extensions.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/gpu_family_extensions.rs
deleted file mode 100644
index a376ad86c..000000000
--- a/crates/uzu-engine/src/backends/metal/metal_extensions/gpu_family_extensions.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-use metal::MTLGPUFamily;
-
-pub trait GpuFamilyExt: Sized {
- /// All Apple GPU family cases, oldest to newest.
- fn all_cases() -> [Self; 10];
-}
-
-impl GpuFamilyExt for MTLGPUFamily {
- fn all_cases() -> [Self; 10] {
- [
- Self::Apple1,
- Self::Apple2,
- Self::Apple3,
- Self::Apple4,
- Self::Apple5,
- Self::Apple6,
- Self::Apple7,
- Self::Apple8,
- Self::Apple9,
- Self::Apple10,
- ]
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rs
index 88e550683..5b9cd8012 100644
--- a/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rs
@@ -1,13 +1,11 @@
mod data_type;
mod device_extensions;
mod function_constant_values_extensions_set_value;
-mod gpu_family_extensions;
mod library_extensions_pipeline;
mod sparse_page_size_extensions;
pub use data_type::MetalDataTypeExt;
pub use device_extensions::DeviceExt;
pub use function_constant_values_extensions_set_value::FunctionConstantValuesSetValue;
-pub use gpu_family_extensions::GpuFamilyExt;
pub use library_extensions_pipeline::LibraryPipelineExtensions;
pub use sparse_page_size_extensions::SparsePageSizeExt;
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index 913832e7e..19d9126b1 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -1,15 +1,10 @@
use futures::executor::block_on;
-use metal::MTLGPUFamily;
use uzu_engine_macros::uzu_test;
use super::*;
use crate::{
backends::{
- common::{
- Context,
- gpu_types::gemm::GemmDTransform,
- kernel::matmul::{MatmulKernel, QuantParamsLayout},
- },
+ common::{Context, kernel::matmul::MatmulKernel},
metal::{context::MetalContext, kernel::matmul::MatmulMetalKernel},
},
tests::matmul::{a8_format, matmul_shape as shape, quantized_matmul_shape},
@@ -27,73 +22,12 @@ fn problem(
shape: MatmulShape,
output_data_type: DataType,
) -> GemmProblem {
- GemmProblem::new(shape, DataType::BF16, output_data_type, true, MTLGPUFamily::Apple7)
+ GemmProblem::new(shape, DataType::BF16, output_data_type, true)
}
#[uzu_test]
-fn policy_boundaries_are_preserved() {
- use GemmTiling::*;
-
- for (is_a_int8, m, n, expected) in [
- (true, 16, 4096, Tile16x32x256_Simdgroups1x1),
- (true, 17, 4096, Tile32x64x256_Simdgroups2x2),
- (true, 64, 63, Tile64x32x256_Simdgroups4x1),
- (true, 512, 4096, Tile128x128x256_Simdgroups4x4),
- (false, 16, 4096, Tile32x64x256_Simdgroups2x2),
- (false, 64, 4096, Tile64x64x256_Simdgroups2x2),
- (false, 256, 4096, Tile128x128x256_Simdgroups4x4),
- ] {
- assert_eq!(policy::mxu_mn_tile(is_a_int8, m, n), expected);
- }
-
- for (m, n, k, expected) in [
- (15, 2560, 2560, Tile16x32x256_Simdgroups1x1),
- (15, 4096, 8192, Tile16x128x256_Simdgroups1x4),
- (15, 131_073, 4096, Tile16x32x256_Simdgroups1x1),
- (64, 4096, 2048, Tile64x64x256_Simdgroups2x2),
- (256, 4096, 2048, Tile128x128x256_Simdgroups4x4),
- ] {
- assert_eq!(policy::mxu_fp_tile(m, n, k), expected);
- }
-
- for (m, n, family, expected) in [
- (16, 4096, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),
- (8, 4096, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),
- (9, 4096, MTLGPUFamily::Apple8, Tile32x32x32_Simdgroups2x2),
- (9, 4096, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),
- (31, 4096, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),
- (32, 4096, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),
- (64, 6143, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),
- (64, 6144, MTLGPUFamily::Apple9, Tile64x64x32_Simdgroups2x2),
- ] {
- assert_eq!(policy::simdgroup_quant_tile(m, n, family), expected);
- }
-}
-
-#[uzu_test]
-fn selection_fallbacks_and_split_k_are_preserved() {
+fn illegal_layouts_fall_back() {
use GemmEngine::*;
- use GemmTiling::*;
-
- for (mut case, expect_split) in [(shape(4, 32, 1024), true), (shape(64, 128, 128), false)] {
- case.d_transform = GemmDTransform::RHT | GemmDTransform::BIAS;
- for engine in [Simdgroup, Mxu] {
- let plan = problem(case, DataType::BF16)
- .select_plan_for_engine(engine)
- .expect("parity anchor supports the forced GEMM engine");
- assert_eq!(plan.split_k > 1, expect_split, "forced {engine:?} RHT+bias parity anchor");
- }
- }
-
- let mut non_split_quant_rht_bias = shape(9, 64, 64);
- non_split_quant_rht_bias.b_prologue = GemmBPrologueKind::ScaleZeroPointDequant;
- non_split_quant_rht_bias.b_bits = Some(4);
- non_split_quant_rht_bias.b_group_size = Some(64);
- non_split_quant_rht_bias.params_layout = Some(QuantParamsLayout::GroupOutput);
- non_split_quant_rht_bias.d_transform = GemmDTransform::RHT | GemmDTransform::BIAS;
- let plan = problem(non_split_quant_rht_bias, DataType::BF16).select_plan();
- assert_eq!(plan.engine, Simdgroup);
- assert_eq!(plan.split_k, 1);
let mut invalid_layout = quant(shape(64, 4096, 4096));
invalid_layout.b_transpose = false;
@@ -101,64 +35,13 @@ fn selection_fallbacks_and_split_k_are_preserved() {
invalid_layout.a = a8_format();
assert_eq!(problem(invalid_layout, DataType::BF16).select_plan().engine, Mxu);
assert_eq!(problem(quant(shape(64, 4096, 4095)), DataType::BF16).select_plan().engine, Simdgroup);
-
- for (m, n, k, expected_tiling, expected_split_k) in [
- (16, 4096, 4096, Tile16x32x256_Simdgroups1x1x4, 2),
- (16, 1024, 4096, Tile16x32x256_Simdgroups1x1x4, 8),
- (16, 34816, 4096, Tile16x32x256_Simdgroups1x1x4, 1),
- (8, 4096, 384, Tile16x32x256_Simdgroups1x1, 1),
- ] {
- let mut a8 = quant(shape(m, n, k));
- a8.a = a8_format();
- let plan = problem(a8, DataType::BF16).select_plan();
- assert_eq!(plan.tiling, expected_tiling);
- assert_eq!(plan.split_k, expected_split_k);
- }
-
- let mut biased = quant(shape(16, 4096, 4096));
- biased.a = a8_format();
- biased.d_transform = GemmDTransform::BIAS;
- assert_eq!(
- GemmProblem::new(biased, DataType::BF16, DataType::F32, true, MTLGPUFamily::Apple7).select_plan().split_k,
- 1
- );
-
- let mut zero = quant(shape(0, 1, 1));
- zero.b_prologue = GemmBPrologueKind::ScaleZeroPointDequant;
- zero.b_group_size = Some(u32::MAX);
- assert_eq!(problem(zero, DataType::BF16).select_plan().split_k, 1);
-}
-
-#[uzu_test]
-fn trellis_plan_matches_projection_cases() {
- use GemmTiling::*;
-
- for (m, n, k, tiling, split_k) in [
- (16, 128, 64, Tile16x32x256_Simdgroups1x1, 1),
- (16, 4096, 2048, Tile32x64x256_Simdgroups2x2, 4),
- (16, 5120, 5120, Tile32x64x256_Simdgroups2x2, 2),
- (16, 5120, 5184, Tile16x32x256_Simdgroups1x1, 3),
- (16, 4096, 1984, Tile16x32x256_Simdgroups1x1, 1),
- (16, 32, 5120, Tile64x64x256_Simdgroups2x2, 80),
- (17, 12_288, 2048, Tile64x64x256_Simdgroups2x2, 2),
- (17, 80, 64, Tile64x64x256_Simdgroups2x2, 1),
- (2048, 2048, 128, Tile128x128x256_Simdgroups4x4, 1),
- (1, 80, 5120, Tile16x32x256_Simdgroups1x1, 80),
- ] {
- let mut trellis_shape = shape(m, n, k);
- trellis_shape.a = ActivationFormat::Trellis;
- trellis_shape.b_is_trellis = true;
- let plan = problem(trellis_shape, DataType::BF16).select_plan();
- assert_eq!((plan.engine, plan.tiling, plan.split_k), (GemmEngine::Mxu, tiling, split_k), "M {m} N {n} K {k}");
- }
}
#[uzu_test]
fn forced_engine_errors_are_preserved() {
let huge = shape(u32::MAX, u32::MAX, u32::MAX);
assert_eq!(
- GemmProblem::new(huge, DataType::BF16, DataType::BF16, false, MTLGPUFamily::Apple7)
- .select_plan_for_engine(GemmEngine::Mxu),
+ GemmProblem::new(huge, DataType::BF16, DataType::BF16, false).select_plan_for_engine(GemmEngine::Mxu),
Err(GemmPlanError::MxuUnavailable)
);
@@ -170,24 +53,6 @@ fn forced_engine_errors_are_preserved() {
);
}
-#[uzu_test]
-fn gemv_gemm_route_boundaries_are_preserved() {
- for (shape, data_type, prefer_gemm) in [
- (shape(4, 4096, 8192), DataType::BF16, true),
- (shape(4, 8192, 4096), DataType::BF16, false),
- (shape(4, 4096, 8192), DataType::F32, false),
- (shape(3, 4096, 8192), DataType::BF16, false),
- (shape(5, 4096, 8192), DataType::BF16, false),
- ] {
- let plan = GemmProblem::new(shape, data_type, data_type, true, MTLGPUFamily::Apple7).select_plan();
- assert_eq!(MatmulMetalKernel::prefer_gemm_over_gemv(shape, plan, data_type, data_type, data_type), prefer_gemm);
- }
- let mut gathered = shape(4, 4096, 8192);
- gathered.gathered = true;
- let plan = problem(gathered, DataType::BF16).select_plan();
- assert!(!MatmulMetalKernel::prefer_gemm_over_gemv(gathered, plan, DataType::BF16, DataType::BF16, DataType::BF16,));
-}
-
#[uzu_test]
fn int8_activations_require_mxu_tiled_weights() {
let context = block_on(MetalContext::new()).expect("Metal context");
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
index da07743a8..36d109f78 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
@@ -1,112 +1,16 @@
use uzu_engine_macros::uzu_test;
-use super::{DEFAULT_RESULTS_PER_SIMDGROUP, GemvTile, fp_tile, quantized_tile, tile};
+use super::GemvTile;
use crate::{
- backends::{
- common::kernel::matmul::{MatmulShape, QuantParamsLayout},
- metal::gpu_types::gemm::GemmBLayout,
- },
- data_type::DataType,
+ backends::{common::kernel::matmul::MatmulShape, metal::gpu_types::gemm::GemmBLayout},
tests::matmul::quantized_matmul_shape as shape,
};
-const LARGE_GPU_CORE_COUNT: u32 = 30;
-
-fn qtile(
- num_simdgroups: u32,
- results_per_simdgroup: u32,
-) -> GemvTile {
- GemvTile::quantized(num_simdgroups, results_per_simdgroup, 1, 32, 2)
-}
-
-#[uzu_test]
-fn fp_policy_cases() {
- let cases = [
- (1, 12288, 1536, true, tile(8, 8, 1)),
- (1, 12288, 1536, false, tile(8, 1, 1)),
- (1, 1536, 256, true, tile(8, 2, 1)),
- (8, 12288, 1536, true, tile(8, 1, 4)),
- (1, 1536, 12288, true, tile(8, 8, 1)),
- (1, 262144, 1536, true, tile(8, 8, 1)),
- ];
-
- for (m, n, k, aligned, expected) in cases {
- assert_eq!(fp_tile(m, n, k, aligned), expected, "m={m} n={n} k={k} aligned={aligned}");
- }
-}
-
-#[uzu_test]
-fn quant_policy_cases() {
- let cases = [
- (1, 256, 4, qtile(8, 4)),
- (1, 262144, 4, qtile(8, 4)),
- (2, 2048, 4, qtile(8, 4)),
- (4, 5120, 4, qtile(8, 4)),
- (1, 2048, 8, GemvTile::quantized(8, 4, 1, 32, 4)),
- ];
-
- for (m, n, bits, expected) in cases {
- let actual = quantized_tile(bits, 32, m, n, true).expect("quantized route");
- assert_eq!(actual, expected, "m={m} n={n} bits={bits}");
- }
-}
-
-#[uzu_test]
-fn quantized_policy_edges() {
- assert_eq!(quantized_tile(4, 64, 4, 8192, true), Some(GemvTile::quantized(8, 4, 1, 32, 4)));
- assert!(quantized_tile(4, 32, 4, 8, true).is_some());
- assert!(quantized_tile(4, 32, 1, 8192, true).is_some());
- assert_eq!(quantized_tile(4, 32, 9, 8192, true), None);
- assert_eq!(quantized_tile(4, 32, 6, 8192, true), None);
- assert_eq!(quantized_tile(4, 32, 6, 64, true), Some(GemvTile::quantized(8, 4, 1, 32, 2)));
- assert_eq!(quantized_tile(4, 32, 1, 8192, false), None);
- assert_eq!(quantized_tile(4, 32, 4, 8192, true), Some(GemvTile::quantized(8, 4, 1, 32, 2)));
-}
-
-#[uzu_test]
-fn block_unaligned_quantized_k_stays_on_gemv() {
- let shape = MatmulShape {
- params_layout: Some(QuantParamsLayout::OutputGroup),
- ..shape(1, 64, 1152, 4, 64)
- };
- assert!(
- super::super::kernel::GemvSpecialization::select_shape(
- &shape,
- GemmBLayout::RowMajor,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- LARGE_GPU_CORE_COUNT,
- )
- .is_some()
- );
-}
-
-#[uzu_test]
-fn specialization_preserves_quantized_route() {
- let select = |n| {
- super::super::kernel::GemvSpecialization::select_shape(
- &MatmulShape {
- params_layout: Some(QuantParamsLayout::OutputGroup),
- ..shape(4, n, 4096, 4, 32)
- },
- GemmBLayout::RowMajor,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- LARGE_GPU_CORE_COUNT,
- )
- };
- let clean = select(8192).expect("quantized specialization");
- assert!(clean.output_row_tile() > DEFAULT_RESULTS_PER_SIMDGROUP);
- assert!(select(8192 + clean.output_row_tile() / 2).is_some());
-}
-
#[uzu_test]
fn gathered_group_major_is_a_gemv_route() {
- let mut shape = shape(1, 8192, 4096, 4, 32);
+ // Gathered readout has no GEMM implementation, so group-major gathered rows must stay on GEMV.
+ let mut shape: MatmulShape = shape(1, 8192, 4096, 4, 32);
shape.gathered = true;
- assert!(
- super::super::kernel::GemvSpecialization::select_tile(&shape, GemmBLayout::RowMajor, qtile(8, 4),).is_some()
- );
+ let tile = GemvTile::quantized(8, 4, 1, 32, 2);
+ assert!(super::super::kernel::GemvSpecialization::select_tile(&shape, GemmBLayout::RowMajor, tile).is_some());
}