Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Introduce Metal device tiers and tier-tuned GEMV/QMV dispatch (#529)

CCYeh6b52bd69dd7a
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index 40d47c2d5..210113317 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -17,6 +17,7 @@ use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{
Metal,
+ device_tier::{DeviceTier, device_tier_for_device},
error::MetalError,
kernel,
metal_extensions::{DeviceExt, LibraryPipelineExtensions},
@@ -44,6 +45,7 @@ pub struct MetalContext {
library: Retained<ProtocolObject<dyn MTLLibrary>>,
pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,
sparse_heap_pool: RefCell<MetalSparseHeapPool>,
+ device_tier: DeviceTier,
weak_self: Weak<MetalContext>,
#[cfg(test)]
timeline_shared_event: Retained<ProtocolObject<dyn MTLSharedEvent>>,
@@ -54,6 +56,10 @@ impl MetalContext {
self.device.supports_mxu()
}
+ pub(crate) fn device_tier(&self) -> DeviceTier {
+ self.device_tier
+ }
+
pub(super) fn update_peak_memory_usage(&self) {
let mut peak_memory_usage_borrow = self.peak_memory_usage.borrow_mut();
*peak_memory_usage_borrow = peak_memory_usage_borrow.max(self.device.current_allocated_size());
@@ -129,6 +135,7 @@ impl Context for MetalContext {
.map_err(|nserror| MetalError::CannotCreateLibrary(nserror.to_string()))?;
let gpu_core_count = device.gpu_core_count();
+ let device_tier = device_tier_for_device(gpu_core_count, device.as_ref());
let page_size = MTLSparsePageSize::KB256;
let heap_capacity = Metal::ALLOCATION_GRANULARITY;
@@ -149,6 +156,7 @@ impl Context for MetalContext {
library,
pipeline_cache: RefCell::new(HashMap::new()),
sparse_heap_pool: RefCell::new(sparse_pool),
+ device_tier,
weak_self: weak_self.clone(),
#[cfg(test)]
timeline_shared_event,
diff --git a/crates/backend-uzu/src/backends/metal/device_tier.rs b/crates/backend-uzu/src/backends/metal/device_tier.rs
new file mode 100644
index 000000000..09e65cf86
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/device_tier.rs
@@ -0,0 +1,45 @@
+use metal::MTLDevice;
+use objc2::runtime::ProtocolObject;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum DeviceTier {
+ /// Small Apple9+ GPUs (M3/A17 and newer, below Max/Ultra class).
+ SmallApple9,
+ /// Small Apple8 GPUs (M2/A15/A16 generation).
+ SmallApple8,
+ /// Small pre-Apple8 GPUs (M1/A14 generation).
+ SmallLegacy,
+ /// Max/Ultra-class GPUs, currently classified by >= 30 GPU cores.
+ Large,
+}
+
+pub(crate) fn device_tier_for_device(
+ gpu_core_count: u32,
+ device: &ProtocolObject<dyn MTLDevice>,
+) -> DeviceTier {
+ device_tier_for(
+ gpu_core_count,
+ device.supports_family(metal::MTLGPUFamily::Apple8),
+ device.supports_family(metal::MTLGPUFamily::Apple9),
+ )
+}
+
+fn device_tier_for(
+ gpu_core_count: u32,
+ supports_apple8_family: bool,
+ supports_apple9_family: bool,
+) -> DeviceTier {
+ if gpu_core_count >= 30 {
+ DeviceTier::Large
+ } else if supports_apple9_family {
+ DeviceTier::SmallApple9
+ } else if supports_apple8_family {
+ DeviceTier::SmallApple8
+ } else {
+ DeviceTier::SmallLegacy
+ }
+}
+
+#[cfg(test)]
+#[path = "../../../tests/unit/backends/metal/device_tier_test.rs"]
+mod tests;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 052fe7790..a51f9a4fb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -33,13 +33,21 @@ VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)
VARIANTS(BITS, 0, 4, 8)
VARIANTS(K_SPLIT, 1, 2, 4, 8)
VARIANTS(INPUT_ALIGNED, false, true)
-VARIANTS(RESULTS_PER_SIMDGROUP, 4)
-VARIANTS(NUM_SIMDGROUPS, 2, 8)
+VARIANTS(RESULTS_PER_SIMDGROUP, 1, 2, 4, 8)
+VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)
CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))
CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")
CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || K_SPLIT == 1)
-CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::FullPrecision || NUM_SIMDGROUPS == 8)
+CONSTRAINT(K_SPLIT <= NUM_SIMDGROUPS)
+// Only selector-reachable tiles are instantiated (fleet-tuned tables): fp
+// always runs 8 simdgroups with 1 or 4 rows each; non-default quantized
+// tiles exist for bf16 IO only. Widen locally when sweeping new configs.
+CONSTRAINT(BITS != 0 || NUM_SIMDGROUPS == 8)
+CONSTRAINT(BITS != 0 || RESULTS_PER_SIMDGROUP == 1 || RESULTS_PER_SIMDGROUP == 4)
+CONSTRAINT(
+ BITS == 0 || (NUM_SIMDGROUPS == 8 && RESULTS_PER_SIMDGROUP == 4) ||
+ (AT == "bfloat" && DT == "bfloat"))
KERNEL(Gemv)(
const device uint32_t* b,
const device BT* scales
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 2662d767d..871e7915e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -3,6 +3,7 @@ use std::{
sync::OnceLock,
};
+use super::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK};
use crate::{
backends::{
common::{
@@ -10,14 +11,11 @@ use crate::{
gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
kernel::matmul::{MatmulArguments, MatmulB, MatmulError},
},
- metal::{Metal, context::MetalContext, kernel::GemvMetalKernel},
+ metal::{Metal, context::MetalContext, device_tier::DeviceTier, kernel::GemvMetalKernel},
},
data_type::DataType,
};
-const FP_BLOCK: u32 = 128;
-const QMV_RESULTS_PER_SIMDGROUP: u32 = 4;
-
const DEFAULT_GEMV_MAX_BATCH: u32 = 8;
static GEMV_MAX_BATCH: OnceLock<u32> = OnceLock::new();
@@ -35,6 +33,7 @@ pub(crate) struct GemvSpecialization {
output_transform: GemmDTransform,
input_aligned: bool,
k_split: u32,
+ results_per_simdgroup: u32,
num_simdgroups: u32,
}
@@ -44,6 +43,7 @@ impl GemvSpecialization {
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
+ device_tier: DeviceTier,
) -> Option<GemvSpecialization> {
if !args.b_transpose || args.b_offset != 0 {
return None;
@@ -64,20 +64,20 @@ impl GemvSpecialization {
return None;
}
if is_quant {
- if args.n < QMV_RESULTS_PER_SIMDGROUP || args.m >= 5 {
+ if args.n < DEFAULT_RESULTS_PER_SIMDGROUP || args.m >= 5 {
return None;
}
} else {
let mixed_precision = weights_data_type == DataType::F32
&& (input_data_type != DataType::F32 || output_data_type != DataType::F32);
- if mixed_precision || args.n < QMV_RESULTS_PER_SIMDGROUP || args.m > max_gemv_batch_threshold() {
+ if mixed_precision || args.n < DEFAULT_RESULTS_PER_SIMDGROUP || args.m > max_gemv_batch_threshold() {
return None;
}
}
let bits = args.b.bits_per_b().unwrap_or(0);
let block_size = if !is_quant {
- FP_BLOCK
+ FP_K_BLOCK
} else if bits == 4 {
512
} else {
@@ -85,45 +85,35 @@ impl GemvSpecialization {
};
let input_aligned = args.k.is_multiple_of(block_size);
let has_rht = args.d_transform.rht_factors.is_some();
- let num_simdgroups = 8;
- let k_split = if is_quant || has_rht {
- 1
+ let bf16_io = input_data_type == DataType::BF16 && output_data_type == DataType::BF16;
+ let tile = if is_quant && bf16_io {
+ policy::quant_tile(args.m, args.n, args.k, bits, has_rht, device_tier)
+ } else if is_quant || has_rht {
+ // Non-bf16 quant IO and fp+RHT keep the default tile (the only
+ // one instantiated for those modes).
+ policy::DEFAULT_TILE
} else {
- fp_k_split(args.n, args.k, input_aligned)
+ policy::fp_tile(args.m, args.n, args.k, input_aligned, device_tier)
};
- Some(GemvSpecialization {
+ Some(Self {
b_prologue: args.b.b_prologue(),
group_size: args.b.group_size().unwrap_or(0),
bits,
output_transform: args.d_transform.mask(),
input_aligned,
- k_split,
- num_simdgroups,
+ k_split: tile.k_split,
+ results_per_simdgroup: tile.results_per_simdgroup,
+ num_simdgroups: tile.num_simdgroups,
})
}
}
fn rows_per_threadgroup(
k_split: u32,
+ results_per_simdgroup: u32,
num_simdgroups: u32,
) -> u32 {
- (num_simdgroups / k_split) * QMV_RESULTS_PER_SIMDGROUP
-}
-
-fn fp_k_split(
- n: u32,
- k: u32,
- input_aligned: bool,
-) -> u32 {
- if !input_aligned || n >= 4096 {
- 1
- } else if k >= 16 * n || n <= 512 {
- 8
- } else if n <= 1024 || k >= 3072 {
- 4
- } else {
- 2
- }
+ (num_simdgroups / k_split) * results_per_simdgroup
}
pub(crate) struct GemvDispatch {
@@ -135,17 +125,16 @@ pub(crate) struct GemvDispatch {
impl GemvDispatch {
pub(crate) fn new(
- _context: &MetalContext,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
- ) -> Result<Self, MatmulError<Metal>> {
- Ok(Self {
+ ) -> Self {
+ Self {
weights_data_type,
input_data_type,
output_data_type,
pipelines: HashMap::new(),
- })
+ }
}
fn get_or_create(
@@ -166,7 +155,7 @@ impl GemvDispatch {
specialization.bits,
specialization.k_split,
specialization.input_aligned,
- QMV_RESULTS_PER_SIMDGROUP,
+ specialization.results_per_simdgroup,
specialization.num_simdgroups,
specialization.output_transform,
)
@@ -197,7 +186,11 @@ impl GemvDispatch {
..
} = arguments;
- let group_count_x = n.div_ceil(rows_per_threadgroup(specialization.k_split, specialization.num_simdgroups));
+ let group_count_x = n.div_ceil(rows_per_threadgroup(
+ specialization.k_split,
+ specialization.results_per_simdgroup,
+ specialization.num_simdgroups,
+ ));
let context = encoder.context();
let pipeline = self.get_or_create(context, specialization)?;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
index a14cbb82f..7f9bc76ef 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
@@ -1,3 +1,4 @@
mod kernel;
+mod policy;
pub(crate) use kernel::{GemvDispatch, GemvSpecialization};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/policy.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/policy.rs
new file mode 100644
index 000000000..12573b6ca
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/policy.rs
@@ -0,0 +1,223 @@
+use crate::backends::metal::device_tier::DeviceTier;
+
+// Full-precision GEMV accumulates four K values per SIMD lane, so one full
+// vectorized K block is 4 * 32 lanes.
+pub(crate) const FP_K_BLOCK: u32 = 128;
+pub(crate) const DEFAULT_RESULTS_PER_SIMDGROUP: u32 = 4;
+pub(crate) const DEFAULT_NUM_SIMDGROUPS: u32 = 8;
+
+/// Tile specialization selected for one GEMV dispatch.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct GemvTile {
+ /// SIMD groups launched in one threadgroup.
+ pub num_simdgroups: u32,
+ /// Number of split-K slices reduced by one threadgroup.
+ pub k_split: u32,
+ /// Output rows computed by each SIMD group.
+ pub results_per_simdgroup: u32,
+}
+
+const SMALL_G13_HUGE_N: u32 = 32768;
+const SMALL_G13_WIDE_ROW_N: u32 = 6144;
+const DEEP_K: u32 = 8192;
+const FP_LARGE_SPLIT_K_MIN_DEPTH: u32 = 4 * FP_K_BLOCK;
+const FP_K_DEPTH_N_MAX: u32 = 4095;
+const FP_K_DEPTH_DEEP_MIN: u32 = 3072;
+const FP_K_DEPTH_VERY_DEEP_RATIO: u32 = 16;
+
+const fn tile(
+ num_simdgroups: u32,
+ k_split: u32,
+ results_per_simdgroup: u32,
+) -> GemvTile {
+ GemvTile {
+ num_simdgroups,
+ k_split,
+ results_per_simdgroup,
+ }
+}
+
+const fn qtile(
+ num_simdgroups: u32,
+ results_per_simdgroup: u32,
+) -> GemvTile {
+ // Q4 table sweeps only covered KS1; quant split-K is a separate future grid.
+ tile(num_simdgroups, 1, results_per_simdgroup)
+}
+
+pub(crate) const DEFAULT_TILE: GemvTile = qtile(DEFAULT_NUM_SIMDGROUPS, DEFAULT_RESULTS_PER_SIMDGROUP);
+// Qxy = qtile(num_simdgroups=x, results_per_simdgroup=y), with KS1.
+const Q21: GemvTile = qtile(2, 1);
+const Q22: GemvTile = qtile(2, 2);
+const Q24: GemvTile = qtile(2, 4);
+const Q42: GemvTile = qtile(4, 2);
+const Q44: GemvTile = qtile(4, 4);
+const Q48: GemvTile = qtile(4, 8);
+const Q82: GemvTile = qtile(8, 2);
+const QUANT_N_BUCKET_MAXES: [u32; 6] = [512, 2048, 4096, 8192, 16384, 32768];
+const QUANT_K_BUCKET_MAXES: [u32; 3] = [512, 2048, 8192];
+const QUANT_RHT_TUNED_N_MIN_EXCLUSIVE: u32 = 2048;
+const QUANT_RHT_TUNED_N_MAX: u32 = 4096;
+const QUANT_RHT_TUNED_K_MIN: u32 = 2048;
+
+fn table_bucket_index(
+ value: u32,
+ bucket_maxes: &[u32],
+) -> usize {
+ bucket_maxes.partition_point(|&max| value > max)
+}
+
+fn cap_k_split_to_complete_fp_k_blocks(
+ k: u32,
+ preferred: u32,
+) -> u32 {
+ // K_SPLIT variants are powers of two. Do not split beyond the number of
+ // complete vectorized K blocks each slice can own.
+ let complete_blocks = k / FP_K_BLOCK;
+ if complete_blocks == 0 {
+ return 1;
+ }
+ preferred.min((1 << complete_blocks.ilog2()).min(DEFAULT_NUM_SIMDGROUPS))
+}
+
+fn preferred_fp_k_split(
+ m: u32,
+ n: u32,
+ k: u32,
+) -> u32 {
+ if m <= 2 {
+ return 8;
+ }
+ if m <= 4 {
+ return if n <= 16384 {
+ 8
+ } else {
+ 1
+ };
+ }
+ if n <= 512 {
+ return 8;
+ }
+ if n <= 1024 {
+ return if n != 0 && k / n >= FP_K_DEPTH_VERY_DEEP_RATIO {
+ 8
+ } else {
+ 4
+ };
+ }
+ if n <= FP_K_DEPTH_N_MAX {
+ return if n != 0 && k / n >= FP_K_DEPTH_VERY_DEEP_RATIO {
+ 8
+ } else if k >= FP_K_DEPTH_DEEP_MIN {
+ 4
+ } else {
+ 2
+ };
+ }
+ 1
+}
+
+/// Selects the full-precision GEMV tile. `m` is the input-vector count,
+/// `n` is the output row count, and `k` is the reduction depth.
+pub(crate) fn fp_tile(
+ m: u32,
+ n: u32,
+ k: u32,
+ input_aligned: bool,
+ tier: DeviceTier,
+) -> GemvTile {
+ // FP sweeps covered SG2/SG4/SG8; SG changes did not produce portable
+ // confirmed wins, so shipped FP policy keeps SG8 and tunes KS/R only.
+ let should_disable_k_split = !input_aligned
+ || (m == 1 && tier == DeviceTier::Large && k < FP_LARGE_SPLIT_K_MIN_DEPTH)
+ || (m == 1 && tier == DeviceTier::SmallLegacy && n >= SMALL_G13_HUGE_N);
+
+ let k_split = if should_disable_k_split {
+ 1
+ } else {
+ cap_k_split_to_complete_fp_k_blocks(k, preferred_fp_k_split(m, n, k))
+ };
+
+ // R1 won most single-row FP sweeps; Large devices only switch back to R4
+ // for deep-K rows, while legacy wide rows keep R4.
+ let results_per_simdgroup = if tier == DeviceTier::SmallLegacy && m == 1 && n >= SMALL_G13_WIDE_ROW_N {
+ DEFAULT_RESULTS_PER_SIMDGROUP
+ } else if m == 1 && (k <= DEEP_K || tier != DeviceTier::Large) {
+ 1
+ } else {
+ DEFAULT_RESULTS_PER_SIMDGROUP
+ };
+
+ tile(DEFAULT_NUM_SIMDGROUPS, k_split, results_per_simdgroup)
+}
+
+/// Selects the quantized GEMV tile. `m` is the input-vector count, `n` is the
+/// output row count, `k` is the reduction depth, and `bits` is the quant width.
+pub(crate) fn quant_tile(
+ m: u32,
+ n: u32,
+ k: u32,
+ bits: u32,
+ has_rht: bool,
+ tier: DeviceTier,
+) -> GemvTile {
+ // These tables are fitted for batch-1 Q4 only; Q8/future widths keep the
+ // deterministic default until they have their own cold sweep.
+ if m != 1 || bits != 4 {
+ return DEFAULT_TILE;
+ }
+ if has_rht {
+ // This special case mirrors quant bucket edges: n in (2048, 4096]
+ // and k at or above the 2048 boundary.
+ return if tier == DeviceTier::Large
+ && n > QUANT_RHT_TUNED_N_MIN_EXCLUSIVE
+ && n <= QUANT_RHT_TUNED_N_MAX
+ && k >= QUANT_RHT_TUNED_K_MIN
+ {
+ qtile(4, 8)
+ } else {
+ DEFAULT_TILE
+ };
+ }
+
+ let k_bucket = table_bucket_index(k, &QUANT_K_BUCKET_MAXES);
+ let n_bucket = table_bucket_index(n, &QUANT_N_BUCKET_MAXES);
+ // Q4 BF16 decode choices from June 2026 gemv_fine_tune sweeps; omitted
+ // cells keep SG8_KS1_R4. Other quant widths keep DEFAULT_TILE until swept.
+ let selected = match (tier, k_bucket, n_bucket) {
+ (DeviceTier::Large, 0, 1) => Q42,
+ (DeviceTier::Large, 1, 0) => Q21,
+ (DeviceTier::Large, 1, 1..=3) => Q22,
+ (DeviceTier::Large, 1, 4) => Q21,
+ (DeviceTier::Large, 1, 5) => Q24,
+ (DeviceTier::Large, 2, 1) => Q42,
+ (DeviceTier::Large, 3, 1) => Q22,
+
+ (DeviceTier::SmallApple9, 0, 1) => Q44,
+ (DeviceTier::SmallApple9, 1, 0) => Q42,
+ (DeviceTier::SmallApple9, 1, 1) => Q22,
+ (DeviceTier::SmallApple9, 1, 2) => Q42,
+ (DeviceTier::SmallApple9, 1, 4) => Q22,
+ (DeviceTier::SmallApple9, 1, 5) => Q42,
+ (DeviceTier::SmallApple9, 2, 1) => Q42,
+ (DeviceTier::SmallApple9, 3, 1) => Q22,
+
+ (DeviceTier::SmallApple8, 0, 1) => Q44,
+ (DeviceTier::SmallApple8, 1, _) | (DeviceTier::SmallApple8, 2, 1) => Q82,
+
+ (DeviceTier::SmallLegacy, 0, 1) => Q48,
+ (DeviceTier::SmallLegacy, 1, 0..=3) => Q82,
+
+ _ => DEFAULT_TILE,
+ };
+ if n < selected.results_per_simdgroup {
+ // Coarse N buckets can include n < R; keep the default R4 tile for tiny rows.
+ DEFAULT_TILE
+ } else {
+ selected
+ }
+}
+
+#[cfg(test)]
+#[path = "../../../../../../tests/unit/backends/metal/kernel/matmul/gemv/policy_test.rs"]
+mod tests;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 3b2f87986..67f6fd11d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -38,8 +38,7 @@ impl MatmulKernel for MatmulMetalKernel {
}
let gemm = GemmKernel::new(context, weights_data_type, input_data_type, output_data_type)?;
- let gemv = GemvDispatch::new(context, weights_data_type, input_data_type, output_data_type)
- .map_err(MetalError::from)?;
+ let gemv = GemvDispatch::new(weights_data_type, input_data_type, output_data_type);
Ok(Self {
gemv,
@@ -60,6 +59,7 @@ impl MatmulKernel for MatmulMetalKernel {
self.weights_data_type,
self.input_data_type,
self.output_data_type,
+ encoder.context().device_tier(),
) {
Some(spec) => self.gemv.encode(arguments, spec, encoder).map_err(MetalError::from),
None => self.gemm.encode(arguments, encoder),
diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rs
index e84dd4691..bfa4c1c5f 100644
--- a/crates/backend-uzu/src/backends/metal/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/mod.rs
@@ -3,6 +3,7 @@ mod buffer;
mod command_buffer;
mod context;
mod dense_buffer;
+mod device_tier;
mod error;
mod kernel;
mod metal_extensions;
diff --git a/crates/backend-uzu/tests/unit/backends/metal/device_tier_test.rs b/crates/backend-uzu/tests/unit/backends/metal/device_tier_test.rs
new file mode 100644
index 000000000..780535713
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/backends/metal/device_tier_test.rs
@@ -0,0 +1,11 @@
+use proc_macros::uzu_test;
+
+use super::*;
+
+#[uzu_test]
+fn device_tier_detection() {
+ assert_eq!(device_tier_for(40, true, true), DeviceTier::Large);
+ assert_eq!(device_tier_for(20, true, true), DeviceTier::SmallApple9);
+ assert_eq!(device_tier_for(10, true, false), DeviceTier::SmallApple8);
+ assert_eq!(device_tier_for(8, false, false), DeviceTier::SmallLegacy);
+}
diff --git a/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
new file mode 100644
index 000000000..21e77352c
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
@@ -0,0 +1,41 @@
+use proc_macros::uzu_test;
+
+use super::*;
+
+#[uzu_test]
+fn fp_policy_cases() {
+ #[rustfmt::skip]
+ let cases = [
+ (DeviceTier::Large, 1, 12288, 1536, true, tile(8, 8, 1)),
+ (DeviceTier::Large, 1, 12288, 1536, false, tile(8, 1, 1)),
+ (DeviceTier::Large, 1, 1536, 256, true, tile(8, 1, 1)),
+ (DeviceTier::SmallApple9, 1, 1536, 256, true, tile(8, 2, 1)),
+ (DeviceTier::Large, 8, 12288, 1536, true, tile(8, 1, 4)),
+ (DeviceTier::Large, 1, 1536, 12288, true, tile(8, 8, 4)),
+ (DeviceTier::SmallLegacy, 1, 262144, 1536, true, tile(8, 1, 4)),
+ ];
+
+ for (tier, m, n, k, aligned, expected) in cases {
+ assert_eq!(fp_tile(m, n, k, aligned, tier), expected, "tier={tier:?} m={m} n={n} k={k}");
+ }
+}
+
+#[uzu_test]
+fn quant_policy_cases() {
+ #[rustfmt::skip]
+ let cases = [
+ (DeviceTier::Large, 1, 256, 1536, 4, false, qtile(2, 1)),
+ (DeviceTier::Large, 1, 262144, 1536, 4, false, DEFAULT_TILE),
+ (DeviceTier::SmallApple9, 1, 1536, 256, 4, false, qtile(4, 4)),
+ (DeviceTier::SmallApple8, 1, 2048, 1536, 4, false, qtile(8, 2)),
+ (DeviceTier::SmallLegacy, 1, 256, 1536, 4, false, qtile(8, 2)),
+ (DeviceTier::SmallLegacy, 1, 1536, 256, 4, false, qtile(4, 8)),
+ (DeviceTier::Large, 2, 2048, 1536, 4, false, DEFAULT_TILE),
+ (DeviceTier::Large, 1, 2048, 1536, 8, false, DEFAULT_TILE),
+ (DeviceTier::Large, 1, 2560, 9216, 4, true, qtile(4, 8)),
+ ];
+
+ for (tier, m, n, k, bits, has_rht, expected) in cases {
+ assert_eq!(quant_tile(m, n, k, bits, has_rht, tier), expected, "tier={tier:?} m={m} n={n} k={k} bits={bits}");
+ }
+}