A high-performance inference engine for AI models
Fix output RHT for low-batch matmul and bias ordering (#874)
CCYeh72c48b316709
diff --git a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rsindex 64afa3ba3..7a9199051 100644--- a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs+++ b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs@@ -56,6 +56,7 @@ impl ActivationQuantization {pub struct ActivationTransform<B: Backend> {kernel: <B::Kernels as Kernels>::ActivationTransformKernel,in_place: bool,+ has_bias: bool,quantization: Option<ActivationQuantization>,}@@ -63,10 +64,12 @@ impl<B: Backend> ActivationTransform<B> {fn new(context: &B::Context,data_type: DataType,+ bias_data_type: Option<DataType>,ops: ActivationTransformOp,in_place: bool,quantization: Option<ActivationQuantization>,) -> Result<Self, B::Error> {+ let has_bias = bias_data_type.is_some();let (codes_grouped_by_nibble, scale_group_size, sum_group_size) = match quantization {Some(q) => (q.code_layout.is_grouped_by_nibble(), q.scale_group_size, q.sum_group_size),None => (false, HADAMARD_TRANSFORM_BLOCK_SIZE, None),@@ -74,15 +77,18 @@ impl<B: Backend> ActivationTransform<B> {let kernel = <B::Kernels as Kernels>::ActivationTransformKernel::new(context,data_type,+ bias_data_type.unwrap_or(data_type),ops,codes_grouped_by_nibble,in_place,scale_group_size,sum_group_size.unwrap_or(HADAMARD_TRANSFORM_BLOCK_SIZE),+ has_bias,)?;Ok(Self {kernel,in_place,+ has_bias,quantization,})}@@ -92,15 +98,16 @@ impl<B: Backend> ActivationTransform<B> {data_type: DataType,in_place: bool,) -> Result<Self, B::Error> {- Self::new(context, data_type, ActivationTransformOp::InputRht, in_place, None)+ Self::new(context, data_type, None, ActivationTransformOp::InputRht, in_place, None)}pub fn output_rht(context: &B::Context,data_type: DataType,+ bias_data_type: Option<DataType>,in_place: bool,) -> Result<Self, B::Error> {- Self::new(context, data_type, ActivationTransformOp::OutputRht, in_place, None)+ Self::new(context, data_type, bias_data_type, ActivationTransformOp::OutputRht, in_place, None)}pub fn quantize(@@ -111,7 +118,7 @@ impl<B: Backend> ActivationTransform<B> {let op = quantization.sum_group_size().map_or(ActivationTransformOp::Quantize, |_| ActivationTransformOp::QuantizeWithGroupSums);- Self::new(context, data_type, op, false, Some(quantization))+ Self::new(context, data_type, None, op, false, Some(quantization))}pub fn scale_group_size(&self) -> u32 {@@ -136,11 +143,12 @@ impl<B: Backend> ActivationTransform<B> {element_count: u32,encoder: &mut Encoder<B>,) {- assert!(self.quantization.is_none() && !self.in_place);+ assert!(self.quantization.is_none() && !self.in_place && !self.has_bias);assert_row_width(element_count);self.kernel.encode(Some(input),Some(output),+ None::<&Allocation<B>>,None::<&mut Allocation<B>>,None::<&mut Allocation<B>>,None::<&mut Allocation<B>>,@@ -155,15 +163,17 @@ impl<B: Backend> ActivationTransform<B> {&self,data: &mut Allocation<B>,rht_factors: &Allocation<B>,+ bias: Option<&Allocation<B>>,batch_size: u32,element_count: u32,encoder: &mut Encoder<B>,) {- assert!(self.quantization.is_none() && self.in_place);+ assert!(self.quantization.is_none() && self.in_place && self.has_bias == bias.is_some());assert_row_width(element_count);self.kernel.encode(None::<&Allocation<B>>,Some(data),+ bias,None::<&mut Allocation<B>>,None::<&mut Allocation<B>>,None::<&mut Allocation<B>>,@@ -198,6 +208,7 @@ impl<B: Backend> ActivationTransform<B> {self.kernel.encode(Some(input),None::<&mut Allocation<B>>,+ None::<&Allocation<B>>,Some(q_out),Some(scales_out),group_sums_out,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rsindex 259045989..1568d5edd 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs@@ -47,11 +47,13 @@ pub fn quantize_transformed_row(#[kernel(ActivationTransform)]#[variants(T, f32, bf16)]-pub fn activation_transform<T: ArrayElement + Float>(+#[variants(BiasT, f32, bf16)]+pub fn activation_transform<T: ArrayElement + Float, BiasT: ArrayElement + Float>(#[optional(!in_place)] input: Option<*const T>,#[optional(ops == ActivationTransformOp::InputRht || ops == ActivationTransformOp::OutputRht)] fp_out: Option<*mut T,>,+ #[optional(has_bias)] bias: Option<*const BiasT>,#[optional(ops == ActivationTransformOp::Quantize|| ops == ActivationTransformOp::QuantizeWithGroupSums)]q_out: Option<*mut i8>,@@ -67,6 +69,7 @@ pub fn activation_transform<T: ArrayElement + Float>(#[specialize] in_place: bool,#[specialize] activation_scale_group_size: u32,#[specialize] sum_group_size: u32,+ #[specialize] has_bias: bool,) {let input = match in_place {true => fp_out.expect("in-place transform requires fp_out"),@@ -76,6 +79,7 @@ pub fn activation_transform<T: ArrayElement + Float>(let columns = element_count as usize;let input_rht = ops != ActivationTransformOp::OutputRht;let quantize = matches!(ops, ActivationTransformOp::Quantize | ActivationTransformOp::QuantizeWithGroupSums);+ assert_eq!(bias.is_some(), has_bias);let mut transformed = vec![0.0f32; columns];for row in 0..rows {@@ -136,8 +140,15 @@ pub fn activation_transform<T: ArrayElement + Float>(} else {let fp_out = fp_out.expect("FP transform requires fp_out");for index in 0..columns {+ let mut value = transformed[index];+ if let Some(bias) = bias {+ let rounded: T = NumCast::from(value).unwrap();+ value = NumCast::from(rounded).unwrap();+ let bias_value: f32 = NumCast::from(unsafe { *bias.add(index) }).unwrap();+ value += bias_value;+ }unsafe {- *fp_out.add(row_offset + index) = <T as NumCast>::from(transformed[index]).unwrap();+ *fp_out.add(row_offset + index) = NumCast::from(value).unwrap();}}}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rsindex 8a4612f41..7670be9c8 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs@@ -37,7 +37,7 @@ impl MatmulKernel for MatmulCpuKernel {return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());}}- let output_rht = ActivationTransform::output_rht(context, output_data_type, true)?;+ let output_rht = ActivationTransform::output_rht(context, output_data_type, None, true)?;let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context,output_data_type,@@ -267,7 +267,7 @@ impl MatmulKernel for MatmulCpuKernel {});if let Some(factors) = post_rht {- self.output_rht.encode_fp_in_place(&mut *d, factors, m, n, encoder);+ self.output_rht.encode_fp_in_place(&mut *d, factors, None, m, n, encoder);if let Some(bias) = bias_alloc {let output_length = m.checked_mul(n).expect("matmul output length must fit in u32");self.bias_add.encode(None::<&Allocation<Cpu>>, bias, &mut *d, n, output_length, encoder);diff --git a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metalindex 65867c8af..a23827380 100644--- a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal@@ -15,11 +15,13 @@ using namespace uzu::activation_transform;#define QUANTIZED (ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)#define EMITS_GROUP_SUMS (ops == ActivationTransformOp::QuantizeWithGroupSums)-template <typename T>+template <typename T, typename BiasT>VARIANTS(T, float, bfloat)+VARIANTS(BiasT, float, bfloat)PUBLIC KERNEL(ActivationTransform)(const device T* input OPTIONAL(!in_place),device T* fp_out OPTIONAL(!QUANTIZED),+ const device BiasT* bias OPTIONAL(has_bias),device int8_t* q_out OPTIONAL(QUANTIZED),device float* scales_out OPTIONAL(QUANTIZED),device int32_t* group_sums_out OPTIONAL(EMITS_GROUP_SUMS),@@ -31,6 +33,7 @@ PUBLIC KERNEL(ActivationTransform)(const bool in_place SPECIALIZE,const uint activation_scale_group_size SPECIALIZE,const uint sum_group_size SPECIALIZE,+ const bool has_bias SPECIALIZE,threadgroup float partial_max OPTIONAL(QUANTIZED && activation_scale_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],threadgroup int partial_sums OPTIONAL(EMITS_GROUP_SUMS && sum_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],uint activation_tile_index GROUPS(element_count.div_ceil(NUM_THREADS)),@@ -61,6 +64,9 @@ PUBLIC KERNEL(ActivationTransform)(if (!QUANTIZED) {if (in_bounds) {+ if (has_bias) {+ value = static_cast<float>(static_cast<T>(value)) + static_cast<float>(bias[factor_index]);+ }fp_out[element_index] = static_cast<T>(value);}return;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.hindex 86c770a1b..30e3519fc 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h@@ -5,10 +5,12 @@using namespace metal;-template <typename T>+template <typename T, typename BiasT>static METAL_FUNC void apply_output_random_hadamard_transform(device T* output_block,const device int32_t* rht_factors_block,+ const device BiasT* bias_block,+ bool apply_bias,ushort tile_block_rows,ushort tile_block_cols,uint leading_dimension_d,@@ -23,7 +25,11 @@ static METAL_FUNC void apply_output_random_hadamard_transform(const ushort stripe = ushort(cell % stripes_per_row);const ushort col_local = stripe * ushort(HADAMARD_TRANSFORM_BLOCK_SIZE) + simd_lane;const size_t output_index = size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);- output_block[output_index] =+ T transformed =simdgroup_output_random_hadamard_transform(simd_lane, output_block[output_index], rht_factors_block[col_local]);+ if (apply_bias) {+ transformed = T(float(transformed) + float(bias_block[col_local]));+ }+ output_block[output_index] = transformed;}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 50ddcce65..301c78864 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -122,6 +122,7 @@ struct MxuMmaCore {const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);const bool apply_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);+ const bool apply_rht = output_transform.contains(GemmDTransform::RHT);const device RightElementType* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);@@ -159,7 +160,7 @@ struct MxuMmaCore {accumulator_tile.map([&](auto value) { return value + AccumulatorType(*(existing_data++)); });}- if (apply_bias) {+ if (apply_bias && !apply_rht) {accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {if constexpr (aligned_n.value) {return value + AccumulatorType(bias_simdgroup[col]);@@ -200,6 +201,8 @@ struct MxuMmaCore {apply_output_random_hadamard_transform(d_block,rht_factors + block_col,+ output_bias + block_col,+ apply_bias,tile_block_rows,tile_block_cols,params->leading_dimension_d,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex d8dc76ddc..e790b6181 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -151,7 +151,7 @@ struct SimdgroupMmaCore {if (needs_epilogue) {accumulator.apply_epilogue(d, params->leading_dimension_d, 1, epilogue);}- if (needs_bias) {+ if (needs_bias && !needs_rht) {accumulator.apply_bias(bias_block);}accumulator.store_result(d, params->leading_dimension_d);@@ -160,7 +160,7 @@ struct SimdgroupMmaCore {accumulator.apply_epilogue_safe(d, params->leading_dimension_d, 1, short2(tile_block_cols, tile_block_rows), epilogue);}- if (needs_bias) {+ if (needs_bias && !needs_rht) {accumulator.apply_bias_safe(bias_block, short2(tile_block_cols, tile_block_rows));}accumulator.store_result_safe(d, params->leading_dimension_d, short2(tile_block_cols, tile_block_rows));@@ -171,6 +171,8 @@ struct SimdgroupMmaCore {apply_output_random_hadamard_transform(d,rht_factors_block,+ bias_block,+ needs_bias,tile_block_rows,tile_block_cols,params->leading_dimension_d,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 63108f7eb..78cad2172 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -1,7 +1,7 @@use std::collections::{HashMap, hash_map::Entry};use super::{- super::supports_integer_right_operand,+ super::{MatmulOutputWork, supports_integer_right_operand},GemmEngine, GemmPlan,selection::{GemmProblem, outer_block_k},specialization::GemmSpecialization,@@ -14,19 +14,16 @@ use crate::{GemmParams,gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform},},- kernel::{- ActivationTransform, TensorAddBiasKernel,- matmul::{- Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulShape, QuantParamsLayout,- QuantParamsStrides,- },+ kernel::matmul::{+ Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulError, MatmulShape,+ QuantParamsLayout, QuantParamsStrides, QuantizedB,},},metal::{Metal,context::MetalContext,error::MetalError,- kernel::{GemmMetalKernel, GemmSplitKReduceMetalKernel, TensorAddBiasMetalKernel},+ kernel::{GemmMetalKernel, GemmSplitKReduceMetalKernel},},},data_type::DataType,@@ -37,30 +34,22 @@ pub struct GemmKernel {input_data_type: DataType,output_data_type: DataType,kernels: HashMap<GemmSpecialization, GemmMetalKernel>,- pub bias_add: TensorAddBiasMetalKernel,- output_rht: ActivationTransform<Metal>,split_k_reduce: HashMap<GemmDTransform, GemmSplitKReduceMetalKernel>,}impl GemmKernel {pub fn new(- context: &MetalContext,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,- ) -> Result<Self, MetalError> {- let bias_add = TensorAddBiasMetalKernel::new(context, output_data_type, weights_data_type, true)?;- let output_rht = ActivationTransform::output_rht(context, output_data_type, true)?;- let kernel = Self {+ ) -> Self {+ Self {weights_data_type,input_data_type,output_data_type,kernels: HashMap::new(),- bias_add,- output_rht,split_k_reduce: HashMap::new(),- };- Ok(kernel)+ }}fn get_or_create(@@ -122,7 +111,7 @@ impl GemmKernel {}#[cfg(test)]- fn select_plan_for_engine(+ pub fn select_plan_for_engine(&self,shape: &MatmulShape,engine: GemmEngine,@@ -133,22 +122,11 @@ impl GemmKernel {.map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))}- #[cfg(test)]- pub fn encode_with_engine<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(- &mut self,- arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,- engine: GemmEngine,- encoder: &mut Encoder<Metal>,- ) -> Result<(), MetalError> {- let shape = MatmulShape::from_arguments(&arguments);- let plan = self.select_plan_for_engine(&shape, engine, encoder.context())?;- self.encode_plan(arguments, plan, encoder)- }-pub fn encode_plan<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(&mut self,arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,plan: GemmPlan,+ output_work: &MatmulOutputWork,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {let shape = MatmulShape::from_arguments(&arguments);@@ -156,154 +134,84 @@ impl GemmKernel {.validate_engine(plan.engine).map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))?;- let is_quant = !matches!(arguments.b, MatmulB::FullPrecision { .. });- if is_quant {- let d_mask = arguments.d_transform.mask();- if d_mask.contains(GemmDTransform::ACCUMULATE) {- return Err(MatmulError::UnsupportedDOp {- bit: GemmDTransform::ACCUMULATE,- path: "QuantGemm",- }- .into());+ if !matches!(arguments.b, MatmulB::FullPrecision { .. })+ && arguments.d_transform.mask().contains(GemmDTransform::ACCUMULATE)+ {+ return Err(MatmulError::UnsupportedDOp {+ bit: GemmDTransform::ACCUMULATE,+ path: "QuantGemm",}+ .into());}- let ab_scale = arguments.d_transform.ab_scale;- let output_bias = arguments.d_transform.bias;- let rht_factors = arguments.d_transform.rht_factors;- let output_transform = arguments.d_transform.mask();-let MatmulArguments {a,b,- b_leading_dimension,- b_transpose,d,- m,- n,- k,+ d_transform,..} = arguments;-- let use_mxu = plan.engine == GemmEngine::Mxu;-- let quantized = match b {+ match b {MatmulB::FullPrecision {b: weights,- } => {- let MatmulA::FullPrecision {- values: a,- offset: a_offset,- } = a- else {- return Err(MatmulError::IncompatibleA {- path: "Gemm",- reason: "int8 activations require quantized weights",- }- .into());- };-- let tiling = plan.tiling;-- let threadgroups_per_row = n.div_ceil(tiling.block_n());- let threadgroups_per_column = m.div_ceil(tiling.block_m());-- let (use_morton, group_count_x, group_count_y) = if use_mxu {- let max_dim = threadgroups_per_row.max(threadgroups_per_column);- let min_dim = threadgroups_per_row.min(threadgroups_per_column);- let morton_dim = max_dim.next_power_of_two();- let morton_total = morton_dim.saturating_mul(morton_dim);- let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);- let use_morton = min_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);- if use_morton {- (true, morton_total, 1)- } else {- (false, threadgroups_per_row, threadgroups_per_column)- }- } else {- (false, threadgroups_per_row, threadgroups_per_column)- };-- let alignment =- GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);+ } => self.encode_weights(a, weights, None, d, d_transform, shape, plan, output_work, encoder),+ MatmulB::Quantized(quantized) => self.encode_weights(+ a,+ quantized.codes,+ Some(quantized),+ d,+ d_transform,+ shape,+ plan,+ output_work,+ encoder,+ ),+ }+ }- if plan.split_k > 1 {- return self.encode_split_k(- MatmulA::FullPrecision {- values: a,- offset: a_offset,- },- weights,- None,- None,- None,- &mut *d,- ab_scale,- None,- None,- shape,- plan,- output_transform,- output_bias,- rht_factors,- encoder,- );- }+ fn encode_weights<'a, 'b, 'd, WB: BufferArg<'b, Metal>>(+ &mut self,+ a: MatmulA<'a, Metal>,+ weights: WB,+ quantized: Option<QuantizedB<'b, Metal>>,+ d: &mut Allocation<Metal>,+ d_transform: MatmulDOps<'d, Metal>,+ shape: MatmulShape,+ plan: GemmPlan,+ output_work: &MatmulOutputWork,+ encoder: &mut Encoder<Metal>,+ ) -> Result<(), MetalError> {+ let (m, n, k) = (shape.m, shape.n, shape.k);+ let ab_scale = d_transform.ab_scale;+ let output_bias = d_transform.bias;+ let rht_factors = d_transform.rht_factors;+ let output_transform = d_transform.mask();+ let (output_bias, bias_after_rht, output_transform) =+ if plan.split_k > 1 && rht_factors.is_some() && output_bias.is_some() {+ (None, output_bias, output_transform.difference(GemmDTransform::BIAS))+ } else {+ (output_bias, None, output_transform)+ };- let default_ldb = if b_transpose {- k- } else {- n- };- let params = GemmParams {- M: m,- N: n,- K: k,- leading_dimension_a: k,- leading_dimension_b: b_leading_dimension.unwrap_or(default_ldb),- leading_dimension_d: n,- threadgroups_per_row,- threadgroups_per_column,- aligned_inner_iterations: k / tiling.block_k(),- use_morton,- ab_scale,- scale_output_stride: 0,- scale_group_stride: 0,- zero_point_output_stride: 0,- zero_point_group_stride: 0,- };+ let use_mxu = plan.engine == GemmEngine::Mxu;- let specialization = GemmSpecialization::from_plan(- plan,- shape,- self.weights_data_type,- output_transform,- alignment,- GemmAPrologueKind::FullPrecision,- None,- )?;- let kernel = self.get_or_create(encoder.context(), specialization)?;- kernel.encode(- Some((a, a_offset)),- weights,- &mut *d,- None::<&Allocation<Metal>>,- None::<&Allocation<Metal>>,- None::<&Allocation<Metal>>,- output_bias,- rht_factors,- None::<&Allocation<Metal>>,- None::<&Allocation<Metal>>,- None::<&Allocation<Metal>>,- std::slice::from_ref(¶ms),- group_count_x,- group_count_y,- 1,- encoder,- );- return Ok(());- },- MatmulB::Quantized(quantized) => quantized,+ let is_quant = quantized.is_some();+ if !is_quant && !matches!(&a, MatmulA::FullPrecision { .. }) {+ return Err(MatmulError::IncompatibleA {+ path: "Gemm",+ reason: "int8 activations require quantized weights",+ }+ .into());+ }+ let (scales, biases, zero_points, scale_strides, zero_point_strides) = match quantized.as_ref() {+ None => (None, None, None, Default::default(), Default::default()),+ Some(quantized) => (+ Some(quantized.scales),+ quantized.biases(),+ quantized.zero_points(),+ quantized.params.scale_strides(),+ quantized.zero_point_strides(),+ ),};let a_prologue = a.prologue_kind();let (a_full_precision, a_int8, a_scales, a_group_sums, a_group_size) = match &a {@@ -336,79 +244,90 @@ impl GemmKernel {},};- let (output_bias, bias_after_rht, output_transform) = if rht_factors.is_some() && output_bias.is_some() {- (None, output_bias, output_transform.difference(GemmDTransform::BIAS))- } else {- (output_bias, None, output_transform)- };-- let tiling = plan.tiling;- let alignment =- GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);- let scale_strides = quantized.params.scale_strides();- let zero_point_strides = quantized.zero_point_strides();- let params = gemm_params(shape, plan, ab_scale, scale_strides, zero_point_strides);- let group_count_x = n.div_ceil(tiling.block_n());- let group_count_y = m.div_ceil(tiling.block_m());-if plan.split_k > 1 {self.encode_split_k(a,- quantized.codes,- Some(quantized.scales),- quantized.biases(),- quantized.zero_points(),+ weights,+ scales,+ biases,+ zero_points,&mut *d,ab_scale,- Some(scale_strides),- Some(zero_point_strides),+ scale_strides,+ zero_point_strides,shape,plan,output_transform,output_bias,- rht_factors,encoder,)?;- } else {- let specialization = GemmSpecialization::from_plan(- plan,- shape,- self.weights_data_type,- output_transform,- alignment,- a_prologue,- a_group_size,- )?;- let kernel = self.get_or_create(encoder.context(), specialization)?;- kernel.encode(- a_full_precision,- quantized.codes,- &mut *d,- Some(quantized.scales),- quantized.biases(),- quantized.zero_points(),- output_bias,- rht_factors,- a_int8,- a_scales,- a_group_sums,- std::slice::from_ref(¶ms),- group_count_x,- group_count_y,- 1,- encoder,- );+ if let Some(factors) = rht_factors {+ output_work.apply(&mut *d, factors, bias_after_rht, m, n, encoder);+ }+ return Ok(());}- if let Some(bias) = bias_after_rht {- let output_length = m.checked_mul(n).expect("GEMM output length must fit in u32");- self.bias_add.encode(None::<&Allocation<Metal>>, bias, &mut *d, n, output_length, encoder);+ let tiling = plan.tiling;+ let alignment =+ GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);+ let mut params = gemm_params(shape, plan, ab_scale, scale_strides, zero_point_strides);+ let threadgroups_per_row = n.div_ceil(tiling.block_n());+ let threadgroups_per_column = m.div_ceil(tiling.block_m());+ let (use_morton, group_count_x, group_count_y) = if !is_quant && use_mxu {+ let max_dim = threadgroups_per_row.max(threadgroups_per_column);+ let min_dim = threadgroups_per_row.min(threadgroups_per_column);+ let morton_dim = max_dim.next_power_of_two();+ let morton_total = morton_dim.saturating_mul(morton_dim);+ let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);+ if min_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total) {+ (true, morton_total, 1)+ } else {+ (false, threadgroups_per_row, threadgroups_per_column)+ }+ } else {+ (false, threadgroups_per_row, threadgroups_per_column)+ };+ if !is_quant {+ params.leading_dimension_b = shape.b_leading_dimension.unwrap_or(if shape.b_transpose {+ k+ } else {+ n+ });}+ params.use_morton = use_morton;++ let specialization = GemmSpecialization::from_plan(+ plan,+ shape,+ self.weights_data_type,+ output_transform,+ alignment,+ a_prologue,+ a_group_size,+ )?;+ let kernel = self.get_or_create(encoder.context(), specialization)?;+ kernel.encode(+ a_full_precision,+ weights,+ &mut *d,+ scales,+ biases,+ zero_points,+ output_bias,+ rht_factors,+ a_int8,+ a_scales,+ a_group_sums,+ std::slice::from_ref(¶ms),+ group_count_x,+ group_count_y,+ 1,+ encoder,+ );Ok(())}- #[allow(clippy::too_many_arguments)]fn encode_split_k<'a, 'b, WB: BufferArg<'b, Metal>>(&mut self,a: MatmulA<'a, Metal>,@@ -418,21 +337,15 @@ impl GemmKernel {zero_points: Option<&Allocation<Metal>>,d: &mut Allocation<Metal>,ab_scale: f32,- scale_strides: Option<QuantParamsStrides>,- zero_point_strides: Option<QuantParamsStrides>,+ scale_strides: QuantParamsStrides,+ zero_point_strides: QuantParamsStrides,shape: MatmulShape,plan: GemmPlan,output_transform: GemmDTransform,output_bias: Option<&Allocation<Metal>>,- rht_factors: Option<&Allocation<Metal>>,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {- let MatmulShape {- m,- n,- k,- ..- } = shape;+ let (m, n, k) = (shape.m, shape.n, shape.k);let (a_full_precision, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match a {MatmulA::FullPrecision {values,@@ -469,26 +382,8 @@ impl GemmKernel {let elem = (m as usize) * (n as usize);let slice_bytes = elem * self.output_data_type.size_in_bytes();let mut temp = encoder.allocate_scratch(split_k as usize * slice_bytes)?;- let scale_strides = scale_strides.unwrap_or_default();- let zero_point_strides = zero_point_strides.unwrap_or_default();-- let params = GemmParams {- M: m,- N: n,- K: k,- leading_dimension_a: k,- leading_dimension_b: k,- leading_dimension_d: n,- threadgroups_per_row: base_gx,- threadgroups_per_column: base_gy,- aligned_inner_iterations: kp / k_step,- use_morton: false,- ab_scale: 1.0,- scale_output_stride: scale_strides.output_stride,- scale_group_stride: scale_strides.group_stride,- zero_point_output_stride: zero_point_strides.output_stride,- zero_point_group_stride: zero_point_strides.group_stride,- };+ let mut params = gemm_params(shape, plan, 1.0, scale_strides, zero_point_strides);+ params.aligned_inner_iterations = kp / k_step;let part_kernel = self.get_or_create(encoder.context(), part_spec)?;part_kernel.encode(a_full_precision,@@ -526,11 +421,6 @@ impl GemmKernel {let reduce = self.get_or_create_split_k_reduce(encoder.context(), reduce_transform)?;reduce.encode((&temp, 0usize), &mut *d, bias_arg, elem as u32, split_k, group_count, n, scale_arg, encoder);- if output_transform.contains(GemmDTransform::RHT)- && let Some(factors) = rht_factors- {- self.output_rht.encode_fp_in_place(&mut *d, factors, m, n, encoder);- }Ok(())}}@@ -589,12 +479,7 @@ fn gemm_params(scale_strides: QuantParamsStrides,zero_point_strides: QuantParamsStrides,) -> GemmParams {- let MatmulShape {- m,- n,- k,- ..- } = shape;+ let (m, n, k) = (shape.m, shape.n, shape.k);let tiling = plan.tiling;GemmParams {M: m,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 126f0b975..2845c4bdb 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -1,6 +1,9 @@use metal::MTLGPUFamily;-use super::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK};+use super::{+ super::MatmulOutputWork,+ policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK},+};use crate::{backends::{common::{@@ -58,7 +61,6 @@ impl GemvSpecialization {shape.m,shape.n,shape.k,- shape.d_transform,bf16_io,)} else {@@ -68,11 +70,7 @@ impl GemvSpecialization {return None;}let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);- if shape.d_transform.contains(GemmDTransform::RHT) {- Some(policy::DEFAULT_TILE)- } else {- Some(policy::fp_tile(gpu_core_count, apple_gpu_family, shape.m, shape.n, shape.k, input_aligned))- }+ Some(policy::fp_tile(gpu_core_count, apple_gpu_family, shape.m, shape.n, shape.k, input_aligned))};Self::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile?)}@@ -96,9 +94,7 @@ impl GemvSpecialization {if bad_leading_dimension {return None;}- if shape.d_transform.contains(GemmDTransform::RHT) && !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE) {- return None;- }+ let output_transform = output_transform_for_tile(shape, tile)?;if shape.d_transform.contains(GemmDTransform::ACCUMULATE) && !shape.n.is_multiple_of(32) {return None;}@@ -126,7 +122,7 @@ impl GemvSpecialization {b_prologue: shape.b_prologue,group_size: shape.b_group_size.unwrap_or(0),bits,- output_transform: shape.d_transform,+ output_transform,input_aligned,k_split: tile.k_split,output_row_tile: tile.output_row_tile(),@@ -145,6 +141,10 @@ impl GemvSpecialization {self.output_row_tile}+ pub fn fuses_rht(&self) -> bool {+ self.output_transform.contains(GemmDTransform::RHT)+ }+fn create_pipeline(&self,context: &MetalContext,@@ -175,6 +175,29 @@ impl GemvSpecialization {}}+fn output_transform_for_tile(+ shape: &MatmulShape,+ tile: policy::GemvTile,+) -> Option<GemmDTransform> {+ let transform = shape.d_transform;+ if !transform.contains(GemmDTransform::RHT) {+ return Some(transform);+ }+ if !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE) {+ return None;+ }++ let output_rows = tile.output_row_tile();+ let can_fuse = tile.k_split == 1+ && output_rows >= HADAMARD_TRANSFORM_BLOCK_SIZE+ && output_rows.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE);+ if can_fuse {+ Some(transform)+ } else {+ Some(transform.difference(GemmDTransform::RHT | GemmDTransform::BIAS))+ }+}+fn full_tile(shape: &MatmulShape,tile: policy::GemvTile,@@ -234,12 +257,19 @@ impl GemvKernel {&mut self,arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,specialization: GemvSpecialization,+ output_work: &MatmulOutputWork,encoder: &mut Encoder<Metal>,) -> Result<(), MatmulError<Metal>> {let ab_scale = arguments.d_transform.ab_scale;let output_bias = arguments.d_transform.bias;let rht_factors = arguments.d_transform.rht_factors;let soft_cap = arguments.d_transform.soft_cap;+ let deferred_factors = rht_factors.filter(|_| !specialization.fuses_rht());+ let (gemv_bias, gemv_rht_factors) = if deferred_factors.is_some() {+ (None, None)+ } else {+ (output_bias, rht_factors)+ };let MatmulArguments {a,@@ -292,8 +322,8 @@ impl GemvKernel {biases,(a, a_offset),&mut *d,- output_bias,- rht_factors,+ gemv_bias,+ gemv_rht_factors,gather_indices,k,n,@@ -308,6 +338,10 @@ impl GemvKernel {encoder,);+ if let Some(factors) = deferred_factors {+ output_work.apply(&mut *d, factors, output_bias, m, n, encoder);+ }+Ok(())}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rsindex c8d7e98d2..cc988e389 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs@@ -51,8 +51,6 @@ const fn tile(}}-pub(super) const DEFAULT_TILE: GemvTile = tile(DEFAULT_NUM_SIMDGROUPS, 1, DEFAULT_RESULTS_PER_SIMDGROUP);-impl GemvTile {pub const fn quantized(num_simdgroups: u32,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rsindex 49876973f..5420da6c0 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs@@ -1,13 +1,10 @@use metal::MTLGPUFamily;use super::{DEFAULT_RESULTS_PER_SIMDGROUP, GemvTile};-use crate::backends::{common::gpu_types::gemm::GemmDTransform, metal::context::LARGE_MIN_GPU_CORES};+use crate::backends::metal::context::LARGE_MIN_GPU_CORES;const QUANT_N_BUCKET_MAXES: [u32; 6] = [512, 2048, 4096, 8192, 16384, 32768];const QUANT_K_BUCKET_MAXES: [u32; 3] = [512, 2048, 8192];-const QUANT_RHT_TUNED_N_MIN_EXCLUSIVE: u32 = 2048;-const QUANT_RHT_TUNED_N_MAX: u32 = 4096;-const QUANT_RHT_TUNED_K_MIN: u32 = 2048;/// Lane-sliced tiles split each quantization group across reduction lanes./// A lane stages 8 bytes: 16 W4 values or 8 W8 values.@@ -60,23 +57,11 @@ fn lane_policy(k: u32,bits: u32,group: u32,- has_rht: bool,) -> GemvTile {let is_large_gpu = gpu_core_count >= LARGE_MIN_GPU_CORES;if m != 1 || bits != 4 {return lane_default(bits, group);}- if has_rht {- return if is_large_gpu- && n > QUANT_RHT_TUNED_N_MIN_EXCLUSIVE- && n <= QUANT_RHT_TUNED_N_MAX- && k >= QUANT_RHT_TUNED_K_MIN- {- lane_tile(4, 8, bits, group)- } else {- lane_default(bits, group)- };- }let k_bucket = table_bucket_index(k, &QUANT_K_BUCKET_MAXES);let n_bucket = table_bucket_index(n, &QUANT_N_BUCKET_MAXES);@@ -118,7 +103,6 @@ pub fn select(m: u32,n: u32,k: u32,- d_transform: GemmDTransform,bf16_io: bool,) -> Option<GemvTile> {if !matches!(bits, 4 | 8) || !matches!(group, 16 | 32 | 64 | 128) || !(1..=8).contains(&m) {@@ -130,19 +114,14 @@ pub fn select(return (m <= 4 && n >= tile.rows_per_lane()).then_some(tile);}- let has_rht = d_transform.contains(GemmDTransform::RHT);let tile = match m {- 1..=4 => lane_policy(gpu_core_count, apple_gpu_family, m, n, k, bits, group, has_rht),+ 1..=4 => lane_policy(gpu_core_count, apple_gpu_family, m, n, k, bits, group),5..=8 if n <= 64 => lane_default(bits, group),_ => return None,};- let rows = tile.output_row_tile();// Input rows are tiled independently; partial tiles clamp loads and stores.if n < tile.rows_per_lane() {return None;}- if has_rht && (!rows.is_multiple_of(32) || !n.is_multiple_of(rows)) {- return None;- }Some(tile)}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rsindex ca226de60..9857dba89 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs@@ -13,10 +13,10 @@ use self::{use crate::{backends::{common::{- BufferArg, Encoder,+ Allocation, BufferArg, Encoder,gpu_types::gemm::{GemmBPrologueKind, GemmTiling},kernel::{- ActivationQuantization,+ ActivationQuantization, ActivationTransform,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,matmul::{ActivationFormat, Int8CodeLayout, MatmulArguments, MatmulError, MatmulKernel, MatmulShape,@@ -32,6 +32,7 @@ use crate::{pub struct MatmulMetalKernel {gemv: GemvKernel,pub gemm: GemmKernel,+ output_work: MatmulOutputWork,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,@@ -42,6 +43,46 @@ enum MatmulDispatch {Gemm(GemmPlan),}+pub struct MatmulOutputWork {+ output_rht: ActivationTransform<Metal>,+ output_rht_with_bias: ActivationTransform<Metal>,+}++impl MatmulOutputWork {+ fn new(+ context: &MetalContext,+ weights_data_type: DataType,+ output_data_type: DataType,+ ) -> Result<Self, MetalError> {+ Ok(Self {+ output_rht: ActivationTransform::output_rht(context, output_data_type, None, true)?,+ output_rht_with_bias: ActivationTransform::output_rht(+ context,+ output_data_type,+ Some(weights_data_type),+ true,+ )?,+ })+ }++ fn apply(+ &self,+ output: &mut Allocation<Metal>,+ factors: &Allocation<Metal>,+ bias: Option<&Allocation<Metal>>,+ m: u32,+ n: u32,+ encoder: &mut Encoder<Metal>,+ ) {+ let transform = if bias.is_some() {+ &self.output_rht_with_bias+ } else {+ &self.output_rht+ };+ transform.encode_fp_in_place(output, factors, bias, m, n, encoder);+ }+}+fn supports_integer_right_operand(shape: &MatmulShape) -> bool {matches!((shape.b_bits, shape.signed_codes), (Some(4), _) | (Some(8), true))}@@ -152,12 +193,14 @@ impl MatmulKernel for MatmulMetalKernel {}}- let gemm = GemmKernel::new(context, weights_data_type, input_data_type, output_data_type)?;+ let output_work = MatmulOutputWork::new(context, weights_data_type, output_data_type)?;+ let gemm = GemmKernel::new(weights_data_type, input_data_type, output_data_type);let gemv = GemvKernel::new(weights_data_type, input_data_type, output_data_type);Ok(Self {gemv,gemm,+ output_work,weights_data_type,input_data_type,output_data_type,@@ -226,7 +269,7 @@ impl MatmulKernel for MatmulMetalKernel {let shape = MatmulShape::from_arguments(&arguments);let plan = match self.select_dispatch(&shape, encoder.context()) {MatmulDispatch::Gemv(gemv) => {- return self.gemv.encode(arguments, gemv, encoder).map_err(MetalError::from);+ return self.gemv.encode(arguments, gemv, &self.output_work, encoder).map_err(MetalError::from);},MatmulDispatch::Gemm(plan) => plan,};@@ -241,6 +284,20 @@ impl MatmulKernel for MatmulMetalKernel {.into(),));}- self.gemm.encode_plan(arguments, plan, encoder)+ self.gemm.encode_plan(arguments, plan, &self.output_work, encoder)+ }+}++#[cfg(test)]+impl MatmulMetalKernel {+ pub fn encode_with_gemm_engine<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(+ &mut self,+ arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,+ engine: gemm::GemmEngine,+ encoder: &mut Encoder<Metal>,+ ) -> Result<(), MetalError> {+ let shape = MatmulShape::from_arguments(&arguments);+ let plan = self.gemm.select_plan_for_engine(&shape, engine, encoder.context())?;+ self.gemm.encode_plan(arguments, plan, &self.output_work, encoder)}}diff --git a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs b/crates/uzu-engine/src/encodable_block/linear/input_rht.rsindex c6ddb820a..93788b823 100644--- a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs+++ b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs@@ -98,7 +98,7 @@ impl<B: Backend> InputRht<B> {}let input_dim = self.input_dim();- self.rht.encode_fp_in_place(&mut input, &self.rht_signs, batch_dim, input_dim, encoder);+ self.rht.encode_fp_in_place(&mut input, &self.rht_signs, None, batch_dim, input_dim, encoder);Ok(LinearInput::FullPrecision(input))}diff --git a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rsindex 03003245b..68401e24d 100644--- a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs@@ -112,7 +112,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {input_factors,)),Some((- ActivationTransform::output_rht(context, output_data_type, true)+ ActivationTransform::output_rht(context, output_data_type, None, true).map_err(QLoRALinearWrapperError::BackendError)?,output_factors,)),@@ -242,7 +242,14 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {}if let Some((output_hadamard_kernel, output_factors)) = &self.output_hadamard {- output_hadamard_kernel.encode_fp_in_place(&mut output, output_factors, batch_dim, self.output_dim, encoder);+ output_hadamard_kernel.encode_fp_in_place(+ &mut output,+ output_factors,+ None,+ batch_dim,+ self.output_dim,+ encoder,+ );}encoder.pop_debug_group();diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rsindex 4091001c2..6294394c1 100644--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs@@ -132,7 +132,7 @@ impl<B: Backend> TransformerLayerConv<B> {let mut output =self.pre_conv.encode(input, &coefficients, 2 * self.coefficient_count, 0, sequence_length, encoder)?;if let Some((transform, factors)) = &self.input_rht {- transform.encode_fp_in_place(&mut output, factors, sequence_length, self.model_dim, encoder);+ transform.encode_fp_in_place(&mut output, factors, None, sequence_length, self.model_dim, encoder);}Ok((output, coefficients))}diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rsindex 3ed476f2d..0c0aad447 100644--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs@@ -32,7 +32,9 @@ fn run<T: ArrayElement + Float + Debug, B: Backend>(let context = B::Context::new().expect("context");let kernel = match order {TransformOrder::Input => ActivationTransform::<B>::input_rht(context.as_ref(), T::data_type(), in_place),- TransformOrder::Output => ActivationTransform::<B>::output_rht(context.as_ref(), T::data_type(), in_place),+ TransformOrder::Output => {+ ActivationTransform::<B>::output_rht(context.as_ref(), T::data_type(), None, in_place)+ },}.expect("activation transform");@@ -42,7 +44,7 @@ fn run<T: ArrayElement + Float + Debug, B: Backend>(let batch_count = (data.len() / channel_count) as u32;let mut encoder = Encoder::new(context.as_ref()).expect("encoder");if in_place {- kernel.encode_fp_in_place(&mut input, &factors, batch_count, channel_count as u32, &mut encoder);+ kernel.encode_fp_in_place(&mut input, &factors, None, batch_count, channel_count as u32, &mut encoder);} else {kernel.encode_fp(&input, &mut output, &factors, batch_count, channel_count as u32, &mut encoder);}@@ -106,6 +108,40 @@ fn input_and_output_rht_bf16() {check::<bf16>(0.1);}+#[uzu_test]+fn output_rht_fused_bias_matches_separate_mixed_dtype() {+ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().expect("context");+ let channels = 2 * BLOCK_SIZE as usize;+ let data: Vec<bf16> = (0..2 * channels).map(|i| bf16::from_f32((i as f32 * 0.17).sin() * 2.0)).collect();+ let bias_data: Vec<f32> = (0..channels).map(|i| (i as f32 * 0.31).cos() * 0.03).collect();+ let factors_data: Vec<i32> = (0..channels)+ .map(|i| {+ if i % 3 == 0 {+ -1+ } else {+ 1+ }+ })+ .collect();+ let expected: Vec<_> = run::<bf16, B>(&data, &factors_data, channels, TransformOrder::Output, true)+ .into_iter()+ .enumerate()+ .map(|(i, value)| bf16::from_f32(value.to_f32() + bias_data[i % channels]))+ .collect();+ let mut fused = alloc_allocation_with_data::<B, bf16>(context.as_ref(), &data);+ let bias = alloc_allocation_with_data::<B, f32>(context.as_ref(), &bias_data);+ let factors = alloc_allocation_with_data::<B, i32>(context.as_ref(), &factors_data);++ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");+ ActivationTransform::output_rht(context.as_ref(), bf16::data_type(), Some(f32::data_type()), true)+ .expect("fused transform")+ .encode_fp_in_place(&mut fused, &factors, Some(&bias), 2, channels as u32, &mut encoder);+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ assert_eq!(allocation_to_vec::<B, bf16>(&fused), expected);+ });+}+mod quantize {use rand::{RngExt, SeedableRng, rngs::SmallRng};use uzu_engine_macros::uzu_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs b/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rsindex f18c273e7..0949aa88e 100644--- a/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs@@ -322,7 +322,7 @@ fn test_zero_point_group16_hadamard<T: ArrayElement + Float + Debug + Display>().collect();let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");- let output_rht = ActivationTransform::<Cpu>::output_rht(context.as_ref(), T::data_type(), false)+ let output_rht = ActivationTransform::<Cpu>::output_rht(context.as_ref(), T::data_type(), None, false).expect("Failed to create ActivationTransform");let plain_allocation = alloc_allocation_with_data::<Cpu, T>(&context, &plain);let hadamard_factors_allocation = alloc_allocation_with_data::<Cpu, i32>(&context, &hadamard_factors);diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rsindex 99caf1b13..c8b0d9c12 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs@@ -200,17 +200,17 @@ fn encode_step(n: data.n,k: data.k,};- matmul.gemm.encode_with_engine(args, GemmEngine::Mxu, encoder).expect("a8 gemm mxu encode");+ matmul.encode_with_gemm_engine(args, GemmEngine::Mxu, encoder).expect("a8 gemm mxu encode");},BenchPath::Bf16GemmMxu => {encoder.encode_copy(&data.activations, .., &mut data.a_working, ..);- hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, data.m, data.k, encoder);+ hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, None, data.m, data.k, encoder);let args = data.bf16_arguments(output);- matmul.gemm.encode_with_engine(args, GemmEngine::Mxu, encoder).expect("bf16 gemm mxu encode");+ matmul.encode_with_gemm_engine(args, GemmEngine::Mxu, encoder).expect("bf16 gemm mxu encode");},BenchPath::Bf16Routed => {encoder.encode_copy(&data.activations, .., &mut data.a_working, ..);- hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, data.m, data.k, encoder);+ hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, None, data.m, data.k, encoder);let args = data.bf16_arguments(output);matmul.encode(args, encoder).expect("routed bf16 matmul encode");},diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/dispatch_paths_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/dispatch_paths_test.rsindex 59f13f8a7..ffb867b61 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/dispatch_paths_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/dispatch_paths_test.rs@@ -120,6 +120,12 @@ fn rht_parity_bf16() {let case = Case::new(shape).with_rht(true);check_case::<bf16>(&context, &mut kernel, Some(engine), case, 1.0);}++ let split_k_case = Case::new(Shape::new(4, 1024, 32)).with_rht(true).with_bias(true);+ check_case::<bf16>(&context, &mut kernel, Some(engine), split_k_case, 0.1);++ let non_split_case = Case::new(Shape::new(64, 128, 128)).with_rht(true).with_bias(true);+ check_case::<bf16>(&context, &mut kernel, Some(engine), non_split_case, 0.1);}}@@ -181,6 +187,10 @@ fn gemv_fp_output_transforms_bf16() {for case in cases {check_case::<bf16>(&context, &mut kernel, None, case, 1.0);}++ // M5 selects the ordinary 8-row, four-slice FP GEMV tile; RHT and bias are deferred.+ let split_k_rht = Case::new(Shape::new(1, 512, 1024)).with_rht(true).with_bias(true);+ check_case::<bf16>(&context, &mut kernel, None, split_k_rht, 1.0);}#[uzu_test]diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rsindex 3d49bc736..66fe353b9 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs@@ -52,8 +52,7 @@ fn bench_gemm(c: &mut Criterion) {group.bench_function(BenchmarkId::new("BF16", shape.to_string()), |b| {iter_encode_loop::<Metal, _>(&context, b, |encoder| {kernel- .gemm- .encode_with_engine(+ .encode_with_gemm_engine(MatmulArguments {a: MatmulA::FullPrecision {values: &a,diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rsindex 62afbd834..a1c2daf04 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs@@ -211,7 +211,7 @@ fn parity_bf16_gs32_4bit_mlx_with_bias() {bias: Some(&bias_pp_buf),..MatmulDOps::none()};- matmul.gemm.encode_with_engine(args, GemmEngine::Simdgroup, &mut encoder).expect("encode quant with bias");+ matmul.encode_with_gemm_engine(args, GemmEngine::Simdgroup, &mut encoder).expect("encode quant with bias");encoder.end_encoding().submit().wait_until_completed().unwrap();let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);@@ -311,10 +311,31 @@ fn parity_gemv_unaligned_width_bf16(run_parity::<bf16>(m, k, n, gs, bits, method, None, "gemv", 0.05, 0.6);}-#[uzu_test]-fn parity_bf16_gemv_quant_rht_with_bias() {+#[rstest]+#[test_attr(uzu_test)]+#[case::decode_fused(1, 256, 64, 32, QuantizationMethod::ScaleBias, true)]+#[case::gemv(1, 256, 64, 32, QuantizationMethod::ScaleBias, false)]+#[case::short_prefill_deferred_rht_then_bias(2, 5120, 6144, 64, QuantizationMethod::ScaleZeroPoint, true)]+#[case::qmv_short_prefill_m2_g64(2, 5120, 6144, 64, QuantizationMethod::ScaleZeroPoint, false)]+#[case::short_prefill_m5_g64(5, 5120, 6144, 64, QuantizationMethod::ScaleZeroPoint, false)]+#[case::short_prefill_m7_g64(7, 5120, 6144, 64, QuantizationMethod::ScaleZeroPoint, false)]+#[case::gemm_nonsplit_bias(9, 64, 64, 64, QuantizationMethod::ScaleZeroPoint, true)]+fn parity_bf16_quant_rht(+ #[case] m: u32,+ #[case] k: u32,+ #[case] n: u32,+ #[case] group_size: u32,+ #[case] method: QuantizationMethod,+ #[case] with_bias: bool,+) {let context = MetalContext::new().expect("Metal context");- let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);+ // W4/G64 matches the Qwen short-prefill route; keep the decode anchor at G32.+ let input = QuantInput::<bf16>::new(m, k, n, group_size, 4, method, 0);+ let input = if group_size == 64 {+ input.with_group_output()+ } else {+ input+ };let rht: Vec<i32> = (0..input.n as usize).map(|i| {if i % 2 == 0 {@@ -327,71 +348,11 @@ fn parity_bf16_gemv_quant_rht_with_bias() {let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.3 + 0.05 * (j % 4) as f32).collect();let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();- let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");- let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);- let cpu_rht = crate::tests::helpers::alloc_allocation_with_data::<Cpu, i32>(&cpu_context, &rht);- let cpu_bias = crate::tests::helpers::alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, &bias_t);- let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(- &cpu_context,- bf16::data_type(),- bf16::data_type(),- bf16::data_type(),- )- .expect("MatmulCpuKernel");- let mut cpu_encoder = Encoder::<Cpu>::new(&cpu_context).expect("cpu encoder");- let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);- cpu_args.d_transform = MatmulDOps {- bias: Some(&cpu_bias),- rht_factors: Some(&cpu_rht),- ..MatmulDOps::none()- };- cpu_matmul.encode(cpu_args, &mut cpu_encoder).expect("cpu encode quant+rht+bias");- cpu_encoder.end_encoding().submit().wait_until_completed().unwrap();- let reference = allocation_to_vec::<Cpu, bf16>(&cpu_buffers.y);-- let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);- let metal_rht = crate::tests::helpers::alloc_allocation_with_data::<Metal, i32>(&context, &rht);- let metal_bias = crate::tests::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(- &context,- bf16::data_type(),- bf16::data_type(),- bf16::data_type(),- )- .expect("MatmulMetalKernel");- let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");- let mut args = quant_arguments(&mut buffers, &input);- args.d_transform = MatmulDOps {- bias: Some(&metal_bias),- rht_factors: Some(&metal_rht),- ..MatmulDOps::none()- };- matmul.encode(args, &mut encoder).expect("encode quant gemv with rht+bias");- encoder.end_encoding().submit().wait_until_completed().unwrap();- let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);-- assert_parity::<bf16>("gemv_quant_rht_bias", &reference, &actual, 0.05, 0.6);-}--#[uzu_test]-fn parity_bf16_gemv_quant_rht() {- let context = MetalContext::new().expect("Metal context");- // n % 32 == 0 so the output RHT covers whole 32-element blocks; m = 1 routes to GEMV.- let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);- let rht: Vec<i32> = (0..input.n as usize)- .map(|i| {- if i % 2 == 0 {- 1- } else {- -1- }- })- .collect();-// CPU reference applies the RHT through the CPU matmul kernel.let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);let cpu_rht = crate::tests::helpers::alloc_allocation_with_data::<Cpu, i32>(&cpu_context, &rht);+ let cpu_bias = crate::tests::helpers::alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, &bias_t);let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(&cpu_context,bf16::data_type(),@@ -402,6 +363,7 @@ fn parity_bf16_gemv_quant_rht() {let mut cpu_encoder = Encoder::<Cpu>::new(&cpu_context).expect("cpu encoder");let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);cpu_args.d_transform = MatmulDOps {+ bias: with_bias.then_some(&cpu_bias),rht_factors: Some(&cpu_rht),..MatmulDOps::none()};@@ -409,9 +371,9 @@ fn parity_bf16_gemv_quant_rht() {cpu_encoder.end_encoding().submit().wait_until_completed().unwrap();let reference = allocation_to_vec::<Cpu, bf16>(&cpu_buffers.y);- // Metal GEMV: m = 1 quant routes to GEMV, RHT selects the 8-simdgroup (32-row) layout.let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);let metal_rht = crate::tests::helpers::alloc_allocation_with_data::<Metal, i32>(&context, &rht);+ let metal_bias = crate::tests::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(&context,bf16::data_type(),@@ -422,14 +384,25 @@ fn parity_bf16_gemv_quant_rht() {let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");let mut args = quant_arguments(&mut buffers, &input);args.d_transform = MatmulDOps {+ bias: with_bias.then_some(&metal_bias),rht_factors: Some(&metal_rht),..MatmulDOps::none()};- matmul.encode(args, &mut encoder).expect("encode quant gemv with rht");+ matmul.encode(args, &mut encoder).expect("encode quant matmul with rht");encoder.end_encoding().submit().wait_until_completed().unwrap();let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);- assert_parity::<bf16>("gemv_quant_rht", &reference, &actual, 0.05, 0.6);+ assert_parity::<bf16>(+ if with_bias {+ "quant_rht_bias"+ } else {+ "quant_rht"+ },+ &reference,+ &actual,+ 0.05,+ 0.6,+ );}#[uzu_test]@@ -525,7 +498,7 @@ fn quant_gemm_parameter_layout_prefix_matches_cpu(let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");let mut args = quant_arguments(&mut buffers, &input);args.n = logical_n;- matmul.gemm.encode_with_engine(args, engine, &mut encoder).unwrap();+ matmul.encode_with_gemm_engine(args, engine, &mut encoder).unwrap();encoder.end_encoding().submit().wait_until_completed().unwrap();let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);let expected: Vec<_> = reference@@ -786,8 +759,7 @@ fn a8w_mxu_output_bias_parity_bf16(..MatmulDOps::none()};metal_matmul- .gemm- .encode_with_engine(metal_arguments, GemmEngine::Mxu, &mut metal_encoder)+ .encode_with_gemm_engine(metal_arguments, GemmEngine::Mxu, &mut metal_encoder).expect("Metal A8W MXU matmul with output bias");metal_encoder.end_encoding().submit().wait_until_completed().unwrap();let actual = allocation_to_vec::<Metal, bf16>(&metal_buffers.y);diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rsindex 265599eb8..70ae2fd96 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs@@ -59,8 +59,7 @@ fn bench_unified_quant_typed<T: ArrayElement + Float>(group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {iter_encode_loop::<Metal, _>(context, b, |encoder| {matmul- .gemm- .encode_with_engine(quant_arguments(&mut buffers, &input), engine, encoder)+ .encode_with_gemm_engine(quant_arguments(&mut buffers, &input), engine, encoder).expect("encode unified quant matmul");});});diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex 759368e1f..466c1a2bf 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -105,6 +105,26 @@ fn selection_fallbacks_and_split_k_are_preserved() {use GemmEngine::*;use GemmTiling::*;+ for (mut case, expect_split) in [(shape(4, 32, 1024), true), (shape(64, 128, 128), false)] {+ case.d_transform = GemmDTransform::RHT | GemmDTransform::BIAS;+ for engine in [Simdgroup, Mxu] {+ let plan = problem(case, DataType::BF16)+ .select_plan_for_engine(engine)+ .expect("parity anchor supports the forced GEMM engine");+ assert_eq!(plan.split_k > 1, expect_split, "forced {engine:?} RHT+bias parity anchor");+ }+ }++ let mut non_split_quant_rht_bias = shape(9, 64, 64);+ non_split_quant_rht_bias.b_prologue = GemmBPrologueKind::ScaleZeroPointDequant;+ non_split_quant_rht_bias.b_bits = Some(4);+ non_split_quant_rht_bias.b_group_size = Some(64);+ non_split_quant_rht_bias.params_layout = Some(QuantParamsLayout::GroupOutput);+ non_split_quant_rht_bias.d_transform = GemmDTransform::RHT | GemmDTransform::BIAS;+ let plan = problem(non_split_quant_rht_bias, DataType::BF16).select_plan();+ assert_eq!(plan.engine, Simdgroup);+ assert_eq!(plan.split_k, 1);+let mut invalid_layout = quant(shape(64, 4096, 4096));invalid_layout.b_transpose = false;assert_eq!(problem(invalid_layout, DataType::BF16).select_plan().engine, Simdgroup);diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rsindex 9eebb6761..2c158982a 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs@@ -40,36 +40,21 @@ fn fp_policy_cases() {#[uzu_test]fn quant_policy_cases() {let cases = [- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 256, 1536, 4, false, qtile(2, 1)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 262144, 1536, 4, false, qtile(8, 4)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 256, 4, false, qtile(4, 4)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 1, 1536, 256, 4, false, qtile(8, 4)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple8, 1, 2048, 1536, 4, false, qtile(8, 2)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 256, 1536, 4, false, qtile(8, 2)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 1536, 256, 4, false, qtile(4, 8)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 2, 2048, 1536, 4, false, qtile(8, 4)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 5120, 5120, 4, false, qtile(8, 4)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 2048, 1536, 8, false, GemvTile::quantized(8, 4, 1, 32, 4)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 2560, 9216, 4, true, qtile(4, 8)),+ (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 256, 1536, 4, qtile(2, 1)),+ (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 262144, 1536, 4, qtile(8, 4)),+ (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 256, 4, qtile(4, 4)),+ (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 1, 1536, 256, 4, qtile(8, 4)),+ (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple8, 1, 2048, 1536, 4, qtile(8, 2)),+ (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 256, 1536, 4, qtile(8, 2)),+ (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 1536, 256, 4, qtile(4, 8)),+ (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 2, 2048, 1536, 4, qtile(8, 4)),+ (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 5120, 5120, 4, qtile(8, 4)),+ (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 2048, 1536, 8, GemvTile::quantized(8, 4, 1, 32, 4)),];- for (gpu_core_count, apple_gpu_family, m, n, k, bits, has_rht, expected) in cases {- let actual = quantized_tile(- gpu_core_count,- apple_gpu_family,- bits,- 32,- m,- n,- k,- if has_rht {- GemmDTransform::RHT- } else {- GemmDTransform::empty()- },- true,- )- .expect("quantized route");+ for (gpu_core_count, apple_gpu_family, m, n, k, bits, expected) in cases {+ let actual =+ quantized_tile(gpu_core_count, apple_gpu_family, bits, 32, m, n, k, true).expect("quantized route");assert_eq!(actual, expected,"m={m} n={n} k={k} bits={bits} gpu_core_count={gpu_core_count} apple_gpu_family={apple_gpu_family:?}"@@ -79,11 +64,8 @@ fn quant_policy_cases() {#[uzu_test]fn gpu_core_count_boundary_selects_large_policy() {- let transform = GemmDTransform::empty();- let small =- quantized(SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536, transform).expect("small GPU route");- let large =- quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536, transform).expect("large GPU route");+ let small = quantized(SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536).expect("small GPU route");+ let large = quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536).expect("large GPU route");assert_eq!(small, qtile(4, 2));assert_eq!(large, qtile(2, 1));@@ -97,70 +79,32 @@ fn quantized(m: u32,n: u32,k: u32,- d_transform: GemmDTransform,) -> Option<GemvTile> {- quantized_tile(gpu_core_count, apple_gpu_family, bits, group, m, n, k, d_transform, true)+ quantized_tile(gpu_core_count, apple_gpu_family, bits, group, m, n, k, true)}#[uzu_test]fn quantized_policy_edges() {- for (gpu_core_count, apple_gpu_family, m, group, expected) in [- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 2, 32, GemvTile::quantized(8, 4, 1, 32, 4)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 3, 32, GemvTile::quantized(8, 4, 1, 32, 4)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple8, 4, 64, GemvTile::quantized(8, 4, 1, 32, 8)),- ] {- assert_eq!(- quantized(gpu_core_count, apple_gpu_family, 8, group, m, 4096, 4096, GemmDTransform::RHT),- Some(expected)- );- }- let none = GemmDTransform::empty();assert_eq!(- quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 64, 4, 8192, 4100, none),+ quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 64, 4, 8192, 4100),Some(GemvTile::quantized(8, 4, 1, 32, 4)));- assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8, 4096, none).is_some());- assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 1, 8192, 4096, none).is_some());- assert_eq!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 9, 8192, 4096, none), None);+ assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8, 4096).is_some());+ assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 1, 8192, 4096).is_some());+ assert_eq!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 9, 8192, 4096), None);assert_eq!(- quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8192, 4096, none, true),+ quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8192, 4096, true),Some(GemvTile::quantized(8, 4, 1, 32, 2)));}#[uzu_test]fn untuned_quantized_io_uses_only_the_generated_fallback() {- assert_eq!(- quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 3, 4096, GemmDTransform::empty(), false,),- None- );- let tile = quantized_tile(- LARGE_GPU_CORE_COUNT,- MTLGPUFamily::Apple10,- 4,- 32,- 4,- 4096,- 4096,- GemmDTransform::empty(),- false,- )- .expect("mixed-IO fallback");+ assert_eq!(quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 3, 4096, false), None);+ let tile = quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 4096, 4096, false)+ .expect("mixed-IO fallback");assert_eq!(tile, GemvTile::quantized(8, 4, 1, 32, 2));- assert_eq!(- quantized_tile(- LARGE_GPU_CORE_COUNT,- MTLGPUFamily::Apple10,- 4,- 32,- 5,- 4096,- 4096,- GemmDTransform::empty(),- false,- ),- None- );+ assert_eq!(quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 5, 4096, 4096, false), None);}fn quant_shape(diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rsindex 8d77a9801..5684be512 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs@@ -17,6 +17,7 @@ use crate::{};const FROZEN_PLAN_FINGERPRINT: u64 = 5_839_212_743_558_880_136;+const DEVICES: [(&str, &str, u32, MTLGPUFamily, bool); 7] = [("m1", "Apple M1", 8, MTLGPUFamily::Apple7, false),("m2", "Apple M2", 10, MTLGPUFamily::Apple8, false),@@ -167,7 +168,9 @@ fn exact_lookup_rejects_non_matrix_inputs() {let QmvRoute::Tuned(tile) = selected else {panic!("test anchor must use a tuned tile");};- assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_some());+ let deferred = GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).unwrap();+ assert_eq!(deferred.output_row_tile(), 16);+ assert!(!deferred.fuses_rht());rht.n -= 1;assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_none());@@ -204,6 +207,23 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {MatmulDispatch::Gemv(actual) if actual == specialization));}++ let mut fp = problem(1, 1024, 512, 4, 32, GemmBPrologueKind::ScaleZeroPointDequant);+ fp.b_prologue = GemmBPrologueKind::FullPrecision;+ fp.b_bits = None;+ fp.b_group_size = None;+ let mut with_rht = fp;+ with_rht.d_transform = GemmDTransform::RHT;+ for (cores, family) in [(8, MTLGPUFamily::Apple7), (40, MTLGPUFamily::Apple10)] {+ let plain =+ GemvSpecialization::select_shape(&fp, DataType::BF16, DataType::BF16, DataType::BF16, cores, family)+ .expect("generic M=1 FP GEMV tile");+ let rht =+ GemvSpecialization::select_shape(&with_rht, DataType::BF16, DataType::BF16, DataType::BF16, cores, family)+ .expect("generic M=1 FP RHT GEMV tile");+ assert!(plain.output_row_tile() < 32);+ assert_eq!(rht, plain);+ }}#[uzu_test]diff --git a/crates/uzu-engine/unit/common/matmul/harness.rs b/crates/uzu-engine/unit/common/matmul/harness.rsindex 90900c958..8bcbef52a 100644--- a/crates/uzu-engine/unit/common/matmul/harness.rs+++ b/crates/uzu-engine/unit/common/matmul/harness.rs@@ -205,7 +205,7 @@ pub fn run_metal<T: ArrayElement + Float>() -> Vec<T> {run::<Metal, T>(context, kernel, input, |kernel, args, encoder| {if let Some(engine) = dispatch {- kernel.gemm.encode_with_engine(args, engine, encoder).expect("forced GEMM engine encode failed");+ kernel.encode_with_gemm_engine(args, engine, encoder).expect("forced GEMM engine encode failed");} else {kernel.encode(args, encoder).expect("matmul encode failed");}diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rsindex 0fbbe3774..987a2dcfa 100644--- a/crates/uzu-engine/unit/common/matmul/quant.rs+++ b/crates/uzu-engine/unit/common/matmul/quant.rs@@ -486,7 +486,7 @@ pub fn run_quant_metal<T: ArrayElement + Float>(let mut encoder = Encoder::<Metal>::new(context).expect("encoder");let args = quant_arguments(&mut buffers, input);if let Some(engine) = dispatch {- matmul.gemm.encode_with_engine(args, engine, &mut encoder).expect("forced GEMM engine encode failed");+ matmul.encode_with_gemm_engine(args, engine, &mut encoder).expect("forced GEMM engine encode failed");} else {matmul.encode(args, &mut encoder).expect("matmul encode failed");}