Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Fix output RHT for low-batch matmul and bias ordering (#874)

CCYeh72c48b316709
diff --git a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
index 64afa3ba3..7a9199051 100644
--- a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
@@ -56,6 +56,7 @@ impl ActivationQuantization {
pub struct ActivationTransform<B: Backend> {
kernel: <B::Kernels as Kernels>::ActivationTransformKernel,
in_place: bool,
+ has_bias: bool,
quantization: Option<ActivationQuantization>,
}
@@ -63,10 +64,12 @@ impl<B: Backend> ActivationTransform<B> {
fn new(
context: &B::Context,
data_type: DataType,
+ bias_data_type: Option<DataType>,
ops: ActivationTransformOp,
in_place: bool,
quantization: Option<ActivationQuantization>,
) -> Result<Self, B::Error> {
+ let has_bias = bias_data_type.is_some();
let (codes_grouped_by_nibble, scale_group_size, sum_group_size) = match quantization {
Some(q) => (q.code_layout.is_grouped_by_nibble(), q.scale_group_size, q.sum_group_size),
None => (false, HADAMARD_TRANSFORM_BLOCK_SIZE, None),
@@ -74,15 +77,18 @@ impl<B: Backend> ActivationTransform<B> {
let kernel = <B::Kernels as Kernels>::ActivationTransformKernel::new(
context,
data_type,
+ bias_data_type.unwrap_or(data_type),
ops,
codes_grouped_by_nibble,
in_place,
scale_group_size,
sum_group_size.unwrap_or(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ has_bias,
)?;
Ok(Self {
kernel,
in_place,
+ has_bias,
quantization,
})
}
@@ -92,15 +98,16 @@ impl<B: Backend> ActivationTransform<B> {
data_type: DataType,
in_place: bool,
) -> Result<Self, B::Error> {
- Self::new(context, data_type, ActivationTransformOp::InputRht, in_place, None)
+ Self::new(context, data_type, None, ActivationTransformOp::InputRht, in_place, None)
}
pub fn output_rht(
context: &B::Context,
data_type: DataType,
+ bias_data_type: Option<DataType>,
in_place: bool,
) -> Result<Self, B::Error> {
- Self::new(context, data_type, ActivationTransformOp::OutputRht, in_place, None)
+ Self::new(context, data_type, bias_data_type, ActivationTransformOp::OutputRht, in_place, None)
}
pub fn quantize(
@@ -111,7 +118,7 @@ impl<B: Backend> ActivationTransform<B> {
let op = quantization
.sum_group_size()
.map_or(ActivationTransformOp::Quantize, |_| ActivationTransformOp::QuantizeWithGroupSums);
- Self::new(context, data_type, op, false, Some(quantization))
+ Self::new(context, data_type, None, op, false, Some(quantization))
}
pub fn scale_group_size(&self) -> u32 {
@@ -136,11 +143,12 @@ impl<B: Backend> ActivationTransform<B> {
element_count: u32,
encoder: &mut Encoder<B>,
) {
- assert!(self.quantization.is_none() && !self.in_place);
+ assert!(self.quantization.is_none() && !self.in_place && !self.has_bias);
assert_row_width(element_count);
self.kernel.encode(
Some(input),
Some(output),
+ None::<&Allocation<B>>,
None::<&mut Allocation<B>>,
None::<&mut Allocation<B>>,
None::<&mut Allocation<B>>,
@@ -155,15 +163,17 @@ impl<B: Backend> ActivationTransform<B> {
&self,
data: &mut Allocation<B>,
rht_factors: &Allocation<B>,
+ bias: Option<&Allocation<B>>,
batch_size: u32,
element_count: u32,
encoder: &mut Encoder<B>,
) {
- assert!(self.quantization.is_none() && self.in_place);
+ assert!(self.quantization.is_none() && self.in_place && self.has_bias == bias.is_some());
assert_row_width(element_count);
self.kernel.encode(
None::<&Allocation<B>>,
Some(data),
+ bias,
None::<&mut Allocation<B>>,
None::<&mut Allocation<B>>,
None::<&mut Allocation<B>>,
@@ -198,6 +208,7 @@ impl<B: Backend> ActivationTransform<B> {
self.kernel.encode(
Some(input),
None::<&mut Allocation<B>>,
+ None::<&Allocation<B>>,
Some(q_out),
Some(scales_out),
group_sums_out,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
index 259045989..1568d5edd 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
@@ -47,11 +47,13 @@ pub fn quantize_transformed_row(
#[kernel(ActivationTransform)]
#[variants(T, f32, bf16)]
-pub fn activation_transform<T: ArrayElement + Float>(
+#[variants(BiasT, f32, bf16)]
+pub fn activation_transform<T: ArrayElement + Float, BiasT: ArrayElement + Float>(
#[optional(!in_place)] input: Option<*const T>,
#[optional(ops == ActivationTransformOp::InputRht || ops == ActivationTransformOp::OutputRht)] fp_out: Option<
*mut T,
>,
+ #[optional(has_bias)] bias: Option<*const BiasT>,
#[optional(ops == ActivationTransformOp::Quantize
|| ops == ActivationTransformOp::QuantizeWithGroupSums)]
q_out: Option<*mut i8>,
@@ -67,6 +69,7 @@ pub fn activation_transform<T: ArrayElement + Float>(
#[specialize] in_place: bool,
#[specialize] activation_scale_group_size: u32,
#[specialize] sum_group_size: u32,
+ #[specialize] has_bias: bool,
) {
let input = match in_place {
true => fp_out.expect("in-place transform requires fp_out"),
@@ -76,6 +79,7 @@ pub fn activation_transform<T: ArrayElement + Float>(
let columns = element_count as usize;
let input_rht = ops != ActivationTransformOp::OutputRht;
let quantize = matches!(ops, ActivationTransformOp::Quantize | ActivationTransformOp::QuantizeWithGroupSums);
+ assert_eq!(bias.is_some(), has_bias);
let mut transformed = vec![0.0f32; columns];
for row in 0..rows {
@@ -136,8 +140,15 @@ pub fn activation_transform<T: ArrayElement + Float>(
} else {
let fp_out = fp_out.expect("FP transform requires fp_out");
for index in 0..columns {
+ let mut value = transformed[index];
+ if let Some(bias) = bias {
+ let rounded: T = NumCast::from(value).unwrap();
+ value = NumCast::from(rounded).unwrap();
+ let bias_value: f32 = NumCast::from(unsafe { *bias.add(index) }).unwrap();
+ value += bias_value;
+ }
unsafe {
- *fp_out.add(row_offset + index) = <T as NumCast>::from(transformed[index]).unwrap();
+ *fp_out.add(row_offset + index) = NumCast::from(value).unwrap();
}
}
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
index 8a4612f41..7670be9c8 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
@@ -37,7 +37,7 @@ impl MatmulKernel for MatmulCpuKernel {
return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());
}
}
- let output_rht = ActivationTransform::output_rht(context, output_data_type, true)?;
+ let output_rht = ActivationTransform::output_rht(context, output_data_type, None, true)?;
let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(
context,
output_data_type,
@@ -267,7 +267,7 @@ impl MatmulKernel for MatmulCpuKernel {
});
if let Some(factors) = post_rht {
- self.output_rht.encode_fp_in_place(&mut *d, factors, m, n, encoder);
+ self.output_rht.encode_fp_in_place(&mut *d, factors, None, m, n, encoder);
if let Some(bias) = bias_alloc {
let output_length = m.checked_mul(n).expect("matmul output length must fit in u32");
self.bias_add.encode(None::<&Allocation<Cpu>>, bias, &mut *d, n, output_length, encoder);
diff --git a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal
index 65867c8af..a23827380 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal
@@ -15,11 +15,13 @@ using namespace uzu::activation_transform;
#define QUANTIZED (ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)
#define EMITS_GROUP_SUMS (ops == ActivationTransformOp::QuantizeWithGroupSums)
-template <typename T>
+template <typename T, typename BiasT>
VARIANTS(T, float, bfloat)
+VARIANTS(BiasT, float, bfloat)
PUBLIC KERNEL(ActivationTransform)(
const device T* input OPTIONAL(!in_place),
device T* fp_out OPTIONAL(!QUANTIZED),
+ const device BiasT* bias OPTIONAL(has_bias),
device int8_t* q_out OPTIONAL(QUANTIZED),
device float* scales_out OPTIONAL(QUANTIZED),
device int32_t* group_sums_out OPTIONAL(EMITS_GROUP_SUMS),
@@ -31,6 +33,7 @@ PUBLIC KERNEL(ActivationTransform)(
const bool in_place SPECIALIZE,
const uint activation_scale_group_size SPECIALIZE,
const uint sum_group_size SPECIALIZE,
+ const bool has_bias SPECIALIZE,
threadgroup float partial_max OPTIONAL(QUANTIZED && activation_scale_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],
threadgroup int partial_sums OPTIONAL(EMITS_GROUP_SUMS && sum_group_size > METAL_SIMD_SIZE)[NUM_SIMDGROUPS],
uint activation_tile_index GROUPS(element_count.div_ceil(NUM_THREADS)),
@@ -61,6 +64,9 @@ PUBLIC KERNEL(ActivationTransform)(
if (!QUANTIZED) {
if (in_bounds) {
+ if (has_bias) {
+ value = static_cast<float>(static_cast<T>(value)) + static_cast<float>(bias[factor_index]);
+ }
fp_out[element_index] = static_cast<T>(value);
}
return;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
index 86c770a1b..30e3519fc 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
@@ -5,10 +5,12 @@
using namespace metal;
-template <typename T>
+template <typename T, typename BiasT>
static METAL_FUNC void apply_output_random_hadamard_transform(
device T* output_block,
const device int32_t* rht_factors_block,
+ const device BiasT* bias_block,
+ bool apply_bias,
ushort tile_block_rows,
ushort tile_block_cols,
uint leading_dimension_d,
@@ -23,7 +25,11 @@ static METAL_FUNC void apply_output_random_hadamard_transform(
const ushort stripe = ushort(cell % stripes_per_row);
const ushort col_local = stripe * ushort(HADAMARD_TRANSFORM_BLOCK_SIZE) + simd_lane;
const size_t output_index = size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);
- output_block[output_index] =
+ T transformed =
simdgroup_output_random_hadamard_transform(simd_lane, output_block[output_index], rht_factors_block[col_local]);
+ if (apply_bias) {
+ transformed = T(float(transformed) + float(bias_block[col_local]));
+ }
+ output_block[output_index] = transformed;
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 50ddcce65..301c78864 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -122,6 +122,7 @@ struct MxuMmaCore {
const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);
const bool apply_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);
const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);
+ const bool apply_rht = output_transform.contains(GemmDTransform::RHT);
const device RightElementType* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);
@@ -159,7 +160,7 @@ struct MxuMmaCore {
accumulator_tile.map([&](auto value) { return value + AccumulatorType(*(existing_data++)); });
}
- if (apply_bias) {
+ if (apply_bias && !apply_rht) {
accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {
if constexpr (aligned_n.value) {
return value + AccumulatorType(bias_simdgroup[col]);
@@ -200,6 +201,8 @@ struct MxuMmaCore {
apply_output_random_hadamard_transform(
d_block,
rht_factors + block_col,
+ output_bias + block_col,
+ apply_bias,
tile_block_rows,
tile_block_cols,
params->leading_dimension_d,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index d8dc76ddc..e790b6181 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -151,7 +151,7 @@ struct SimdgroupMmaCore {
if (needs_epilogue) {
accumulator.apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
}
- if (needs_bias) {
+ if (needs_bias && !needs_rht) {
accumulator.apply_bias(bias_block);
}
accumulator.store_result(d, params->leading_dimension_d);
@@ -160,7 +160,7 @@ struct SimdgroupMmaCore {
accumulator
.apply_epilogue_safe(d, params->leading_dimension_d, 1, short2(tile_block_cols, tile_block_rows), epilogue);
}
- if (needs_bias) {
+ if (needs_bias && !needs_rht) {
accumulator.apply_bias_safe(bias_block, short2(tile_block_cols, tile_block_rows));
}
accumulator.store_result_safe(d, params->leading_dimension_d, short2(tile_block_cols, tile_block_rows));
@@ -171,6 +171,8 @@ struct SimdgroupMmaCore {
apply_output_random_hadamard_transform(
d,
rht_factors_block,
+ bias_block,
+ needs_bias,
tile_block_rows,
tile_block_cols,
params->leading_dimension_d,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 63108f7eb..78cad2172 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -1,7 +1,7 @@
use std::collections::{HashMap, hash_map::Entry};
use super::{
- super::supports_integer_right_operand,
+ super::{MatmulOutputWork, supports_integer_right_operand},
GemmEngine, GemmPlan,
selection::{GemmProblem, outer_block_k},
specialization::GemmSpecialization,
@@ -14,19 +14,16 @@ use crate::{
GemmParams,
gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform},
},
- kernel::{
- ActivationTransform, TensorAddBiasKernel,
- matmul::{
- Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulShape, QuantParamsLayout,
- QuantParamsStrides,
- },
+ kernel::matmul::{
+ Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulError, MatmulShape,
+ QuantParamsLayout, QuantParamsStrides, QuantizedB,
},
},
metal::{
Metal,
context::MetalContext,
error::MetalError,
- kernel::{GemmMetalKernel, GemmSplitKReduceMetalKernel, TensorAddBiasMetalKernel},
+ kernel::{GemmMetalKernel, GemmSplitKReduceMetalKernel},
},
},
data_type::DataType,
@@ -37,30 +34,22 @@ pub struct GemmKernel {
input_data_type: DataType,
output_data_type: DataType,
kernels: HashMap<GemmSpecialization, GemmMetalKernel>,
- pub bias_add: TensorAddBiasMetalKernel,
- output_rht: ActivationTransform<Metal>,
split_k_reduce: HashMap<GemmDTransform, GemmSplitKReduceMetalKernel>,
}
impl GemmKernel {
pub fn new(
- context: &MetalContext,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
- ) -> Result<Self, MetalError> {
- let bias_add = TensorAddBiasMetalKernel::new(context, output_data_type, weights_data_type, true)?;
- let output_rht = ActivationTransform::output_rht(context, output_data_type, true)?;
- let kernel = Self {
+ ) -> Self {
+ Self {
weights_data_type,
input_data_type,
output_data_type,
kernels: HashMap::new(),
- bias_add,
- output_rht,
split_k_reduce: HashMap::new(),
- };
- Ok(kernel)
+ }
}
fn get_or_create(
@@ -122,7 +111,7 @@ impl GemmKernel {
}
#[cfg(test)]
- fn select_plan_for_engine(
+ pub fn select_plan_for_engine(
&self,
shape: &MatmulShape,
engine: GemmEngine,
@@ -133,22 +122,11 @@ impl GemmKernel {
.map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))
}
- #[cfg(test)]
- pub fn encode_with_engine<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(
- &mut self,
- arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,
- engine: GemmEngine,
- encoder: &mut Encoder<Metal>,
- ) -> Result<(), MetalError> {
- let shape = MatmulShape::from_arguments(&arguments);
- let plan = self.select_plan_for_engine(&shape, engine, encoder.context())?;
- self.encode_plan(arguments, plan, encoder)
- }
-
pub fn encode_plan<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(
&mut self,
arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,
plan: GemmPlan,
+ output_work: &MatmulOutputWork,
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
let shape = MatmulShape::from_arguments(&arguments);
@@ -156,154 +134,84 @@ impl GemmKernel {
.validate_engine(plan.engine)
.map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))?;
- let is_quant = !matches!(arguments.b, MatmulB::FullPrecision { .. });
- if is_quant {
- let d_mask = arguments.d_transform.mask();
- if d_mask.contains(GemmDTransform::ACCUMULATE) {
- return Err(MatmulError::UnsupportedDOp {
- bit: GemmDTransform::ACCUMULATE,
- path: "QuantGemm",
- }
- .into());
+ if !matches!(arguments.b, MatmulB::FullPrecision { .. })
+ && arguments.d_transform.mask().contains(GemmDTransform::ACCUMULATE)
+ {
+ return Err(MatmulError::UnsupportedDOp {
+ bit: GemmDTransform::ACCUMULATE,
+ path: "QuantGemm",
}
+ .into());
}
- let ab_scale = arguments.d_transform.ab_scale;
- let output_bias = arguments.d_transform.bias;
- let rht_factors = arguments.d_transform.rht_factors;
- let output_transform = arguments.d_transform.mask();
-
let MatmulArguments {
a,
b,
- b_leading_dimension,
- b_transpose,
d,
- m,
- n,
- k,
+ d_transform,
..
} = arguments;
-
- let use_mxu = plan.engine == GemmEngine::Mxu;
-
- let quantized = match b {
+ match b {
MatmulB::FullPrecision {
b: weights,
- } => {
- let MatmulA::FullPrecision {
- values: a,
- offset: a_offset,
- } = a
- else {
- return Err(MatmulError::IncompatibleA {
- path: "Gemm",
- reason: "int8 activations require quantized weights",
- }
- .into());
- };
-
- let tiling = plan.tiling;
-
- let threadgroups_per_row = n.div_ceil(tiling.block_n());
- let threadgroups_per_column = m.div_ceil(tiling.block_m());
-
- let (use_morton, group_count_x, group_count_y) = if use_mxu {
- let max_dim = threadgroups_per_row.max(threadgroups_per_column);
- let min_dim = threadgroups_per_row.min(threadgroups_per_column);
- let morton_dim = max_dim.next_power_of_two();
- let morton_total = morton_dim.saturating_mul(morton_dim);
- let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);
- let use_morton = min_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);
- if use_morton {
- (true, morton_total, 1)
- } else {
- (false, threadgroups_per_row, threadgroups_per_column)
- }
- } else {
- (false, threadgroups_per_row, threadgroups_per_column)
- };
-
- let alignment =
- GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);
+ } => self.encode_weights(a, weights, None, d, d_transform, shape, plan, output_work, encoder),
+ MatmulB::Quantized(quantized) => self.encode_weights(
+ a,
+ quantized.codes,
+ Some(quantized),
+ d,
+ d_transform,
+ shape,
+ plan,
+ output_work,
+ encoder,
+ ),
+ }
+ }
- if plan.split_k > 1 {
- return self.encode_split_k(
- MatmulA::FullPrecision {
- values: a,
- offset: a_offset,
- },
- weights,
- None,
- None,
- None,
- &mut *d,
- ab_scale,
- None,
- None,
- shape,
- plan,
- output_transform,
- output_bias,
- rht_factors,
- encoder,
- );
- }
+ fn encode_weights<'a, 'b, 'd, WB: BufferArg<'b, Metal>>(
+ &mut self,
+ a: MatmulA<'a, Metal>,
+ weights: WB,
+ quantized: Option<QuantizedB<'b, Metal>>,
+ d: &mut Allocation<Metal>,
+ d_transform: MatmulDOps<'d, Metal>,
+ shape: MatmulShape,
+ plan: GemmPlan,
+ output_work: &MatmulOutputWork,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<(), MetalError> {
+ let (m, n, k) = (shape.m, shape.n, shape.k);
+ let ab_scale = d_transform.ab_scale;
+ let output_bias = d_transform.bias;
+ let rht_factors = d_transform.rht_factors;
+ let output_transform = d_transform.mask();
+ let (output_bias, bias_after_rht, output_transform) =
+ if plan.split_k > 1 && rht_factors.is_some() && output_bias.is_some() {
+ (None, output_bias, output_transform.difference(GemmDTransform::BIAS))
+ } else {
+ (output_bias, None, output_transform)
+ };
- let default_ldb = if b_transpose {
- k
- } else {
- n
- };
- let params = GemmParams {
- M: m,
- N: n,
- K: k,
- leading_dimension_a: k,
- leading_dimension_b: b_leading_dimension.unwrap_or(default_ldb),
- leading_dimension_d: n,
- threadgroups_per_row,
- threadgroups_per_column,
- aligned_inner_iterations: k / tiling.block_k(),
- use_morton,
- ab_scale,
- scale_output_stride: 0,
- scale_group_stride: 0,
- zero_point_output_stride: 0,
- zero_point_group_stride: 0,
- };
+ let use_mxu = plan.engine == GemmEngine::Mxu;
- let specialization = GemmSpecialization::from_plan(
- plan,
- shape,
- self.weights_data_type,
- output_transform,
- alignment,
- GemmAPrologueKind::FullPrecision,
- None,
- )?;
- let kernel = self.get_or_create(encoder.context(), specialization)?;
- kernel.encode(
- Some((a, a_offset)),
- weights,
- &mut *d,
- None::<&Allocation<Metal>>,
- None::<&Allocation<Metal>>,
- None::<&Allocation<Metal>>,
- output_bias,
- rht_factors,
- None::<&Allocation<Metal>>,
- None::<&Allocation<Metal>>,
- None::<&Allocation<Metal>>,
- std::slice::from_ref(&params),
- group_count_x,
- group_count_y,
- 1,
- encoder,
- );
- return Ok(());
- },
- MatmulB::Quantized(quantized) => quantized,
+ let is_quant = quantized.is_some();
+ if !is_quant && !matches!(&a, MatmulA::FullPrecision { .. }) {
+ return Err(MatmulError::IncompatibleA {
+ path: "Gemm",
+ reason: "int8 activations require quantized weights",
+ }
+ .into());
+ }
+ let (scales, biases, zero_points, scale_strides, zero_point_strides) = match quantized.as_ref() {
+ None => (None, None, None, Default::default(), Default::default()),
+ Some(quantized) => (
+ Some(quantized.scales),
+ quantized.biases(),
+ quantized.zero_points(),
+ quantized.params.scale_strides(),
+ quantized.zero_point_strides(),
+ ),
};
let a_prologue = a.prologue_kind();
let (a_full_precision, a_int8, a_scales, a_group_sums, a_group_size) = match &a {
@@ -336,79 +244,90 @@ impl GemmKernel {
},
};
- let (output_bias, bias_after_rht, output_transform) = if rht_factors.is_some() && output_bias.is_some() {
- (None, output_bias, output_transform.difference(GemmDTransform::BIAS))
- } else {
- (output_bias, None, output_transform)
- };
-
- let tiling = plan.tiling;
- let alignment =
- GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);
- let scale_strides = quantized.params.scale_strides();
- let zero_point_strides = quantized.zero_point_strides();
- let params = gemm_params(shape, plan, ab_scale, scale_strides, zero_point_strides);
- let group_count_x = n.div_ceil(tiling.block_n());
- let group_count_y = m.div_ceil(tiling.block_m());
-
if plan.split_k > 1 {
self.encode_split_k(
a,
- quantized.codes,
- Some(quantized.scales),
- quantized.biases(),
- quantized.zero_points(),
+ weights,
+ scales,
+ biases,
+ zero_points,
&mut *d,
ab_scale,
- Some(scale_strides),
- Some(zero_point_strides),
+ scale_strides,
+ zero_point_strides,
shape,
plan,
output_transform,
output_bias,
- rht_factors,
encoder,
)?;
- } else {
- let specialization = GemmSpecialization::from_plan(
- plan,
- shape,
- self.weights_data_type,
- output_transform,
- alignment,
- a_prologue,
- a_group_size,
- )?;
- let kernel = self.get_or_create(encoder.context(), specialization)?;
- kernel.encode(
- a_full_precision,
- quantized.codes,
- &mut *d,
- Some(quantized.scales),
- quantized.biases(),
- quantized.zero_points(),
- output_bias,
- rht_factors,
- a_int8,
- a_scales,
- a_group_sums,
- std::slice::from_ref(&params),
- group_count_x,
- group_count_y,
- 1,
- encoder,
- );
+ if let Some(factors) = rht_factors {
+ output_work.apply(&mut *d, factors, bias_after_rht, m, n, encoder);
+ }
+ return Ok(());
}
- if let Some(bias) = bias_after_rht {
- let output_length = m.checked_mul(n).expect("GEMM output length must fit in u32");
- self.bias_add.encode(None::<&Allocation<Metal>>, bias, &mut *d, n, output_length, encoder);
+ let tiling = plan.tiling;
+ let alignment =
+ GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);
+ let mut params = gemm_params(shape, plan, ab_scale, scale_strides, zero_point_strides);
+ let threadgroups_per_row = n.div_ceil(tiling.block_n());
+ let threadgroups_per_column = m.div_ceil(tiling.block_m());
+ let (use_morton, group_count_x, group_count_y) = if !is_quant && use_mxu {
+ let max_dim = threadgroups_per_row.max(threadgroups_per_column);
+ let min_dim = threadgroups_per_row.min(threadgroups_per_column);
+ let morton_dim = max_dim.next_power_of_two();
+ let morton_total = morton_dim.saturating_mul(morton_dim);
+ let actual_total = threadgroups_per_row.saturating_mul(threadgroups_per_column);
+ if min_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total) {
+ (true, morton_total, 1)
+ } else {
+ (false, threadgroups_per_row, threadgroups_per_column)
+ }
+ } else {
+ (false, threadgroups_per_row, threadgroups_per_column)
+ };
+ if !is_quant {
+ params.leading_dimension_b = shape.b_leading_dimension.unwrap_or(if shape.b_transpose {
+ k
+ } else {
+ n
+ });
}
+ params.use_morton = use_morton;
+
+ let specialization = GemmSpecialization::from_plan(
+ plan,
+ shape,
+ self.weights_data_type,
+ output_transform,
+ alignment,
+ a_prologue,
+ a_group_size,
+ )?;
+ let kernel = self.get_or_create(encoder.context(), specialization)?;
+ kernel.encode(
+ a_full_precision,
+ weights,
+ &mut *d,
+ scales,
+ biases,
+ zero_points,
+ output_bias,
+ rht_factors,
+ a_int8,
+ a_scales,
+ a_group_sums,
+ std::slice::from_ref(&params),
+ group_count_x,
+ group_count_y,
+ 1,
+ encoder,
+ );
Ok(())
}
- #[allow(clippy::too_many_arguments)]
fn encode_split_k<'a, 'b, WB: BufferArg<'b, Metal>>(
&mut self,
a: MatmulA<'a, Metal>,
@@ -418,21 +337,15 @@ impl GemmKernel {
zero_points: Option<&Allocation<Metal>>,
d: &mut Allocation<Metal>,
ab_scale: f32,
- scale_strides: Option<QuantParamsStrides>,
- zero_point_strides: Option<QuantParamsStrides>,
+ scale_strides: QuantParamsStrides,
+ zero_point_strides: QuantParamsStrides,
shape: MatmulShape,
plan: GemmPlan,
output_transform: GemmDTransform,
output_bias: Option<&Allocation<Metal>>,
- rht_factors: Option<&Allocation<Metal>>,
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
- let MatmulShape {
- m,
- n,
- k,
- ..
- } = shape;
+ let (m, n, k) = (shape.m, shape.n, shape.k);
let (a_full_precision, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match a {
MatmulA::FullPrecision {
values,
@@ -469,26 +382,8 @@ impl GemmKernel {
let elem = (m as usize) * (n as usize);
let slice_bytes = elem * self.output_data_type.size_in_bytes();
let mut temp = encoder.allocate_scratch(split_k as usize * slice_bytes)?;
- let scale_strides = scale_strides.unwrap_or_default();
- let zero_point_strides = zero_point_strides.unwrap_or_default();
-
- let params = GemmParams {
- M: m,
- N: n,
- K: k,
- leading_dimension_a: k,
- leading_dimension_b: k,
- leading_dimension_d: n,
- threadgroups_per_row: base_gx,
- threadgroups_per_column: base_gy,
- aligned_inner_iterations: kp / k_step,
- use_morton: false,
- ab_scale: 1.0,
- scale_output_stride: scale_strides.output_stride,
- scale_group_stride: scale_strides.group_stride,
- zero_point_output_stride: zero_point_strides.output_stride,
- zero_point_group_stride: zero_point_strides.group_stride,
- };
+ let mut params = gemm_params(shape, plan, 1.0, scale_strides, zero_point_strides);
+ params.aligned_inner_iterations = kp / k_step;
let part_kernel = self.get_or_create(encoder.context(), part_spec)?;
part_kernel.encode(
a_full_precision,
@@ -526,11 +421,6 @@ impl GemmKernel {
let reduce = self.get_or_create_split_k_reduce(encoder.context(), reduce_transform)?;
reduce.encode((&temp, 0usize), &mut *d, bias_arg, elem as u32, split_k, group_count, n, scale_arg, encoder);
- if output_transform.contains(GemmDTransform::RHT)
- && let Some(factors) = rht_factors
- {
- self.output_rht.encode_fp_in_place(&mut *d, factors, m, n, encoder);
- }
Ok(())
}
}
@@ -589,12 +479,7 @@ fn gemm_params(
scale_strides: QuantParamsStrides,
zero_point_strides: QuantParamsStrides,
) -> GemmParams {
- let MatmulShape {
- m,
- n,
- k,
- ..
- } = shape;
+ let (m, n, k) = (shape.m, shape.n, shape.k);
let tiling = plan.tiling;
GemmParams {
M: m,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 126f0b975..2845c4bdb 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -1,6 +1,9 @@
use metal::MTLGPUFamily;
-use super::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK};
+use super::{
+ super::MatmulOutputWork,
+ policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK},
+};
use crate::{
backends::{
common::{
@@ -58,7 +61,6 @@ impl GemvSpecialization {
shape.m,
shape.n,
shape.k,
- shape.d_transform,
bf16_io,
)
} else {
@@ -68,11 +70,7 @@ impl GemvSpecialization {
return None;
}
let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);
- if shape.d_transform.contains(GemmDTransform::RHT) {
- Some(policy::DEFAULT_TILE)
- } else {
- Some(policy::fp_tile(gpu_core_count, apple_gpu_family, shape.m, shape.n, shape.k, input_aligned))
- }
+ Some(policy::fp_tile(gpu_core_count, apple_gpu_family, shape.m, shape.n, shape.k, input_aligned))
};
Self::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile?)
}
@@ -96,9 +94,7 @@ impl GemvSpecialization {
if bad_leading_dimension {
return None;
}
- if shape.d_transform.contains(GemmDTransform::RHT) && !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE) {
- return None;
- }
+ let output_transform = output_transform_for_tile(shape, tile)?;
if shape.d_transform.contains(GemmDTransform::ACCUMULATE) && !shape.n.is_multiple_of(32) {
return None;
}
@@ -126,7 +122,7 @@ impl GemvSpecialization {
b_prologue: shape.b_prologue,
group_size: shape.b_group_size.unwrap_or(0),
bits,
- output_transform: shape.d_transform,
+ output_transform,
input_aligned,
k_split: tile.k_split,
output_row_tile: tile.output_row_tile(),
@@ -145,6 +141,10 @@ impl GemvSpecialization {
self.output_row_tile
}
+ pub fn fuses_rht(&self) -> bool {
+ self.output_transform.contains(GemmDTransform::RHT)
+ }
+
fn create_pipeline(
&self,
context: &MetalContext,
@@ -175,6 +175,29 @@ impl GemvSpecialization {
}
}
+fn output_transform_for_tile(
+ shape: &MatmulShape,
+ tile: policy::GemvTile,
+) -> Option<GemmDTransform> {
+ let transform = shape.d_transform;
+ if !transform.contains(GemmDTransform::RHT) {
+ return Some(transform);
+ }
+ if !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE) {
+ return None;
+ }
+
+ let output_rows = tile.output_row_tile();
+ let can_fuse = tile.k_split == 1
+ && output_rows >= HADAMARD_TRANSFORM_BLOCK_SIZE
+ && output_rows.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE);
+ if can_fuse {
+ Some(transform)
+ } else {
+ Some(transform.difference(GemmDTransform::RHT | GemmDTransform::BIAS))
+ }
+}
+
fn full_tile(
shape: &MatmulShape,
tile: policy::GemvTile,
@@ -234,12 +257,19 @@ impl GemvKernel {
&mut self,
arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,
specialization: GemvSpecialization,
+ output_work: &MatmulOutputWork,
encoder: &mut Encoder<Metal>,
) -> Result<(), MatmulError<Metal>> {
let ab_scale = arguments.d_transform.ab_scale;
let output_bias = arguments.d_transform.bias;
let rht_factors = arguments.d_transform.rht_factors;
let soft_cap = arguments.d_transform.soft_cap;
+ let deferred_factors = rht_factors.filter(|_| !specialization.fuses_rht());
+ let (gemv_bias, gemv_rht_factors) = if deferred_factors.is_some() {
+ (None, None)
+ } else {
+ (output_bias, rht_factors)
+ };
let MatmulArguments {
a,
@@ -292,8 +322,8 @@ impl GemvKernel {
biases,
(a, a_offset),
&mut *d,
- output_bias,
- rht_factors,
+ gemv_bias,
+ gemv_rht_factors,
gather_indices,
k,
n,
@@ -308,6 +338,10 @@ impl GemvKernel {
encoder,
);
+ if let Some(factors) = deferred_factors {
+ output_work.apply(&mut *d, factors, output_bias, m, n, encoder);
+ }
+
Ok(())
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
index c8d7e98d2..cc988e389 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
@@ -51,8 +51,6 @@ const fn tile(
}
}
-pub(super) const DEFAULT_TILE: GemvTile = tile(DEFAULT_NUM_SIMDGROUPS, 1, DEFAULT_RESULTS_PER_SIMDGROUP);
-
impl GemvTile {
pub const fn quantized(
num_simdgroups: u32,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
index 49876973f..5420da6c0 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
@@ -1,13 +1,10 @@
use metal::MTLGPUFamily;
use super::{DEFAULT_RESULTS_PER_SIMDGROUP, GemvTile};
-use crate::backends::{common::gpu_types::gemm::GemmDTransform, metal::context::LARGE_MIN_GPU_CORES};
+use crate::backends::metal::context::LARGE_MIN_GPU_CORES;
const QUANT_N_BUCKET_MAXES: [u32; 6] = [512, 2048, 4096, 8192, 16384, 32768];
const QUANT_K_BUCKET_MAXES: [u32; 3] = [512, 2048, 8192];
-const QUANT_RHT_TUNED_N_MIN_EXCLUSIVE: u32 = 2048;
-const QUANT_RHT_TUNED_N_MAX: u32 = 4096;
-const QUANT_RHT_TUNED_K_MIN: u32 = 2048;
/// Lane-sliced tiles split each quantization group across reduction lanes.
/// A lane stages 8 bytes: 16 W4 values or 8 W8 values.
@@ -60,23 +57,11 @@ fn lane_policy(
k: u32,
bits: u32,
group: u32,
- has_rht: bool,
) -> GemvTile {
let is_large_gpu = gpu_core_count >= LARGE_MIN_GPU_CORES;
if m != 1 || bits != 4 {
return lane_default(bits, group);
}
- if has_rht {
- return if is_large_gpu
- && n > QUANT_RHT_TUNED_N_MIN_EXCLUSIVE
- && n <= QUANT_RHT_TUNED_N_MAX
- && k >= QUANT_RHT_TUNED_K_MIN
- {
- lane_tile(4, 8, bits, group)
- } else {
- lane_default(bits, group)
- };
- }
let k_bucket = table_bucket_index(k, &QUANT_K_BUCKET_MAXES);
let n_bucket = table_bucket_index(n, &QUANT_N_BUCKET_MAXES);
@@ -118,7 +103,6 @@ pub fn select(
m: u32,
n: u32,
k: u32,
- d_transform: GemmDTransform,
bf16_io: bool,
) -> Option<GemvTile> {
if !matches!(bits, 4 | 8) || !matches!(group, 16 | 32 | 64 | 128) || !(1..=8).contains(&m) {
@@ -130,19 +114,14 @@ pub fn select(
return (m <= 4 && n >= tile.rows_per_lane()).then_some(tile);
}
- let has_rht = d_transform.contains(GemmDTransform::RHT);
let tile = match m {
- 1..=4 => lane_policy(gpu_core_count, apple_gpu_family, m, n, k, bits, group, has_rht),
+ 1..=4 => lane_policy(gpu_core_count, apple_gpu_family, m, n, k, bits, group),
5..=8 if n <= 64 => lane_default(bits, group),
_ => return None,
};
- let rows = tile.output_row_tile();
// Input rows are tiled independently; partial tiles clamp loads and stores.
if n < tile.rows_per_lane() {
return None;
}
- if has_rht && (!rows.is_multiple_of(32) || !n.is_multiple_of(rows)) {
- return None;
- }
Some(tile)
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
index ca226de60..9857dba89 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
@@ -13,10 +13,10 @@ use self::{
use crate::{
backends::{
common::{
- BufferArg, Encoder,
+ Allocation, BufferArg, Encoder,
gpu_types::gemm::{GemmBPrologueKind, GemmTiling},
kernel::{
- ActivationQuantization,
+ ActivationQuantization, ActivationTransform,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
matmul::{
ActivationFormat, Int8CodeLayout, MatmulArguments, MatmulError, MatmulKernel, MatmulShape,
@@ -32,6 +32,7 @@ use crate::{
pub struct MatmulMetalKernel {
gemv: GemvKernel,
pub gemm: GemmKernel,
+ output_work: MatmulOutputWork,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
@@ -42,6 +43,46 @@ enum MatmulDispatch {
Gemm(GemmPlan),
}
+pub struct MatmulOutputWork {
+ output_rht: ActivationTransform<Metal>,
+ output_rht_with_bias: ActivationTransform<Metal>,
+}
+
+impl MatmulOutputWork {
+ fn new(
+ context: &MetalContext,
+ weights_data_type: DataType,
+ output_data_type: DataType,
+ ) -> Result<Self, MetalError> {
+ Ok(Self {
+ output_rht: ActivationTransform::output_rht(context, output_data_type, None, true)?,
+ output_rht_with_bias: ActivationTransform::output_rht(
+ context,
+ output_data_type,
+ Some(weights_data_type),
+ true,
+ )?,
+ })
+ }
+
+ fn apply(
+ &self,
+ output: &mut Allocation<Metal>,
+ factors: &Allocation<Metal>,
+ bias: Option<&Allocation<Metal>>,
+ m: u32,
+ n: u32,
+ encoder: &mut Encoder<Metal>,
+ ) {
+ let transform = if bias.is_some() {
+ &self.output_rht_with_bias
+ } else {
+ &self.output_rht
+ };
+ transform.encode_fp_in_place(output, factors, bias, m, n, encoder);
+ }
+}
+
fn supports_integer_right_operand(shape: &MatmulShape) -> bool {
matches!((shape.b_bits, shape.signed_codes), (Some(4), _) | (Some(8), true))
}
@@ -152,12 +193,14 @@ impl MatmulKernel for MatmulMetalKernel {
}
}
- let gemm = GemmKernel::new(context, weights_data_type, input_data_type, output_data_type)?;
+ let output_work = MatmulOutputWork::new(context, weights_data_type, output_data_type)?;
+ let gemm = GemmKernel::new(weights_data_type, input_data_type, output_data_type);
let gemv = GemvKernel::new(weights_data_type, input_data_type, output_data_type);
Ok(Self {
gemv,
gemm,
+ output_work,
weights_data_type,
input_data_type,
output_data_type,
@@ -226,7 +269,7 @@ impl MatmulKernel for MatmulMetalKernel {
let shape = MatmulShape::from_arguments(&arguments);
let plan = match self.select_dispatch(&shape, encoder.context()) {
MatmulDispatch::Gemv(gemv) => {
- return self.gemv.encode(arguments, gemv, encoder).map_err(MetalError::from);
+ return self.gemv.encode(arguments, gemv, &self.output_work, encoder).map_err(MetalError::from);
},
MatmulDispatch::Gemm(plan) => plan,
};
@@ -241,6 +284,20 @@ impl MatmulKernel for MatmulMetalKernel {
.into(),
));
}
- self.gemm.encode_plan(arguments, plan, encoder)
+ self.gemm.encode_plan(arguments, plan, &self.output_work, encoder)
+ }
+}
+
+#[cfg(test)]
+impl MatmulMetalKernel {
+ pub fn encode_with_gemm_engine<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(
+ &mut self,
+ arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,
+ engine: gemm::GemmEngine,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<(), MetalError> {
+ let shape = MatmulShape::from_arguments(&arguments);
+ let plan = self.gemm.select_plan_for_engine(&shape, engine, encoder.context())?;
+ self.gemm.encode_plan(arguments, plan, &self.output_work, encoder)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
index c6ddb820a..93788b823 100644
--- a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
@@ -98,7 +98,7 @@ impl<B: Backend> InputRht<B> {
}
let input_dim = self.input_dim();
- self.rht.encode_fp_in_place(&mut input, &self.rht_signs, batch_dim, input_dim, encoder);
+ self.rht.encode_fp_in_place(&mut input, &self.rht_signs, None, batch_dim, input_dim, encoder);
Ok(LinearInput::FullPrecision(input))
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
index 03003245b..68401e24d 100644
--- a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
@@ -112,7 +112,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
input_factors,
)),
Some((
- ActivationTransform::output_rht(context, output_data_type, true)
+ ActivationTransform::output_rht(context, output_data_type, None, true)
.map_err(QLoRALinearWrapperError::BackendError)?,
output_factors,
)),
@@ -242,7 +242,14 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
}
if let Some((output_hadamard_kernel, output_factors)) = &self.output_hadamard {
- output_hadamard_kernel.encode_fp_in_place(&mut output, output_factors, batch_dim, self.output_dim, encoder);
+ output_hadamard_kernel.encode_fp_in_place(
+ &mut output,
+ output_factors,
+ None,
+ batch_dim,
+ self.output_dim,
+ encoder,
+ );
}
encoder.pop_debug_group();
diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
index 4091001c2..6294394c1 100644
--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
@@ -132,7 +132,7 @@ impl<B: Backend> TransformerLayerConv<B> {
let mut output =
self.pre_conv.encode(input, &coefficients, 2 * self.coefficient_count, 0, sequence_length, encoder)?;
if let Some((transform, factors)) = &self.input_rht {
- transform.encode_fp_in_place(&mut output, factors, sequence_length, self.model_dim, encoder);
+ transform.encode_fp_in_place(&mut output, factors, None, sequence_length, self.model_dim, encoder);
}
Ok((output, coefficients))
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
index 3ed476f2d..0c0aad447 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
@@ -32,7 +32,9 @@ fn run<T: ArrayElement + Float + Debug, B: Backend>(
let context = B::Context::new().expect("context");
let kernel = match order {
TransformOrder::Input => ActivationTransform::<B>::input_rht(context.as_ref(), T::data_type(), in_place),
- TransformOrder::Output => ActivationTransform::<B>::output_rht(context.as_ref(), T::data_type(), in_place),
+ TransformOrder::Output => {
+ ActivationTransform::<B>::output_rht(context.as_ref(), T::data_type(), None, in_place)
+ },
}
.expect("activation transform");
@@ -42,7 +44,7 @@ fn run<T: ArrayElement + Float + Debug, B: Backend>(
let batch_count = (data.len() / channel_count) as u32;
let mut encoder = Encoder::new(context.as_ref()).expect("encoder");
if in_place {
- kernel.encode_fp_in_place(&mut input, &factors, batch_count, channel_count as u32, &mut encoder);
+ kernel.encode_fp_in_place(&mut input, &factors, None, batch_count, channel_count as u32, &mut encoder);
} else {
kernel.encode_fp(&input, &mut output, &factors, batch_count, channel_count as u32, &mut encoder);
}
@@ -106,6 +108,40 @@ fn input_and_output_rht_bf16() {
check::<bf16>(0.1);
}
+#[uzu_test]
+fn output_rht_fused_bias_matches_separate_mixed_dtype() {
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().expect("context");
+ let channels = 2 * BLOCK_SIZE as usize;
+ let data: Vec<bf16> = (0..2 * channels).map(|i| bf16::from_f32((i as f32 * 0.17).sin() * 2.0)).collect();
+ let bias_data: Vec<f32> = (0..channels).map(|i| (i as f32 * 0.31).cos() * 0.03).collect();
+ let factors_data: Vec<i32> = (0..channels)
+ .map(|i| {
+ if i % 3 == 0 {
+ -1
+ } else {
+ 1
+ }
+ })
+ .collect();
+ let expected: Vec<_> = run::<bf16, B>(&data, &factors_data, channels, TransformOrder::Output, true)
+ .into_iter()
+ .enumerate()
+ .map(|(i, value)| bf16::from_f32(value.to_f32() + bias_data[i % channels]))
+ .collect();
+ let mut fused = alloc_allocation_with_data::<B, bf16>(context.as_ref(), &data);
+ let bias = alloc_allocation_with_data::<B, f32>(context.as_ref(), &bias_data);
+ let factors = alloc_allocation_with_data::<B, i32>(context.as_ref(), &factors_data);
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");
+ ActivationTransform::output_rht(context.as_ref(), bf16::data_type(), Some(f32::data_type()), true)
+ .expect("fused transform")
+ .encode_fp_in_place(&mut fused, &factors, Some(&bias), 2, channels as u32, &mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ assert_eq!(allocation_to_vec::<B, bf16>(&fused), expected);
+ });
+}
+
mod quantize {
use rand::{RngExt, SeedableRng, rngs::SmallRng};
use uzu_engine_macros::uzu_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs b/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs
index f18c273e7..0949aa88e 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs
@@ -322,7 +322,7 @@ fn test_zero_point_group16_hadamard<T: ArrayElement + Float + Debug + Display>()
.collect();
let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");
- let output_rht = ActivationTransform::<Cpu>::output_rht(context.as_ref(), T::data_type(), false)
+ let output_rht = ActivationTransform::<Cpu>::output_rht(context.as_ref(), T::data_type(), None, false)
.expect("Failed to create ActivationTransform");
let plain_allocation = alloc_allocation_with_data::<Cpu, T>(&context, &plain);
let hadamard_factors_allocation = alloc_allocation_with_data::<Cpu, i32>(&context, &hadamard_factors);
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
index 99caf1b13..c8b0d9c12 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
@@ -200,17 +200,17 @@ fn encode_step(
n: data.n,
k: data.k,
};
- matmul.gemm.encode_with_engine(args, GemmEngine::Mxu, encoder).expect("a8 gemm mxu encode");
+ matmul.encode_with_gemm_engine(args, GemmEngine::Mxu, encoder).expect("a8 gemm mxu encode");
},
BenchPath::Bf16GemmMxu => {
encoder.encode_copy(&data.activations, .., &mut data.a_working, ..);
- hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, data.m, data.k, encoder);
+ hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, None, data.m, data.k, encoder);
let args = data.bf16_arguments(output);
- matmul.gemm.encode_with_engine(args, GemmEngine::Mxu, encoder).expect("bf16 gemm mxu encode");
+ matmul.encode_with_gemm_engine(args, GemmEngine::Mxu, encoder).expect("bf16 gemm mxu encode");
},
BenchPath::Bf16Routed => {
encoder.encode_copy(&data.activations, .., &mut data.a_working, ..);
- hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, data.m, data.k, encoder);
+ hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, None, data.m, data.k, encoder);
let args = data.bf16_arguments(output);
matmul.encode(args, encoder).expect("routed bf16 matmul encode");
},
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/dispatch_paths_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/dispatch_paths_test.rs
index 59f13f8a7..ffb867b61 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/dispatch_paths_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/dispatch_paths_test.rs
@@ -120,6 +120,12 @@ fn rht_parity_bf16() {
let case = Case::new(shape).with_rht(true);
check_case::<bf16>(&context, &mut kernel, Some(engine), case, 1.0);
}
+
+ let split_k_case = Case::new(Shape::new(4, 1024, 32)).with_rht(true).with_bias(true);
+ check_case::<bf16>(&context, &mut kernel, Some(engine), split_k_case, 0.1);
+
+ let non_split_case = Case::new(Shape::new(64, 128, 128)).with_rht(true).with_bias(true);
+ check_case::<bf16>(&context, &mut kernel, Some(engine), non_split_case, 0.1);
}
}
@@ -181,6 +187,10 @@ fn gemv_fp_output_transforms_bf16() {
for case in cases {
check_case::<bf16>(&context, &mut kernel, None, case, 1.0);
}
+
+ // M5 selects the ordinary 8-row, four-slice FP GEMV tile; RHT and bias are deferred.
+ let split_k_rht = Case::new(Shape::new(1, 512, 1024)).with_rht(true).with_bias(true);
+ check_case::<bf16>(&context, &mut kernel, None, split_k_rht, 1.0);
}
#[uzu_test]
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs
index 3d49bc736..66fe353b9 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs
@@ -52,8 +52,7 @@ fn bench_gemm(c: &mut Criterion) {
group.bench_function(BenchmarkId::new("BF16", shape.to_string()), |b| {
iter_encode_loop::<Metal, _>(&context, b, |encoder| {
kernel
- .gemm
- .encode_with_engine(
+ .encode_with_gemm_engine(
MatmulArguments {
a: MatmulA::FullPrecision {
values: &a,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
index 62afbd834..a1c2daf04 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
@@ -211,7 +211,7 @@ fn parity_bf16_gs32_4bit_mlx_with_bias() {
bias: Some(&bias_pp_buf),
..MatmulDOps::none()
};
- matmul.gemm.encode_with_engine(args, GemmEngine::Simdgroup, &mut encoder).expect("encode quant with bias");
+ matmul.encode_with_gemm_engine(args, GemmEngine::Simdgroup, &mut encoder).expect("encode quant with bias");
encoder.end_encoding().submit().wait_until_completed().unwrap();
let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);
@@ -311,10 +311,31 @@ fn parity_gemv_unaligned_width_bf16(
run_parity::<bf16>(m, k, n, gs, bits, method, None, "gemv", 0.05, 0.6);
}
-#[uzu_test]
-fn parity_bf16_gemv_quant_rht_with_bias() {
+#[rstest]
+#[test_attr(uzu_test)]
+#[case::decode_fused(1, 256, 64, 32, QuantizationMethod::ScaleBias, true)]
+#[case::gemv(1, 256, 64, 32, QuantizationMethod::ScaleBias, false)]
+#[case::short_prefill_deferred_rht_then_bias(2, 5120, 6144, 64, QuantizationMethod::ScaleZeroPoint, true)]
+#[case::qmv_short_prefill_m2_g64(2, 5120, 6144, 64, QuantizationMethod::ScaleZeroPoint, false)]
+#[case::short_prefill_m5_g64(5, 5120, 6144, 64, QuantizationMethod::ScaleZeroPoint, false)]
+#[case::short_prefill_m7_g64(7, 5120, 6144, 64, QuantizationMethod::ScaleZeroPoint, false)]
+#[case::gemm_nonsplit_bias(9, 64, 64, 64, QuantizationMethod::ScaleZeroPoint, true)]
+fn parity_bf16_quant_rht(
+ #[case] m: u32,
+ #[case] k: u32,
+ #[case] n: u32,
+ #[case] group_size: u32,
+ #[case] method: QuantizationMethod,
+ #[case] with_bias: bool,
+) {
let context = MetalContext::new().expect("Metal context");
- let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
+ // W4/G64 matches the Qwen short-prefill route; keep the decode anchor at G32.
+ let input = QuantInput::<bf16>::new(m, k, n, group_size, 4, method, 0);
+ let input = if group_size == 64 {
+ input.with_group_output()
+ } else {
+ input
+ };
let rht: Vec<i32> = (0..input.n as usize)
.map(|i| {
if i % 2 == 0 {
@@ -327,71 +348,11 @@ fn parity_bf16_gemv_quant_rht_with_bias() {
let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.3 + 0.05 * (j % 4) as f32).collect();
let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();
- let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");
- let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);
- let cpu_rht = crate::tests::helpers::alloc_allocation_with_data::<Cpu, i32>(&cpu_context, &rht);
- let cpu_bias = crate::tests::helpers::alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, &bias_t);
- let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
- &cpu_context,
- bf16::data_type(),
- bf16::data_type(),
- bf16::data_type(),
- )
- .expect("MatmulCpuKernel");
- let mut cpu_encoder = Encoder::<Cpu>::new(&cpu_context).expect("cpu encoder");
- let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);
- cpu_args.d_transform = MatmulDOps {
- bias: Some(&cpu_bias),
- rht_factors: Some(&cpu_rht),
- ..MatmulDOps::none()
- };
- cpu_matmul.encode(cpu_args, &mut cpu_encoder).expect("cpu encode quant+rht+bias");
- cpu_encoder.end_encoding().submit().wait_until_completed().unwrap();
- let reference = allocation_to_vec::<Cpu, bf16>(&cpu_buffers.y);
-
- let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
- let metal_rht = crate::tests::helpers::alloc_allocation_with_data::<Metal, i32>(&context, &rht);
- let metal_bias = crate::tests::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);
- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
- &context,
- bf16::data_type(),
- bf16::data_type(),
- bf16::data_type(),
- )
- .expect("MatmulMetalKernel");
- let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
- let mut args = quant_arguments(&mut buffers, &input);
- args.d_transform = MatmulDOps {
- bias: Some(&metal_bias),
- rht_factors: Some(&metal_rht),
- ..MatmulDOps::none()
- };
- matmul.encode(args, &mut encoder).expect("encode quant gemv with rht+bias");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
- let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);
-
- assert_parity::<bf16>("gemv_quant_rht_bias", &reference, &actual, 0.05, 0.6);
-}
-
-#[uzu_test]
-fn parity_bf16_gemv_quant_rht() {
- let context = MetalContext::new().expect("Metal context");
- // n % 32 == 0 so the output RHT covers whole 32-element blocks; m = 1 routes to GEMV.
- let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
- let rht: Vec<i32> = (0..input.n as usize)
- .map(|i| {
- if i % 2 == 0 {
- 1
- } else {
- -1
- }
- })
- .collect();
-
// CPU reference applies the RHT through the CPU matmul kernel.
let cpu_context = <Cpu as Backend>::Context::new().expect("Cpu context");
let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);
let cpu_rht = crate::tests::helpers::alloc_allocation_with_data::<Cpu, i32>(&cpu_context, &rht);
+ let cpu_bias = crate::tests::helpers::alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, &bias_t);
let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
&cpu_context,
bf16::data_type(),
@@ -402,6 +363,7 @@ fn parity_bf16_gemv_quant_rht() {
let mut cpu_encoder = Encoder::<Cpu>::new(&cpu_context).expect("cpu encoder");
let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);
cpu_args.d_transform = MatmulDOps {
+ bias: with_bias.then_some(&cpu_bias),
rht_factors: Some(&cpu_rht),
..MatmulDOps::none()
};
@@ -409,9 +371,9 @@ fn parity_bf16_gemv_quant_rht() {
cpu_encoder.end_encoding().submit().wait_until_completed().unwrap();
let reference = allocation_to_vec::<Cpu, bf16>(&cpu_buffers.y);
- // Metal GEMV: m = 1 quant routes to GEMV, RHT selects the 8-simdgroup (32-row) layout.
let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
let metal_rht = crate::tests::helpers::alloc_allocation_with_data::<Metal, i32>(&context, &rht);
+ let metal_bias = crate::tests::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);
let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
&context,
bf16::data_type(),
@@ -422,14 +384,25 @@ fn parity_bf16_gemv_quant_rht() {
let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
let mut args = quant_arguments(&mut buffers, &input);
args.d_transform = MatmulDOps {
+ bias: with_bias.then_some(&metal_bias),
rht_factors: Some(&metal_rht),
..MatmulDOps::none()
};
- matmul.encode(args, &mut encoder).expect("encode quant gemv with rht");
+ matmul.encode(args, &mut encoder).expect("encode quant matmul with rht");
encoder.end_encoding().submit().wait_until_completed().unwrap();
let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);
- assert_parity::<bf16>("gemv_quant_rht", &reference, &actual, 0.05, 0.6);
+ assert_parity::<bf16>(
+ if with_bias {
+ "quant_rht_bias"
+ } else {
+ "quant_rht"
+ },
+ &reference,
+ &actual,
+ 0.05,
+ 0.6,
+ );
}
#[uzu_test]
@@ -525,7 +498,7 @@ fn quant_gemm_parameter_layout_prefix_matches_cpu(
let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
let mut args = quant_arguments(&mut buffers, &input);
args.n = logical_n;
- matmul.gemm.encode_with_engine(args, engine, &mut encoder).unwrap();
+ matmul.encode_with_gemm_engine(args, engine, &mut encoder).unwrap();
encoder.end_encoding().submit().wait_until_completed().unwrap();
let actual = allocation_to_vec::<Metal, bf16>(&buffers.y);
let expected: Vec<_> = reference
@@ -786,8 +759,7 @@ fn a8w_mxu_output_bias_parity_bf16(
..MatmulDOps::none()
};
metal_matmul
- .gemm
- .encode_with_engine(metal_arguments, GemmEngine::Mxu, &mut metal_encoder)
+ .encode_with_gemm_engine(metal_arguments, GemmEngine::Mxu, &mut metal_encoder)
.expect("Metal A8W MXU matmul with output bias");
metal_encoder.end_encoding().submit().wait_until_completed().unwrap();
let actual = allocation_to_vec::<Metal, bf16>(&metal_buffers.y);
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs
index 265599eb8..70ae2fd96 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs
@@ -59,8 +59,7 @@ fn bench_unified_quant_typed<T: ArrayElement + Float>(
group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {
iter_encode_loop::<Metal, _>(context, b, |encoder| {
matmul
- .gemm
- .encode_with_engine(quant_arguments(&mut buffers, &input), engine, encoder)
+ .encode_with_gemm_engine(quant_arguments(&mut buffers, &input), engine, encoder)
.expect("encode unified quant matmul");
});
});
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index 759368e1f..466c1a2bf 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -105,6 +105,26 @@ fn selection_fallbacks_and_split_k_are_preserved() {
use GemmEngine::*;
use GemmTiling::*;
+ for (mut case, expect_split) in [(shape(4, 32, 1024), true), (shape(64, 128, 128), false)] {
+ case.d_transform = GemmDTransform::RHT | GemmDTransform::BIAS;
+ for engine in [Simdgroup, Mxu] {
+ let plan = problem(case, DataType::BF16)
+ .select_plan_for_engine(engine)
+ .expect("parity anchor supports the forced GEMM engine");
+ assert_eq!(plan.split_k > 1, expect_split, "forced {engine:?} RHT+bias parity anchor");
+ }
+ }
+
+ let mut non_split_quant_rht_bias = shape(9, 64, 64);
+ non_split_quant_rht_bias.b_prologue = GemmBPrologueKind::ScaleZeroPointDequant;
+ non_split_quant_rht_bias.b_bits = Some(4);
+ non_split_quant_rht_bias.b_group_size = Some(64);
+ non_split_quant_rht_bias.params_layout = Some(QuantParamsLayout::GroupOutput);
+ non_split_quant_rht_bias.d_transform = GemmDTransform::RHT | GemmDTransform::BIAS;
+ let plan = problem(non_split_quant_rht_bias, DataType::BF16).select_plan();
+ assert_eq!(plan.engine, Simdgroup);
+ assert_eq!(plan.split_k, 1);
+
let mut invalid_layout = quant(shape(64, 4096, 4096));
invalid_layout.b_transpose = false;
assert_eq!(problem(invalid_layout, DataType::BF16).select_plan().engine, Simdgroup);
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
index 9eebb6761..2c158982a 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
@@ -40,36 +40,21 @@ fn fp_policy_cases() {
#[uzu_test]
fn quant_policy_cases() {
let cases = [
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 256, 1536, 4, false, qtile(2, 1)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 262144, 1536, 4, false, qtile(8, 4)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 256, 4, false, qtile(4, 4)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 1, 1536, 256, 4, false, qtile(8, 4)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple8, 1, 2048, 1536, 4, false, qtile(8, 2)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 256, 1536, 4, false, qtile(8, 2)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 1536, 256, 4, false, qtile(4, 8)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 2, 2048, 1536, 4, false, qtile(8, 4)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 5120, 5120, 4, false, qtile(8, 4)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 2048, 1536, 8, false, GemvTile::quantized(8, 4, 1, 32, 4)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 2560, 9216, 4, true, qtile(4, 8)),
+ (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 256, 1536, 4, qtile(2, 1)),
+ (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 262144, 1536, 4, qtile(8, 4)),
+ (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 256, 4, qtile(4, 4)),
+ (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 1, 1536, 256, 4, qtile(8, 4)),
+ (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple8, 1, 2048, 1536, 4, qtile(8, 2)),
+ (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 256, 1536, 4, qtile(8, 2)),
+ (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 1536, 256, 4, qtile(4, 8)),
+ (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 2, 2048, 1536, 4, qtile(8, 4)),
+ (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 5120, 5120, 4, qtile(8, 4)),
+ (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 2048, 1536, 8, GemvTile::quantized(8, 4, 1, 32, 4)),
];
- for (gpu_core_count, apple_gpu_family, m, n, k, bits, has_rht, expected) in cases {
- let actual = quantized_tile(
- gpu_core_count,
- apple_gpu_family,
- bits,
- 32,
- m,
- n,
- k,
- if has_rht {
- GemmDTransform::RHT
- } else {
- GemmDTransform::empty()
- },
- true,
- )
- .expect("quantized route");
+ for (gpu_core_count, apple_gpu_family, m, n, k, bits, expected) in cases {
+ let actual =
+ quantized_tile(gpu_core_count, apple_gpu_family, bits, 32, m, n, k, true).expect("quantized route");
assert_eq!(
actual, expected,
"m={m} n={n} k={k} bits={bits} gpu_core_count={gpu_core_count} apple_gpu_family={apple_gpu_family:?}"
@@ -79,11 +64,8 @@ fn quant_policy_cases() {
#[uzu_test]
fn gpu_core_count_boundary_selects_large_policy() {
- let transform = GemmDTransform::empty();
- let small =
- quantized(SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536, transform).expect("small GPU route");
- let large =
- quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536, transform).expect("large GPU route");
+ let small = quantized(SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536).expect("small GPU route");
+ let large = quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536).expect("large GPU route");
assert_eq!(small, qtile(4, 2));
assert_eq!(large, qtile(2, 1));
@@ -97,70 +79,32 @@ fn quantized(
m: u32,
n: u32,
k: u32,
- d_transform: GemmDTransform,
) -> Option<GemvTile> {
- quantized_tile(gpu_core_count, apple_gpu_family, bits, group, m, n, k, d_transform, true)
+ quantized_tile(gpu_core_count, apple_gpu_family, bits, group, m, n, k, true)
}
#[uzu_test]
fn quantized_policy_edges() {
- for (gpu_core_count, apple_gpu_family, m, group, expected) in [
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 2, 32, GemvTile::quantized(8, 4, 1, 32, 4)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 3, 32, GemvTile::quantized(8, 4, 1, 32, 4)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple8, 4, 64, GemvTile::quantized(8, 4, 1, 32, 8)),
- ] {
- assert_eq!(
- quantized(gpu_core_count, apple_gpu_family, 8, group, m, 4096, 4096, GemmDTransform::RHT),
- Some(expected)
- );
- }
- let none = GemmDTransform::empty();
assert_eq!(
- quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 64, 4, 8192, 4100, none),
+ quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 64, 4, 8192, 4100),
Some(GemvTile::quantized(8, 4, 1, 32, 4))
);
- assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8, 4096, none).is_some());
- assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 1, 8192, 4096, none).is_some());
- assert_eq!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 9, 8192, 4096, none), None);
+ assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8, 4096).is_some());
+ assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 1, 8192, 4096).is_some());
+ assert_eq!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 9, 8192, 4096), None);
assert_eq!(
- quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8192, 4096, none, true),
+ quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8192, 4096, true),
Some(GemvTile::quantized(8, 4, 1, 32, 2))
);
}
#[uzu_test]
fn untuned_quantized_io_uses_only_the_generated_fallback() {
- assert_eq!(
- quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 3, 4096, GemmDTransform::empty(), false,),
- None
- );
- let tile = quantized_tile(
- LARGE_GPU_CORE_COUNT,
- MTLGPUFamily::Apple10,
- 4,
- 32,
- 4,
- 4096,
- 4096,
- GemmDTransform::empty(),
- false,
- )
- .expect("mixed-IO fallback");
+ assert_eq!(quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 3, 4096, false), None);
+ let tile = quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 4096, 4096, false)
+ .expect("mixed-IO fallback");
assert_eq!(tile, GemvTile::quantized(8, 4, 1, 32, 2));
- assert_eq!(
- quantized_tile(
- LARGE_GPU_CORE_COUNT,
- MTLGPUFamily::Apple10,
- 4,
- 32,
- 5,
- 4096,
- 4096,
- GemmDTransform::empty(),
- false,
- ),
- None
- );
+ assert_eq!(quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 5, 4096, 4096, false), None);
}
fn quant_shape(
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
index 8d77a9801..5684be512 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
@@ -17,6 +17,7 @@ use crate::{
};
const FROZEN_PLAN_FINGERPRINT: u64 = 5_839_212_743_558_880_136;
+
const DEVICES: [(&str, &str, u32, MTLGPUFamily, bool); 7] = [
("m1", "Apple M1", 8, MTLGPUFamily::Apple7, false),
("m2", "Apple M2", 10, MTLGPUFamily::Apple8, false),
@@ -167,7 +168,9 @@ fn exact_lookup_rejects_non_matrix_inputs() {
let QmvRoute::Tuned(tile) = selected else {
panic!("test anchor must use a tuned tile");
};
- assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_some());
+ let deferred = GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).unwrap();
+ assert_eq!(deferred.output_row_tile(), 16);
+ assert!(!deferred.fuses_rht());
rht.n -= 1;
assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_none());
@@ -204,6 +207,23 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {
MatmulDispatch::Gemv(actual) if actual == specialization
));
}
+
+ let mut fp = problem(1, 1024, 512, 4, 32, GemmBPrologueKind::ScaleZeroPointDequant);
+ fp.b_prologue = GemmBPrologueKind::FullPrecision;
+ fp.b_bits = None;
+ fp.b_group_size = None;
+ let mut with_rht = fp;
+ with_rht.d_transform = GemmDTransform::RHT;
+ for (cores, family) in [(8, MTLGPUFamily::Apple7), (40, MTLGPUFamily::Apple10)] {
+ let plain =
+ GemvSpecialization::select_shape(&fp, DataType::BF16, DataType::BF16, DataType::BF16, cores, family)
+ .expect("generic M=1 FP GEMV tile");
+ let rht =
+ GemvSpecialization::select_shape(&with_rht, DataType::BF16, DataType::BF16, DataType::BF16, cores, family)
+ .expect("generic M=1 FP RHT GEMV tile");
+ assert!(plain.output_row_tile() < 32);
+ assert_eq!(rht, plain);
+ }
}
#[uzu_test]
diff --git a/crates/uzu-engine/unit/common/matmul/harness.rs b/crates/uzu-engine/unit/common/matmul/harness.rs
index 90900c958..8bcbef52a 100644
--- a/crates/uzu-engine/unit/common/matmul/harness.rs
+++ b/crates/uzu-engine/unit/common/matmul/harness.rs
@@ -205,7 +205,7 @@ pub fn run_metal<T: ArrayElement + Float>(
) -> Vec<T> {
run::<Metal, T>(context, kernel, input, |kernel, args, encoder| {
if let Some(engine) = dispatch {
- kernel.gemm.encode_with_engine(args, engine, encoder).expect("forced GEMM engine encode failed");
+ kernel.encode_with_gemm_engine(args, engine, encoder).expect("forced GEMM engine encode failed");
} else {
kernel.encode(args, encoder).expect("matmul encode failed");
}
diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rs
index 0fbbe3774..987a2dcfa 100644
--- a/crates/uzu-engine/unit/common/matmul/quant.rs
+++ b/crates/uzu-engine/unit/common/matmul/quant.rs
@@ -486,7 +486,7 @@ pub fn run_quant_metal<T: ArrayElement + Float>(
let mut encoder = Encoder::<Metal>::new(context).expect("encoder");
let args = quant_arguments(&mut buffers, input);
if let Some(engine) = dispatch {
- matmul.gemm.encode_with_engine(args, engine, &mut encoder).expect("forced GEMM engine encode failed");
+ matmul.encode_with_gemm_engine(args, engine, &mut encoder).expect("forced GEMM engine encode failed");
} else {
matmul.encode(args, &mut encoder).expect("matmul encode failed");
}