A high-performance inference engine for AI models
Mixed precision matmul (#451)
uuuvndb8613d5754c
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rsindex 152799751..b9cb93d36 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs@@ -9,7 +9,9 @@ pub trait MatmulKernel: Sized {fn new(context: &<Self::Backend as Backend>::Context,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,) -> Result<Self, <Self::Backend as Backend>::Error>;fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Self::Backend>>>(diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rsindex cbafef074..bfe3410e2 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs@@ -6,11 +6,11 @@ use crate::{};#[inline(always)]-unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(- a: SendPtr<T>,- b: SendPtr<T>,+unsafe fn matmul_gemm_compute_row<TA: ArrayElement + Float, TB: ArrayElement + Float, TD: ArrayElement + Float>(+ a: SendPtr<TA>,+ b: SendPtr<TB>,ab_scale: f32,- d: SendPtrMut<T>,+ d: SendPtrMut<TD>,row: usize,n: usize,k: usize,@@ -37,16 +37,16 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(if is_accumulate {acc += (*d_element).to_f32().unwrap();}- *d_element = T::from(acc).unwrap();+ *d_element = TD::from(acc).unwrap();}}}-pub fn matmul_gemm_impl<T: ArrayElement + Float>(- a: *const T,- b: *const T,+pub fn matmul_gemm_impl<TA: ArrayElement + Float, TB: ArrayElement + Float, TD: ArrayElement + Float>(+ a: *const TA,+ b: *const TB,ab_scale: f32,- d: *mut T,+ d: *mut TD,m: usize,n: usize,k: usize,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rsindex 63b3cf00b..f306865e0 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs@@ -11,13 +11,13 @@ use crate::{ArrayElement, backends::common::gpu_types::QuantizationMethod};/// where groups are along the K dimension.////// Computes y[i, j] = sum_l( x[i, l] * dequant(w[j, l]) ) for each batch i and output j.-pub fn qmv<T: ArrayElement + Float>(+pub fn qmv<WeightT: ArrayElement + Float, InputT: ArrayElement + Float, OutputT: ArrayElement + Float>(weights: *const u32,- scales: *const T,+ scales: *const WeightT,zero_points: Option<*const u8>,- biases: Option<*const T>,- input: *const T,- output: *mut T,+ biases: Option<*const WeightT>,+ input: *const InputT,+ output: *mut OutputT,in_vec_size: usize,out_vec_size: usize,batch_size: usize,@@ -90,29 +90,37 @@ pub fn qmv<T: ArrayElement + Float>(acc += val_a * (scale * val_q + bias);}- *output.add(i * out_vec_size + j) = T::from(acc).unwrap();+ *output.add(i * out_vec_size + j) = OutputT::from(acc).unwrap();}}}}#[kernel(QuantizedMatmulQmv)]-#[variants(T, f32, f16, bf16)]+#[variants(WeightT, f32, f16, bf16)]+#[variants(InputT, f32, f16, bf16)]+#[variants(OutputT, f32, f16, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(+pub fn quantized_matmul_qmv<+ WeightT: ArrayElement + Float,+ InputT: ArrayElement + Float,+ OutputT: ArrayElement + Float,+ const GROUP_SIZE: u32,+ const BITS: u32,+>(weights: *const u32,- scales: *const T,+ scales: *const WeightT,#[optional(quant_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,- #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const T>,- input: *const T,- output: *mut T,+ #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const WeightT>,+ input: *const InputT,+ output: *mut OutputT,in_vec_size: u32,out_vec_size: u32,batch_size: u32,#[specialize] quant_method: QuantizationMethod,) {- qmv::<T>(+ qmv::<WeightT, InputT, OutputT>(weights,scales,zero_points,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rsindex 0a42788ff..dee1f9c6a 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs@@ -8,16 +8,24 @@ use crate::{};#[kernel(QuantizedMatmulQmvFast)]-#[variants(T, f32, f16, bf16)]+#[variants(WeightT, f32, f16, bf16)]+#[variants(InputT, f32, f16, bf16)]+#[variants(OutputT, f32, f16, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(+pub fn quantized_matmul_qmv_fast<+ WeightT: ArrayElement + Float,+ InputT: ArrayElement + Float,+ OutputT: ArrayElement + Float,+ const GROUP_SIZE: u32,+ const BITS: u32,+>(weights: *const u32,- scales: *const T,+ scales: *const WeightT,#[optional(quant_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,- #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const T>,- input: *const T,- output: *mut T,+ #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const WeightT>,+ input: *const InputT,+ output: *mut OutputT,#[optional(use_hadamard)] hadamard_factors: Option<*const i32>,in_vec_size: u32,out_vec_size: u32,@@ -28,7 +36,7 @@ pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: u32,if use_hadamard {unimplemented!("not supported yet");}- qmv::<T>(+ qmv::<WeightT, InputT, OutputT>(weights,scales,zero_points,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rsindex 178acc315..85916fa0e 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -1,5 +1,3 @@-use half::{bf16, f16};-use crate::{DataType,backends::{@@ -15,11 +13,14 @@ use crate::{BufferDowncastExt, Cpu, context::CpuContext, error::CpuError, kernel::matmul::quant::encode_quantized_gemm,},},+ dispatch_dtype,utils::pointers::{SendPtr, SendPtrMut},};pub struct MatmulCpuKernel {- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,hadamard: <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel,bias_add: <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel,}@@ -29,19 +30,30 @@ impl MatmulKernel for MatmulCpuKernel {fn new(context: &CpuContext,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,) -> Result<Self, CpuError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());+ for data_type in [weights_data_type, input_data_type, output_data_type] {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());+ }}let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context,- data_type,+ output_data_type,HadamardTransformOrder::Output,)?;- let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, true)?;+ let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(+ context,+ output_data_type,+ weights_data_type,+ true,+ )?;Ok(Self {- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,hadamard,bias_add,})@@ -114,7 +126,9 @@ impl MatmulCpuKernel {n_u});let ldd = n_u;- let data_type = self.data_type;+ let weights_data_type = self.weights_data_type;+ let input_data_type = self.input_data_type;+ let output_data_type = self.output_data_type;let a_buffer_range = a.as_buffer_range_ref();let b_buffer_range = weights.as_buffer_range_ref();let d_buffer_range = d.as_buffer_range_mut();@@ -134,32 +148,13 @@ impl MatmulCpuKernel {SendPtr(unsafe { &*bias_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(bias_range.start))});- command_buffer.push_command(move || match data_type {- DataType::F32 => {- super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f32>(- a_ptr.as_ptr() as *const f32,- b_ptr.as_ptr() as *const f32,- ab_scale,- d_ptr.as_ptr() as *mut f32,- m_u,- n_u,- k_u,- lda,- ldb,- ldd,- b_transpose,- is_accumulate,- );- if let Some(bias) = bias_ptr {- apply_bias::<f32>(d_ptr.as_ptr() as *mut f32, bias.as_ptr() as *const f32, m_u, n_u);- }- },- DataType::F16 => {- super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f16>(- a_ptr.as_ptr() as *const f16,- b_ptr.as_ptr() as *const f16,+ command_buffer.push_command(move || {+ dispatch_dtype!(|(TW: weights_data_type, TA: input_data_type, TD: output_data_type)| {+ super::gemm::shaders::steel_gemm::matmul_gemm_impl::<TA, TW, TD>(+ a_ptr.as_ptr() as *const TA,+ b_ptr.as_ptr() as *const TW,ab_scale,- d_ptr.as_ptr() as *mut f16,+ d_ptr.as_ptr() as *mut TD,m_u,n_u,k_u,@@ -170,29 +165,13 @@ impl MatmulCpuKernel {is_accumulate,);if let Some(bias) = bias_ptr {- apply_bias::<f16>(d_ptr.as_ptr() as *mut f16, bias.as_ptr() as *const f16, m_u, n_u);+ assert_eq!(+ weights_data_type, output_data_type,+ "mixed precision matmul with bias is not supported until bias dtype ownership is settled",+ );+ apply_bias::<TD>(d_ptr.as_ptr() as *mut TD, bias.as_ptr() as *const TD, m_u, n_u);}- },- DataType::BF16 => {- super::gemm::shaders::steel_gemm::matmul_gemm_impl::<bf16>(- a_ptr.as_ptr() as *const bf16,- b_ptr.as_ptr() as *const bf16,- ab_scale,- d_ptr.as_ptr() as *mut bf16,- m_u,- n_u,- k_u,- lda,- ldb,- ldd,- b_transpose,- is_accumulate,- );- if let Some(bias) = bias_ptr {- apply_bias::<bf16>(d_ptr.as_ptr() as *mut bf16, bias.as_ptr() as *const bf16, m_u, n_u);- }- },- _ => unreachable!(),+ })});Ok(())@@ -243,6 +222,14 @@ impl MatmulCpuKernel {let post_bias = arguments.d_transform.bias;if arguments.m >= 5 && arguments.n > 1 {+ assert_eq!(+ self.input_data_type, self.weights_data_type,+ "mixed precision quantized GEMM input dtype is not supported yet",+ );+ assert_eq!(+ self.output_data_type, self.weights_data_type,+ "mixed precision quantized GEMM output dtype is not supported yet",+ );assert!(!(post_bias.is_some() && post_rht.is_some()),"MatmulCpuKernel/Quant GEMM with both output bias and output RHT is not supported: bias must be applied after RHT",@@ -276,7 +263,7 @@ impl MatmulCpuKernel {n,k,},- self.data_type,+ self.weights_data_type,);if let Some(bias) = post_bias {self.bias_add.encode(None::<&Allocation<Cpu>>, bias, &mut *d, n, m * n, encoder);@@ -334,7 +321,9 @@ impl MatmulCpuKernel {let kernel =<<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(context,- self.data_type,+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,group_size,bits,method,@@ -364,7 +353,9 @@ impl MatmulCpuKernel {let kernel =<<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(context,- self.data_type,+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,group_size,bits,method,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rsindex bc37cff7d..9f36dc287 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs@@ -6,11 +6,12 @@ use crate::ArrayElement;#[kernel(TensorAddBias)]#[variants(T, f32, f16, bf16)]-pub fn tensor_add_bias<T: ArrayElement + Float>(+#[variants(BiasT, f32, f16, bf16)]+pub fn tensor_add_bias<T: ArrayElement + Float, BiasT: ArrayElement + Float>(#[allow(unused)]#[optional(!in_place)]input: Option<*const T>,- #[allow(unused)] bias: *const T,+ #[allow(unused)] bias: *const BiasT,#[allow(unused)] output: *mut T,#[allow(unused)] num_cols: u32,#[allow(unused)] length: u32,@@ -21,11 +22,13 @@ pub fn tensor_add_bias<T: ArrayElement + Float>(for i in 0usize..(length as usize) {let bias_position = (i % num_cols as usize);unsafe {- if let Some(in_data) = input {- *output.add(i) = *in_data.add(i) + *bias.add(bias_position);+ let value: f32 = if let Some(in_data) = input {+ num_traits::cast(*in_data.add(i)).unwrap()} else {- *output.add(i) = *output.add(i) + *bias.add(bias_position);- }+ num_traits::cast(*output.add(i)).unwrap()+ };+ let bias_value: f32 = num_traits::cast(*bias.add(bias_position)).unwrap();+ *output.add(i) = num_traits::cast(value + bias_value).unwrap();}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.hindex 609f3bb40..a62acb567 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h@@ -10,7 +10,8 @@ namespace uzu {namespace matmul {template <- typename T,+ typename AT,+ typename BT,ushort SIMDGROUP_BLOCK_M,ushort SIMDGROUP_BLOCK_N,ushort SIMDGROUP_BLOCK_K,@@ -22,8 +23,8 @@ template <bool aligned_k,typename AccumulatorType = float>METAL_FUNC auto gemm_loop(- const device T* left_ptr,- const device T* right_ptr,+ const device AT* left_ptr,+ const device BT* right_ptr,int leading_dimension_a,int leading_dimension_b,int K,@@ -53,10 +54,10 @@ METAL_FUNC auto gemm_loop(METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < BLOCK_K; inner_k += SIMDGROUP_BLOCK_K) {- Fragment<T, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(thread_context);- Fragment<T, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(thread_context);@@ -115,10 +116,10 @@ METAL_FUNC auto gemm_loop(METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < remaining_k; inner_k += SIMDGROUP_BLOCK_K) {- Fragment<T, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(thread_context);- Fragment<T, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(thread_context);diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.hindex c07e65a5b..9b64d6989 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h@@ -13,8 +13,9 @@ namespace matmul {///////////////////////////////////////////////////////////////////////////////template <- typename T,- typename U,+ typename AT,+ typename BT,+ typename DT,int BLOCK_ROWS,int BLOCK_COLS,int BLOCK_DEPTH,@@ -25,7 +26,7 @@ template <ushort THREADGROUP_LEADING_DIMENSION_A,ushort THREADGROUP_LEADING_DIMENSION_B,typename AccumulatorType = float,- typename Epilogue = TransformNone<U, AccumulatorType>>+ typename Epilogue = TransformNone<DT, AccumulatorType>>struct ThreadgroupTile {METAL_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<@@ -107,8 +108,8 @@ struct ThreadgroupTile {}METAL_FUNC void multiply_accumulate(- const threadgroup T* a_shared,- const threadgroup T* b_shared+ const threadgroup AT* a_shared,+ const threadgroup BT* b_shared) {a_shared += a_shared_offset;b_shared += b_shared_offset;@@ -119,7 +120,7 @@ struct ThreadgroupTile {simdgroup_barrier(mem_flags::mem_none);a_fragment.template load<- T,+ AT,SIMDGROUPS_PER_ROW,1,A_STRIDE_ROW,@@ -128,7 +129,7 @@ struct ThreadgroupTile {simdgroup_barrier(mem_flags::mem_none);b_fragment.template load<- T,+ BT,1,SIMDGROUPS_PER_COLUMN,B_STRIDE_INNER,@@ -143,7 +144,7 @@ struct ThreadgroupTile {}}- METAL_FUNC void store_result(device U* D, const int leading_dimension_d) {+ METAL_FUNC void store_result(device DT* D, const int leading_dimension_d) {METAL_PRAGMA_UNROLLfor (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);@@ -151,14 +152,14 @@ struct ThreadgroupTile {D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;- c_fragment.template store<U, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(+ c_fragment.template store<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(D,leading_dimension_d);}METAL_FUNC void store_result_safe(- device U* D,+ device DT* D,const int leading_dimension_d,short2 destination_tile_dimensions) {@@ -176,7 +177,7 @@ struct ThreadgroupTile {return;c_fragment- .template store_safe<U, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(+ .template store_safe<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(D,leading_dimension_d,destination_tile_dimensions@@ -185,13 +186,14 @@ struct ThreadgroupTile {template <typename EpilogueOp>METAL_FUNC void apply_epilogue(- const device U* C,+ const device DT* C,const int leading_dimension_c,const int column_stride_c,thread const EpilogueOp& epilogue_operation) {- const device U* c_pointer = C + simdgroup_row_offset * leading_dimension_c +- simdgroup_col_offset * column_stride_c;+ const device DT* c_pointer = C ++ simdgroup_row_offset * leading_dimension_c ++ simdgroup_col_offset * column_stride_c;METAL_PRAGMA_UNROLLfor (ushort i = 0; i < TILE_ROWS; i++) {@@ -206,7 +208,7 @@ struct ThreadgroupTile {i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;const ushort col_offset =j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;- U c_value = c_pointer+ DT c_value = c_pointer[row_offset * leading_dimension_c + col_offset * column_stride_c +k];block_data[k] = epilogue_operation.apply(@@ -220,14 +222,15 @@ struct ThreadgroupTile {template <typename EpilogueOp>METAL_FUNC void apply_epilogue_safe(- const device U* C,+ const device DT* C,const int leading_dimension_c,const int column_stride_c,short2 tile_dimensions,thread const EpilogueOp& epilogue_operation) {- const device U* c_pointer = C + simdgroup_row_offset * leading_dimension_c +- simdgroup_col_offset * column_stride_c;+ const device DT* c_pointer = C ++ simdgroup_row_offset * leading_dimension_c ++ simdgroup_col_offset * column_stride_c;tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);METAL_PRAGMA_UNROLL@@ -244,7 +247,7 @@ struct ThreadgroupTile {k++) {if (row_offset < tile_dimensions.y &&col_offset + k < tile_dimensions.x) {- U c_value = c_pointer+ DT c_value = c_pointer[row_offset * leading_dimension_c +col_offset * column_stride_c + k];block_data[k] = epilogue_operation.apply(@@ -257,8 +260,8 @@ struct ThreadgroupTile {}}- METAL_FUNC void apply_bias(const device U* bias) {- const device U* bias_pointer = bias + simdgroup_col_offset;+ METAL_FUNC void apply_bias(const device BT* bias) {+ const device BT* bias_pointer = bias + simdgroup_col_offset;METAL_PRAGMA_UNROLLfor (ushort i = 0; i < TILE_ROWS; i++) {@@ -279,10 +282,10 @@ struct ThreadgroupTile {}METAL_FUNC void apply_bias_safe(- const device U* bias,+ const device BT* bias,short2 tile_dimensions) {- const device U* bias_pointer = bias + simdgroup_col_offset;+ const device BT* bias_pointer = bias + simdgroup_col_offset;tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);METAL_PRAGMA_UNROLLdiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex b3dcb051c..cca0830a2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -15,7 +15,12 @@ using namespace metal;namespace uzu {namespace gemm {-template <typename T, GemmTiling GEMM_TILING, bool TRANSPOSE_B>+template <+ typename AT,+ typename BT,+ typename DT,+ GemmTiling GEMM_TILING,+ bool TRANSPOSE_B>struct MxuMmaCore {METAL_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);METAL_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);@@ -37,13 +42,13 @@ struct MxuMmaCore {using AccumulatorType = float;static METAL_FUNC void run(- const device T* a,- const device T* b,- device T* d,+ const device AT* a,+ const device BT* b,+ device DT* d,const constant uzu::matmul::GemmParams* params,GemmAlignment alignment,GemmDTransform output_transform,- const device T* output_bias,+ const device BT* output_bias,const thread ThreadContext& thread_context) {(void)output_bias;@@ -58,8 +63,8 @@ struct MxuMmaCore {const size_t block_row = size_t(geometry.block_row_start);const size_t block_col = size_t(geometry.block_col_start);- const device T* a_block = a + block_row * params->leading_dimension_a;- const device T* b_block =+ const device AT* a_block = a + block_row * params->leading_dimension_a;+ const device BT* b_block =b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);const ushort tile_row_offset =@@ -69,7 +74,7 @@ struct MxuMmaCore {SIMDGROUP_BLOCK_N *(thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);- device T* d_simdgroup =+ device DT* d_simdgroup =d + block_row * params->leading_dimension_d + block_col +tile_row_offset * params->leading_dimension_d + tile_col_offset;@@ -90,9 +95,9 @@ struct MxuMmaCore {int(geometry.block_col_start + tile_col_offset)));- const device T* a_simdgroup =+ const device AT* a_simdgroup =a_block + size_t(tile_row_offset) * params->leading_dimension_a;- const device T* b_simdgroup =+ const device BT* b_simdgroup =b_block + (TRANSPOSE_B ? size_t(tile_col_offset) *int(params->leading_dimension_b): size_t(tile_col_offset));@@ -113,7 +118,8 @@ struct MxuMmaCore {(simdgroup_limit_n == SIMDGROUP_BLOCK_N),[&](auto aligned_n) {auto accumulator_tile = uzu::matmul::gemm_loop<- T,+ AT,+ BT,SIMDGROUP_BLOCK_M,SIMDGROUP_BLOCK_N,SIMDGROUP_BLOCK_K,@@ -147,7 +153,7 @@ struct MxuMmaCore {if (apply_accumulate) {uzu::matmul::Fragment<- T,+ DT,TILES_M,TILES_N,uzu::matmul::MxuFragmentOps>diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex c860bc745..4f4626f2c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -19,7 +19,9 @@ namespace uzu {namespace gemm {template <- typename T,+ typename AT,+ typename BT,+ typename DT,GemmTiling GEMM_TILING,bool TRANSPOSE_B,GemmWeightPrologueKind WEIGHT_PROLOGUE =@@ -34,8 +36,8 @@ struct SimdgroupMmaCore {gemm_tiling_simdgroups_per_row(GEMM_TILING);METAL_CONST int SIMDGROUPS_PER_COLUMN =gemm_tiling_simdgroups_per_column(GEMM_TILING);- METAL_CONST ushort PADDING_A = 16 / sizeof(T);- METAL_CONST ushort PADDING_B = 16 / sizeof(T);+ METAL_CONST ushort PADDING_A = 16 / sizeof(AT);+ METAL_CONST ushort PADDING_B = 16 / sizeof(BT);METAL_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;METAL_CONST ushort SHARED_STRIDE_B =(TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;@@ -43,21 +45,21 @@ struct SimdgroupMmaCore {SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;using ALoader = uzu::matmul::ThreadgroupLoader<- T,+ AT,THREADGROUP_BLOCK_M,THREADGROUP_BLOCK_K,SHARED_STRIDE_A,true,THREADGROUP_THREADS>;using BLoaderFp = uzu::matmul::ThreadgroupLoader<- T,+ BT,TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N,SHARED_STRIDE_B,TRANSPOSE_B,THREADGROUP_THREADS>;using BLoaderScaleBias = QuantizedBlockLoaderScaleBias<- T,+ BT,THREADGROUP_BLOCK_N,THREADGROUP_BLOCK_K,SHARED_STRIDE_B,@@ -66,7 +68,7 @@ struct SimdgroupMmaCore {GROUP_SIZE,BITS>;using BLoaderScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<- T,+ BT,THREADGROUP_BLOCK_N,THREADGROUP_BLOCK_K,SHARED_STRIDE_B,@@ -75,8 +77,9 @@ struct SimdgroupMmaCore {GROUP_SIZE,BITS>;using TileAccumulator = uzu::matmul::ThreadgroupTile<- T,- T,+ AT,+ BT,+ DT,THREADGROUP_BLOCK_M,THREADGROUP_BLOCK_N,THREADGROUP_BLOCK_K,@@ -87,12 +90,12 @@ struct SimdgroupMmaCore {SHARED_STRIDE_A,SHARED_STRIDE_B,float,- uzu::matmul::TransformNone<T, float>>;+ uzu::matmul::TransformNone<DT, float>>;template <uint GEMM_ALIGNMENT_RAW, typename BLoader>static METAL_FUNC void k_loop(- threadgroup T* a_shared,- threadgroup T* b_shared,+ threadgroup AT* a_shared,+ threadgroup BT* b_shared,const int aligned_k_iterations,thread ALoader& loader_a,thread BLoader& loader_b,@@ -147,14 +150,14 @@ struct SimdgroupMmaCore {template <uint GEMM_ALIGNMENT_RAW>static METAL_FUNC void finalize(thread TileAccumulator& accumulator,- device T* d,+ device DT* d,const constant uzu::matmul::GemmParams* params,const thread ushort& tile_block_rows,const thread ushort& tile_block_cols,const bool needs_epilogue,const thread uzu::matmul::TransformScaleAccumulate<float, float>&epilogue,- const device T* bias_block,+ const device BT* bias_block,const bool needs_bias) {constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};@@ -194,18 +197,18 @@ struct SimdgroupMmaCore {}static METAL_FUNC void run(- const device T* a,+ const device AT* a,const device uint8_t* b_packed,- device T* d,+ device DT* d,const constant uzu::matmul::GemmParams* params,GemmAlignment alignment,GemmDTransform output_transform,- const device T* scales,- const device T* biases,+ const device BT* scales,+ const device BT* biases,const device uint8_t* zero_points,- const device T* output_bias,- threadgroup T* a_shared,- threadgroup T* b_shared,+ const device BT* output_bias,+ threadgroup AT* a_shared,+ threadgroup BT* b_shared,const thread ThreadContext& thread_context) {const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);@@ -247,7 +250,7 @@ struct SimdgroupMmaCore {const float alpha = needs_scale ? params->ab_scale : 1.0f;const float beta = needs_accumulate ? 1.0f : 0.0f;uzu::matmul::TransformScaleAccumulate<float, float> epilogue(alpha, beta);- const device T* bias_block = output_bias + block_col;+ const device BT* bias_block = output_bias + block_col;const bool all_aligned = ((alignment.contains(GemmAlignment::M)) ||(tile_block_rows == THREADGROUP_BLOCK_M)) &&@@ -277,7 +280,7 @@ struct SimdgroupMmaCore {};if constexpr (WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision) {- const device T* b = reinterpret_cast<const device T*>(b_packed);+ const device BT* b = reinterpret_cast<const device BT*>(b_packed);b += TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col;thread BLoaderFploader_b(b, params->leading_dimension_b, b_shared, thread_context);@@ -318,8 +321,8 @@ struct SimdgroupMmaCore {const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;const device uint8_t* weights_block =b_packed + block_col * weights_row_stride_bytes;- const device T* scales_offset = scales + block_col * groups_per_row;- const device T* biases_offset = biases + block_col * groups_per_row;+ const device BT* scales_offset = scales + block_col * groups_per_row;+ const device BT* biases_offset = biases + block_col * groups_per_row;thread BLoaderScaleBias loader_b(weights_block,scales_offset,@@ -368,7 +371,7 @@ struct SimdgroupMmaCore {const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;const device uint8_t* weights_block =b_packed + block_col * weights_row_stride_bytes;- const device T* scales_offset = scales + block_col * groups_per_row;+ const device BT* scales_offset = scales + block_col * groups_per_row;const int zero_point_stride_per_row =(BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;const device uint8_t* zero_points_row_start =diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 3cffd4945..b8c22eab5 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -13,21 +13,25 @@ using namespace uzu::gemm;#define GEMM_TGA_ELEMENTS \(USE_MXU ? 1 \: (gemm_tiling_block_m(GEMM_TILING) * \- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(T)))))+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))#define GEMM_TGB_ELEMENTS \(USE_MXU ? 1 \: (gemm_tiling_block_n(GEMM_TILING) * \- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(T)))))+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))template <- typename T,+ typename AT,+ typename BT,+ typename DT,GemmTiling GEMM_TILING,bool TRANSPOSE_B,bool USE_MXU,GemmWeightPrologueKind WEIGHT_PROLOGUE,uint BITS,uint GROUP_SIZE>-VARIANTS(T, half, bfloat, float)+VARIANTS(AT, half, bfloat, float)+VARIANTS(BT, half, bfloat, float)+VARIANTS(DT, half, bfloat, float)VARIANTS(GEMM_TILING,GemmTiling::T8x32x32_1x1,@@ -71,24 +75,24 @@ CONSTRAINT(GEMM_TILING != GemmTiling::T64x64x64_2x2 ||GROUP_SIZE == 64 || GROUP_SIZE == 128)KERNEL(Gemm)(- const device T* a,+ const device AT* a,const device uint8_t* b_packed,- device T* d,- const device T* scales+ device DT* d,+ const device BT* scalesOPTIONAL(WEIGHT_PROLOGUE != GemmWeightPrologueKind::FullPrecision),- const device T* biases+ const device BT* biasesOPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleBiasDequant),const device uint8_t* zero_pointsOPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleZeroPointDequant),- const device T* output_bias+ const device BT* output_biasOPTIONAL(output_transform.contains(GemmDTransform::BIAS)),const constant uzu::matmul::GemmParams* params,const constant uint& group_count_x,const constant uint& group_count_y,const GemmDTransform output_transform SPECIALIZE,const GemmAlignment alignment SPECIALIZE,- threadgroup T a_shared[GEMM_TGA_ELEMENTS],- threadgroup T b_shared[GEMM_TGB_ELEMENTS],+ threadgroup AT a_shared[GEMM_TGA_ELEMENTS],+ threadgroup BT b_shared[GEMM_TGB_ELEMENTS],const uint group_x GROUPS(group_count_x),const uint group_y GROUPS(group_count_y),const uint thread_x THREADS(METAL_SIMD_SIZE),@@ -106,8 +110,8 @@ KERNEL(Gemm)((void)scales;(void)biases;(void)zero_points;- const device T* b = reinterpret_cast<const device T*>(b_packed);- MxuMmaCore<T, GEMM_TILING, TRANSPOSE_B>::run(+ const device BT* b = reinterpret_cast<const device BT*>(b_packed);+ MxuMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B>::run(a,b,d,@@ -119,7 +123,9 @@ KERNEL(Gemm)();} else {SimdgroupMmaCore<- T,+ AT,+ BT,+ DT,GEMM_TILING,TRANSPOSE_B,WEIGHT_PROLOGUE,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 155d38f64..6be34454c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -32,7 +32,9 @@ pub enum GemmDispatchPath {}pub struct GemmKernel {- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,kernels: HashMap<GemmSpecialization, GemmMetalKernel>,pub bias_add: TensorAddBiasMetalKernel,pub hadamard: <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel,@@ -41,21 +43,25 @@ pub struct GemmKernel {impl GemmKernel {pub(crate) fn new(context: &MetalContext,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,) -> Result<Self, MetalError> {- let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true)?;+ let bias_add = TensorAddBiasMetalKernel::new(context, output_data_type, weights_data_type, true)?;let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context,- data_type,+ output_data_type,HadamardTransformOrder::Output,)?;let mut kernel = Self {- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,kernels: HashMap::new(),bias_add,hadamard,};- for specialization in GemmSpecialization::precompile_configs(data_type) {+ for specialization in GemmSpecialization::precompile_configs(weights_data_type) {kernel.get_or_create(context, specialization)?;}Ok(kernel)@@ -66,7 +72,7 @@ impl GemmKernel {context: &MetalContext,combo: MatmulQuantCombo,) -> Result<(), MetalError> {- for specialization in GemmSpecialization::quant_combo_specs(self.data_type, combo) {+ for specialization in GemmSpecialization::quant_combo_specs(self.weights_data_type, combo) {self.get_or_create(context, specialization)?;}Ok(())@@ -82,7 +88,9 @@ impl GemmKernel {Entry::Vacant(entry) => {let kernel = GemmMetalKernel::new(context,- specialization.data_type,+ self.input_data_type,+ self.weights_data_type,+ self.output_data_type,specialization.tiling,specialization.transpose_b,specialization.use_mxu,@@ -103,7 +111,9 @@ impl GemmKernel {encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {let path = if encoder.context().device.supports_mxu()- && matches!(self.data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ && [self.weights_data_type, self.input_data_type, self.output_data_type]+ .into_iter()+ .all(|data_type| matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32))&& matches!(arguments.b, MatmulB::FullPrecision { .. }){GemmDispatchPath::Mxu@@ -126,9 +136,13 @@ impl GemmKernel {"GemmDispatchPath::Mxu requested on hardware without MXU support",);assert!(- matches!(self.data_type, DataType::F16 | DataType::BF16 | DataType::F32),- "GemmDispatchPath::Mxu requires F16, BF16, or F32 data type, got {:?}",- self.data_type,+ [self.weights_data_type, self.input_data_type, self.output_data_type]+ .into_iter()+ .all(|data_type| matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)),+ "GemmDispatchPath::Mxu requires F16, BF16, or F32 data types, got weights {:?}, input {:?}, output {:?}",+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,);assert!(matches!(arguments.b, MatmulB::FullPrecision { .. }),@@ -254,7 +268,7 @@ impl GemmKernel {};let specialization = GemmSpecialization {- data_type: self.data_type,+ weights_data_type: self.weights_data_type,tiling,use_mxu,output_transform,@@ -310,7 +324,7 @@ impl GemmKernel {let group_count_y = m.div_ceil(tiling.block_m());let specialization = GemmSpecialization {- data_type: self.data_type,+ weights_data_type: self.weights_data_type,tiling,use_mxu,output_transform,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 43e2b80ff..04985b10f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -12,7 +12,7 @@ use crate::{#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub(crate) struct GemmSpecialization {- pub(crate) data_type: DataType,+ pub(crate) weights_data_type: DataType,pub(crate) tiling: GemmTiling,pub(crate) use_mxu: bool,pub(crate) output_transform: GemmDTransform,@@ -44,13 +44,13 @@ impl GemmSpecialization {Ok(())}- pub(crate) fn precompile_configs(data_type: DataType) -> Vec<Self> {+ pub(crate) fn precompile_configs(weights_data_type: DataType) -> Vec<Self> {let mut out = Vec::new();- for &tiling in simdgroup_tiling_set(data_type) {+ for &tiling in simdgroup_tiling_set(weights_data_type) {for align_mn in [true, false] {for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {out.push(Self {- data_type,+ weights_data_type,tiling,use_mxu: false,output_transform,@@ -64,7 +64,7 @@ impl GemmSpecialization {}}- for &tiling in mxu_tiling_set(data_type) {+ for &tiling in mxu_tiling_set(weights_data_type) {for align_m in [true, false] {for align_n in [true, false] {for align_k in [true, false] {@@ -75,7 +75,7 @@ impl GemmSpecialization {GemmDTransform::SCALE | GemmDTransform::ACCUMULATE,] {out.push(Self {- data_type,+ weights_data_type,tiling,use_mxu: true,output_transform,@@ -94,7 +94,7 @@ impl GemmSpecialization {}pub(crate) fn quant_combo_specs(- data_type: DataType,+ weights_data_type: DataType,combo: MatmulQuantCombo,) -> Vec<Self> {let bits = DataType::from(combo.mode).size_in_bits() as u32;@@ -104,14 +104,14 @@ impl GemmSpecialization {QuantizationMethod::ScaleZeroPoint => GemmWeightPrologueKind::ScaleZeroPointDequant,};let mut out = Vec::new();- for &tiling in quant_tiling_set(data_type) {+ for &tiling in quant_tiling_set(weights_data_type) {if tiling.block_k() > group_size {continue;}for align_n in [true, false] {for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {out.push(Self {- data_type,+ weights_data_type,tiling,use_mxu: false,output_transform,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metalindex 91362ea91..bbf14ffbc 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -13,13 +13,15 @@ using namespace metal;// → 16 * (16 + 4) = 320. Unused when tg_simd_cols == 1.#define GEMV_MAX_THREADGROUP_MEMORY 320-template <typename T>-VARIANTS(T, half, bfloat, float)+template <typename WeightT, typename InputT, typename OutputT>+VARIANTS(WeightT, half, bfloat, float)+VARIANTS(InputT, half, bfloat, float)+VARIANTS(OutputT, half, bfloat, float)KERNEL(MatmulGemv)(- const device T* matrix,- const device T* input_vector,- const device T* output_bias OPTIONAL(is_bias),- device T* output_vector,+ const device WeightT* matrix,+ const device InputT* input_vector,+ const device WeightT* output_bias OPTIONAL(is_bias),+ device OutputT* output_vector,const constant uint& input_dimension,const constant uint& output_dimension,const constant uint& matrix_leading_dimension,@@ -63,7 +65,7 @@ KERNEL(MatmulGemv)(// Thread local accumulation resultsthread float accumulated_values[4] = {0.0f, 0.0f, 0.0f, 0.0f};- thread T matrix_values[4];+ thread WeightT matrix_values[4];thread float vector_coefficients[4];const uint thread_row_in_simdgroup =@@ -112,7 +114,7 @@ KERNEL(MatmulGemv)(? output_row_start: output_dimension - uint(thread_out_rows);- const device T* thread_matrix =+ const device WeightT* thread_matrix =matrix + output_row_start * matrix_leading_dimension;const uniform<uint> input_block_stride =@@ -128,7 +130,7 @@ KERNEL(MatmulGemv)(++input_block_index) {// Load vector coefficients (unchecked){- const device T* input_vector_row =+ const device InputT* input_vector_row =input_vector + batch_row * input_dimension;METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;@@ -170,7 +172,7 @@ KERNEL(MatmulGemv)(if (remaining_input_columns > 0) {// Load vector coefficients (checked){- const device T* input_vector_row =+ const device InputT* input_vector_row =input_vector + batch_row * input_dimension;if (input_block_col_offset + uint(thread_out_cols) <=input_vector_length) {@@ -221,7 +223,7 @@ KERNEL(MatmulGemv)(? thread_matrix[output_row_offset * matrix_leading_dimension +input_block_col_offset + input_col_offset]- : T(0);+ : WeightT(0);}}@@ -292,20 +294,25 @@ KERNEL(MatmulGemv)(// Write outputsif (simdgroup_col_thread_base == 0 && thread_col_in_simdgroup == 0) {- device T* output_row_values = output_vector + batch_row * output_dimension;+ device OutputT* output_row_values =+ output_vector + batch_row * output_dimension;METAL_PRAGMA_UNROLLfor (uint output_row_offset = 0; output_row_offset < thread_out_rows;output_row_offset++) {- T accumulated_c = static_cast<T>(output_scale) *- static_cast<T>(accumulated_values[output_row_offset]);+ float accumulated_c =+ output_scale * accumulated_values[output_row_offset];if (is_accumulate) {- accumulated_c +=- output_row_values[output_row_start + output_row_offset];+ accumulated_c += static_cast<float>(+ output_row_values[output_row_start + output_row_offset]+ );}if (is_bias) {- accumulated_c += output_bias[output_row_start + output_row_offset];+ accumulated_c += static_cast<float>(+ output_bias[output_row_start + output_row_offset]+ );}- output_row_values[output_row_start + output_row_offset] = accumulated_c;+ output_row_values[output_row_start + output_row_offset] =+ static_cast<OutputT>(accumulated_c);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 03d67e584..100a59aef 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -10,20 +10,26 @@ use crate::{};pub(crate) struct GemvKernel {- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,pipelines: HashMap<GemvSpecialization, MatmulGemvMetalKernel>,}impl GemvKernel {pub(crate) fn new(context: &MetalContext,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,) -> Result<Self, MatmulError<Metal>> {let mut kernel = Self {- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,pipelines: HashMap::new(),};- for &config in GemvSpecialization::precompile_configs(data_type) {+ for &config in GemvSpecialization::precompile_configs(weights_data_type) {kernel.get_or_create(context, config)?;}Ok(kernel)@@ -39,7 +45,9 @@ impl GemvKernel {Entry::Vacant(entry) => {let kernel = MatmulGemvMetalKernel::new(context,- self.data_type,+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,specialization.threadgroup_rows,specialization.threadgroup_cols,specialization.threads_per_simdgroup_row,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metalindex e376972a9..c1da8180f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal@@ -7,17 +7,24 @@using namespace uzu::quantization_method;using namespace uzu::gemm;-template <typename T, uint GROUP_SIZE, uint BITS>-VARIANTS(T, float, half, bfloat)+template <+ typename WeightT,+ typename InputT,+ typename OutputT,+ uint GROUP_SIZE,+ uint BITS>+VARIANTS(WeightT, float, half, bfloat)+VARIANTS(InputT, float, half, bfloat)+VARIANTS(OutputT, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)PUBLIC KERNEL(QuantizedMatmulQmv)(const device uint32_t* weights,- const device T* scales,+ const device WeightT* scales,const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),- const device T* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),- const device T* input,- device T* output,+ const device WeightT* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),+ const device InputT* input,+ device OutputT* output,const constant uint& in_vec_size,const constant uint& out_vec_size,const constant uint& batch_size,@@ -44,7 +51,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;const uint in_vec_size_g = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;- const device T* scales_base = scales;+ const device WeightT* scales_base = scales;const uint out_row =out_block_idx * (num_simdgroups * results_per_simdgroup) +simd_group * results_per_simdgroup;@@ -62,7 +69,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(const uint zp_stride =BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;const device uint8_t* zps_row_base = nullptr;- const device T* biases_row_base = nullptr;+ const device WeightT* biases_row_base = nullptr;if (quant_method == QuantizationMethod::ScaleBias) {biases_row_base = biases + out_row * in_vec_size_g;} else {@@ -74,19 +81,19 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(uint k = 0;for (; k + block_size <= in_vec_size; k += block_size) {- U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);+ U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);for (uint row = 0;out_row + row < out_vec_size && row < results_per_simdgroup;row++) {auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);const uint row_idx = out_row + row;- const device T* sr = scales_base + row_idx * in_vec_size_g;+ const device WeightT* sr = scales_base + row_idx * in_vec_size_g;uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;U s = static_cast<U>(sr[g]);if (quant_method == QuantizationMethod::ScaleBias) {- const device T* bl = biases_row_base + row * in_vec_size_g;+ const device WeightT* bl = biases_row_base + row * in_vec_size_g;U b = static_cast<U>(bl[g]);result[row] +=qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);@@ -114,7 +121,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(? min(in_vec_size - k - thread_offset, values_per_thread): 0;if (remaining > 0) {- U sum = load_vector_safe<T, U, values_per_thread, BITS>(+ U sum = load_vector_safe<InputT, U, values_per_thread, BITS>(input,x_thread,remaining@@ -125,12 +132,12 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(row++) {auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);const uint row_idx = out_row + row;- const device T* sr = scales_base + row_idx * in_vec_size_g;+ const device WeightT* sr = scales_base + row_idx * in_vec_size_g;uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;U s = static_cast<U>(sr[g]);if (quant_method == QuantizationMethod::ScaleBias) {- const device T* bl = biases_row_base + row * in_vec_size_g;+ const device WeightT* bl = biases_row_base + row * in_vec_size_g;U b = static_cast<U>(bl[g]);result[row] +=qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);@@ -154,7 +161,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(row++) {result[row] = simd_sum(result[row]);if (simd_lane == 0) {- output[row] = static_cast<T>(result[row]);+ output[row] = static_cast<OutputT>(result[row]);}}} else {@@ -165,7 +172,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(const uint zp_stride =BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;const device uint8_t* zps_row_base = nullptr;- const device T* biases_row_base = nullptr;+ const device WeightT* biases_row_base = nullptr;if (quant_method == QuantizationMethod::ScaleBias) {biases_row_base = biases + used_out_row * in_vec_size_g;} else {@@ -177,17 +184,17 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(uint k = 0;for (; k + block_size <= in_vec_size; k += block_size) {- U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);+ U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);for (uint row = 0; row < results_per_simdgroup; row++) {auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);const uint row_idx = used_out_row + row;- const device T* sr = scales_base + row_idx * in_vec_size_g;+ const device WeightT* sr = scales_base + row_idx * in_vec_size_g;uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;U s = static_cast<U>(sr[g]);if (quant_method == QuantizationMethod::ScaleBias) {- const device T* bl = biases_row_base + row * in_vec_size_g;+ const device WeightT* bl = biases_row_base + row * in_vec_size_g;U b = static_cast<U>(bl[g]);result[row] +=qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);@@ -216,7 +223,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(: 0;if (remaining > 0) {- U sum = load_vector_safe<T, U, values_per_thread, BITS>(+ U sum = load_vector_safe<InputT, U, values_per_thread, BITS>(input,x_thread,remaining@@ -225,12 +232,12 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(for (uint row = 0; row < results_per_simdgroup; row++) {auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);const uint row_idx = used_out_row + row;- const device T* sr = scales_base + row_idx * in_vec_size_g;+ const device WeightT* sr = scales_base + row_idx * in_vec_size_g;uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;U s = static_cast<U>(sr[g]);if (quant_method == QuantizationMethod::ScaleBias) {- const device T* bl = biases_row_base + row * in_vec_size_g;+ const device WeightT* bl = biases_row_base + row * in_vec_size_g;U b = static_cast<U>(bl[g]);result[row] += qdot_safe<U, values_per_thread, BITS>(wl,@@ -258,7 +265,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(for (uint row = 0; row < results_per_simdgroup; row++) {result[row] = simd_sum(result[row]);if (simd_lane == 0) {- output[row] = static_cast<T>(result[row]);+ output[row] = static_cast<OutputT>(result[row]);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metalindex 8d0fbfb2c..89859f704 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal@@ -8,17 +8,24 @@using namespace uzu::quantization_method;using namespace uzu::gemm;-template <typename T, uint GROUP_SIZE, uint BITS>-VARIANTS(T, float, half, bfloat)+template <+ typename WeightT,+ typename InputT,+ typename OutputT,+ uint GROUP_SIZE,+ uint BITS>+VARIANTS(WeightT, float, half, bfloat)+VARIANTS(InputT, float, half, bfloat)+VARIANTS(OutputT, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)PUBLIC KERNEL(QuantizedMatmulQmvFast)(const device uint32_t* weights,- const device T* scales,+ const device WeightT* scales,const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),- const device T* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),- const device T* input,- device T* output,+ const device WeightT* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),+ const device InputT* input,+ device OutputT* output,const device int32_t* hadamard_factors OPTIONAL(use_hadamard),const constant uint& in_vec_size,const constant uint& out_vec_size,@@ -76,7 +83,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(output += batch_idx * out_vec_size + out_row;for (uint k = 0; k < in_vec_size; k += block_size) {- U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);+ U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);{auto wl0 = (const device uint8_t*)(ws);@@ -179,7 +186,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(if (global_out_idx < out_vec_size) {output[simd_lane] = simdgroup_output_random_hadamard_transform(static_cast<ushort>(simd_lane),- T(shared_results[simd_lane]),+ OutputT(shared_results[simd_lane]),hadamard_factors[global_out_idx]);}@@ -187,7 +194,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(} else {if (simd_lane == 0) {for (uint row = 0; row < results_per_simdgroup; row++) {- output[row] = static_cast<T>(result[row]);+ output[row] = static_cast<OutputT>(result[row]);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rsindex 322c016b4..c2e4d6b33 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs@@ -26,7 +26,9 @@ struct QmvKey {}pub(crate) struct QuantGemvKernel {- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,qmv: HashMap<QmvKey, <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel>,qmv_fast: HashMap<QmvKey,@@ -37,10 +39,14 @@ pub(crate) struct QuantGemvKernel {impl QuantGemvKernel {pub(crate) fn new(_context: &MetalContext,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,) -> Self {Self {- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,qmv: HashMap::new(),qmv_fast: HashMap::new(),}@@ -126,7 +132,9 @@ impl QuantGemvKernel {Entry::Vacant(entry) => {let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(context,- self.data_type,+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,group_size,bits,method,@@ -156,7 +164,9 @@ impl QuantGemvKernel {Entry::Vacant(entry) => {let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(context,- self.data_type,+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,group_size,bits,method,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rsindex bd985238f..fac0b723c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs@@ -11,9 +11,9 @@ pub struct GemvSpecialization {}impl GemvSpecialization {- pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {+ pub fn precompile_configs(weights_data_type: crate::DataType) -> &'static [Self] {use crate::DataType;- match data_type {+ match weights_data_type {DataType::F32 => &[Self {threadgroup_rows: 4,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex b94707ccb..9c8015bf2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -140,15 +140,19 @@ impl MatmulKernel for MatmulMetalKernel {fn new(context: &MetalContext,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,) -> Result<Self, MetalError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());+ for data_type in [weights_data_type, input_data_type, output_data_type] {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());+ }}- let gemm = GemmKernel::new(context, data_type)?;- let gemv = GemvKernel::new(context, data_type)?;- let quant_gemv = QuantGemvKernel::new(context, data_type);+ let gemm = GemmKernel::new(context, weights_data_type, input_data_type, output_data_type)?;+ let gemv = GemvKernel::new(context, weights_data_type, input_data_type, output_data_type)?;+ let quant_gemv = QuantGemvKernel::new(context, weights_data_type, input_data_type, output_data_type);Ok(Self {gemv,diff --git a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metalindex 184350e6f..3809cb350 100644--- a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal@@ -1,11 +1,12 @@#include <metal_stdlib>#include "../common/dsl.h"-template <typename T>+template <typename T, typename BiasT>VARIANTS(T, float, half, bfloat)+VARIANTS(BiasT, float, half, bfloat)PUBLIC KERNEL(TensorAddBias)(const device T* input OPTIONAL(!in_place),- const device T* bias,+ const device BiasT* bias,device T* output,constant uint& num_cols,constant uint& length,@@ -17,5 +18,7 @@ PUBLIC KERNEL(TensorAddBias)(}uint col = position % num_cols;- output[position] = input[position] + bias[col];-}\ No newline at end of file+ output[position] = static_cast<T>(+ static_cast<float>(input[position]) + static_cast<float>(bias[col])+ );+}diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rsindex ce5e3df68..704f9387b 100644--- a/crates/backend-uzu/src/encodable_block/attention.rs+++ b/crates/backend-uzu/src/encodable_block/attention.rs@@ -235,7 +235,9 @@ impl<B: Backend> Attention<B> {<B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(context, data_type).map_err(AttentionError::BackendError)?;let matmul_kernel =- <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).ok().map(RefCell::new);+ <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type, data_type, data_type)+ .ok()+ .map(RefCell::new);let update_kv_cache_kernel =<B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, false).map_err(AttentionError::BackendError)?;diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 1f5a917e0..3f846f377 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -158,8 +158,13 @@ impl<B: Backend> Embedding<B> {).map_err(EmbeddingError::BackendError)?;let readout = RefCell::new(- <B::Kernels as ManualKernels>::MatmulKernel::new(context, model_shape.data_type)- .map_err(EmbeddingError::BackendError)?,+ <B::Kernels as ManualKernels>::MatmulKernel::new(+ context,+ model_shape.data_type,+ model_shape.data_type,+ model_shape.data_type,+ )+ .map_err(EmbeddingError::BackendError)?,);(@@ -362,8 +367,13 @@ impl<B: Backend> Embedding<B> {.validate(&[vocab_size as usize, model_dim as usize], model_shape.data_type)?.read_allocation()?;let readout = RefCell::new(- <B::Kernels as ManualKernels>::MatmulKernel::new(context, model_shape.data_type)- .map_err(EmbeddingError::BackendError)?,+ <B::Kernels as ManualKernels>::MatmulKernel::new(+ context,+ model_shape.data_type,+ model_shape.data_type,+ model_shape.data_type,+ )+ .map_err(EmbeddingError::BackendError)?,);UntiedEmbeddingReadoutType::FullPrecision {@@ -730,8 +740,8 @@ fn quantized_readout<B: Backend>(method: QuantizationMethod,group_size: usize,) -> Result<(RefCell<<B::Kernels as ManualKernels>::MatmulKernel>, ReadoutQuantConfig), EmbeddingError<B>> {- let mut readout =- <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;+ let mut readout = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type, data_type, data_type)+ .map_err(EmbeddingError::BackendError)?;readout.preheat_quant_combo(context,diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rsindex 2d3bfe7b6..afe203009 100644--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs@@ -105,8 +105,8 @@ impl<B: Backend> LayerExecutables<B> {};#[cfg(feature = "tracing")]- let tensor_add =- TensorAddBiasKernel::new(context, data_type, false).map_err(LayerExecutablesError::BackendError)?;+ let tensor_add = TensorAddBiasKernel::new(context, data_type, data_type, false)+ .map_err(LayerExecutablesError::BackendError)?;let mixer_subtree = parameter_tree.subtree("mixer")?;diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsindex 4fccea9ee..da54a4872 100644--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -48,7 +48,7 @@ pub struct LinearMatmul<B: Backend> {biases: Option<Allocation<B>>,input_dim: usize,output_dim: usize,- data_type: DataType,+ output_data_type: DataType,mode: Mode<B>,}@@ -58,19 +58,31 @@ impl<B: Backend> LinearMatmul<B> {input_dim: usize,output_dim: usize,has_biases: bool,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,parameter_tree: &ParameterTree<B>,) -> Result<Self, LinearMatmulError<B>> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(LinearMatmulError::UnsupportedDataType(data_type));+ for data_type in [weights_data_type, input_data_type, output_data_type] {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(LinearMatmulError::UnsupportedDataType(data_type));+ }}- let weights =- parameter_tree.leaf("weights.weights")?.validate(&[output_dim, input_dim], data_type)?.read_allocation()?;- let biases = load_biases(data_type, output_dim, has_biases.then_some(parameter_tree))?;+ let weights = parameter_tree+ .leaf("weights.weights")?+ .validate(&[output_dim, input_dim], weights_data_type)?+ .read_allocation()?;+ let biases =+ load_biases(weights_data_type, output_data_type, output_dim, has_biases.then_some(parameter_tree))?;- let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)- .map_err(LinearMatmulError::BackendError)?;+ let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(+ context,+ weights_data_type,+ input_data_type,+ output_data_type,+ )+ .map_err(LinearMatmulError::BackendError)?;Ok(Self {kernel: RefCell::new(kernel),@@ -78,7 +90,7 @@ impl<B: Backend> LinearMatmul<B> {biases,input_dim,output_dim,- data_type,+ output_data_type,mode: Mode::FullPrecision,})}@@ -88,7 +100,9 @@ impl<B: Backend> LinearMatmul<B> {spec: AnyWeightMatrixSpec,input_dim: usize,output_dim: usize,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,weights_tree: &ParameterTree<B>,bias_tree: Option<&ParameterTree<B>>,output_hadamard_factors: Option<Allocation<B>>,@@ -120,8 +134,10 @@ impl<B: Backend> LinearMatmul<B> {},};- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(LinearMatmulError::UnsupportedDataType(data_type));+ for data_type in [weights_data_type, input_data_type, output_data_type] {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(LinearMatmulError::UnsupportedDataType(data_type));+ }}let packing_divisor = weight_quantization_mode.packing_divisor();@@ -132,10 +148,10 @@ impl<B: Backend> LinearMatmul<B> {.leaf("weights")?.validate(&[output_dim, input_dim / packing_divisor], storage_type)?.read_allocation()?;- let scales = weights_tree.leaf("scales")?.validate(&[output_dim, k_g], data_type)?.read_allocation()?;+ let scales = weights_tree.leaf("scales")?.validate(&[output_dim, k_g], weights_data_type)?.read_allocation()?;let zero_points_or_biases = match quantization_method {QuantizationMethod::ScaleBias => {- weights_tree.leaf("biases")?.validate(&[output_dim, k_g], data_type)?.read_allocation()?+ weights_tree.leaf("biases")?.validate(&[output_dim, k_g], weights_data_type)?.read_allocation()?},QuantizationMethod::ScaleZeroPoint => {let expected_zero_points_entries = k_g.div_ceil(packing_divisor);@@ -146,10 +162,15 @@ impl<B: Backend> LinearMatmul<B> {},};- let biases = load_biases(data_type, output_dim, bias_tree)?;+ let biases = load_biases(weights_data_type, output_data_type, output_dim, bias_tree)?;- let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)- .map_err(LinearMatmulError::BackendError)?;+ let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(+ context,+ weights_data_type,+ input_data_type,+ output_data_type,+ )+ .map_err(LinearMatmulError::BackendError)?;kernel.preheat_quant_combo(context,@@ -167,7 +188,7 @@ impl<B: Backend> LinearMatmul<B> {biases,input_dim,output_dim,- data_type,+ output_data_type,mode: Mode::Quantized {method: quantization_method,mode: weight_quantization_mode,@@ -181,12 +202,18 @@ impl<B: Backend> LinearMatmul<B> {}fn load_biases<B: Backend>(- data_type: DataType,+ weights_data_type: DataType,+ output_data_type: DataType,output_dim: usize,parameter_tree: Option<&ParameterTree<B>>,) -> Result<Option<Allocation<B>>, LinearMatmulError<B>> {+ if parameter_tree.is_some() && weights_data_type != output_data_type {+ return Err(LinearMatmulError::UnsupportedConfiguration(format!(+ "mixed precision linear with biases is not supported: weights={weights_data_type:?}, output={output_data_type:?}",+ )));+ }Ok(parameter_tree- .map(|tree| tree.leaf("biases")?.validate(&[output_dim], data_type)?.read_allocation())+ .map(|tree| tree.leaf("biases")?.validate(&[output_dim], weights_data_type)?.read_allocation()).transpose()?)}@@ -197,7 +224,8 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.data_type))?;+ let mut output =+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;let b = match &self.mode {Mode::FullPrecision => MatmulB::FullPrecision {diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rsindex 178f45d0b..6b044aabf 100644--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs@@ -54,12 +54,14 @@ pub enum OutputHadamardLinearError<B: Backend> {}impl<B: Backend> dyn Linear<B> {- pub fn new<const N: usize>(+ pub fn new_mixed_precision<const N: usize>(input_dimension: usize,output_dimensions: [usize; N],has_biases: bool,context: &B::Context,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,parameter_tree: &ParameterTree<B>,) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {let output_dimension_sum: usize = output_dimensions.iter().sum();@@ -75,7 +77,9 @@ impl<B: Backend> dyn Linear<B> {input_dimension,output_dimension_sum,has_biases,- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,parameter_tree,)?;Ok(Box::new(block))@@ -90,7 +94,9 @@ impl<B: Backend> dyn Linear<B> {input_dimension,output_dimension_sum,has_biases,- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,parameter_tree,)?)),AnyWeightMatrixSpec::HybridSpec(HybridSpec {@@ -113,7 +119,9 @@ impl<B: Backend> dyn Linear<B> {incoherence_processing_mode,input_dimension,output_dimension_sum,- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,&weights_tree,)?))},@@ -123,7 +131,9 @@ impl<B: Backend> dyn Linear<B> {spec,input_dimension,output_dimension_sum,- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,&weights_tree,has_biases.then_some(parameter_tree),None,@@ -134,14 +144,36 @@ impl<B: Backend> dyn Linear<B> {}}- pub fn new_with_output_hadamard(+ pub fn new<const N: usize>(+ input_dimension: usize,+ output_dimensions: [usize; N],+ has_biases: bool,+ context: &B::Context,+ data_type: DataType,+ parameter_tree: &ParameterTree<B>,+ ) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {+ Self::new_mixed_precision(+ input_dimension,+ output_dimensions,+ has_biases,+ context,+ data_type,+ data_type,+ data_type,+ parameter_tree,+ )+ }++ pub fn new_with_output_hadamard_mixed_precision(context: &B::Context,parameter_tree: &ParameterTree<B>,output_factors: Allocation<B>,input_dim: usize,output_dim: usize,has_biases: bool,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,) -> Result<Box<dyn Linear<B>>, OutputHadamardLinearError<B>> {let weights_tree = parameter_tree.subtree("weights")?.subtree("quantized")?;let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;@@ -152,7 +184,9 @@ impl<B: Backend> dyn Linear<B> {spec,input_dim,output_dim,- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,&weights_tree,has_biases.then_some(parameter_tree),Some(output_factors),@@ -164,12 +198,14 @@ impl<B: Backend> dyn Linear<B> {}}- pub fn new_extracting_input_hadamard<const N: usize>(+ pub fn new_extracting_input_hadamard_mixed_precision<const N: usize>(input_dimension: usize,output_dimensions: [usize; N],has_biases: bool,context: &B::Context,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,parameter_tree: &ParameterTree<B>,) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {let output_dimension_sum: usize = output_dimensions.iter().sum();@@ -190,19 +226,50 @@ impl<B: Backend> dyn Linear<B> {.leaf("incoherence_signs.output_signs")?.validate(&[output_dimension_sum], DataType::I32)?.read_allocation()?;- let inner_linear = Self::new_with_output_hadamard(+ let inner_linear = Self::new_with_output_hadamard_mixed_precision(context,parameter_tree,output_factors,input_dimension,output_dimension_sum,has_biases,- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,)?;Ok((inner_linear, Some(input_factors)))} else {- let linear = Self::new(input_dimension, output_dimensions, has_biases, context, data_type, parameter_tree)?;+ let linear = Self::new_mixed_precision(+ input_dimension,+ output_dimensions,+ has_biases,+ context,+ weights_data_type,+ input_data_type,+ output_data_type,+ parameter_tree,+ )?;Ok((linear, None))}}++ pub fn new_extracting_input_hadamard<const N: usize>(+ input_dimension: usize,+ output_dimensions: [usize; N],+ has_biases: bool,+ context: &B::Context,+ data_type: DataType,+ parameter_tree: &ParameterTree<B>,+ ) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {+ Self::new_extracting_input_hadamard_mixed_precision(+ input_dimension,+ output_dimensions,+ has_biases,+ context,+ data_type,+ data_type,+ data_type,+ parameter_tree,+ )+ }}diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rsindex 08bc3aab4..793912a5a 100644--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -37,13 +37,15 @@ pub struct QLoRALinearWrapper<B: Backend> {base_linear: LinearMatmul<B>,input_hadamard: Option<(<B::Kernels as Kernels>::HadamardTransformKernel, Allocation<B>)>,output_hadamard: Option<(<B::Kernels as Kernels>::HadamardTransformKernel, Allocation<B>)>,- adapter_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,+ adapter_down_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,+ adapter_up_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,adapter_down: Allocation<B>,adapter_up: Allocation<B>,input_dim: usize,output_dim: usize,lora_rank: usize,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,}impl<B: Backend> QLoRALinearWrapper<B> {@@ -55,7 +57,9 @@ impl<B: Backend> QLoRALinearWrapper<B> {incoherence_processing_mode: IncoherenceProcessingMode,input_dim: usize,output_dim: usize,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,weights_tree: &ParameterTree<B>,) -> Result<Self, QLoRALinearWrapperError<B>> {let use_incoherence_signs = match (incoherence_block_size, incoherence_processing_mode) {@@ -74,7 +78,9 @@ impl<B: Backend> QLoRALinearWrapper<B> {quantization_spec,input_dim,output_dim,- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,&quantized_tree,None,None,@@ -93,7 +99,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {Some((<B::Kernels as Kernels>::HadamardTransformKernel::new(context,- data_type,+ input_data_type,HadamardTransformOrder::Input,).map_err(QLoRALinearWrapperError::BackendError)?,@@ -102,7 +108,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {Some((<B::Kernels as Kernels>::HadamardTransformKernel::new(context,- data_type,+ output_data_type,HadamardTransformOrder::Output,).map_err(QLoRALinearWrapperError::BackendError)?,@@ -113,32 +119,48 @@ impl<B: Backend> QLoRALinearWrapper<B> {(None, None)};- let adapter_kernel = RefCell::new(- <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)- .map_err(QLoRALinearWrapperError::BackendError)?,+ let adapter_down_kernel = RefCell::new(+ <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(+ context,+ weights_data_type,+ input_data_type,+ weights_data_type,+ )+ .map_err(QLoRALinearWrapperError::BackendError)?,+ );+ let adapter_up_kernel = RefCell::new(+ <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(+ context,+ weights_data_type,+ weights_data_type,+ output_data_type,+ )+ .map_err(QLoRALinearWrapperError::BackendError)?,);let adapter_down = weights_tree.leaf("adapter.down_projection")?- .validate(&[adapter_spec.rank, input_dim], data_type)?+ .validate(&[adapter_spec.rank, input_dim], weights_data_type)?.read_allocation()?;let adapter_up = weights_tree.leaf("adapter.up_projection")?- .validate(&[output_dim, adapter_spec.rank], data_type)?+ .validate(&[output_dim, adapter_spec.rank], weights_data_type)?.read_allocation()?;Ok(Self {base_linear,input_hadamard,output_hadamard,- adapter_kernel,+ adapter_down_kernel,+ adapter_up_kernel,adapter_down,adapter_up,input_dim,output_dim,lora_rank: adapter_spec.rank,- data_type,+ weights_data_type,+ input_data_type,})}}@@ -151,10 +173,10 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let mut intermediate =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.data_type))?;+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type))?;{- let mut adapter_kernel = self.adapter_kernel.borrow_mut();+ let mut adapter_kernel = self.adapter_down_kernel.borrow_mut();adapter_kernel.encode(MatmulArguments {a: &input,@@ -177,7 +199,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {let base_input = if let Some((input_hadamard_kernel, input_factors)) = &self.input_hadamard {let mut base_input =- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dim], self.data_type))?;+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dim], self.input_data_type))?;encoder.encode_copy(&input, .., &mut base_input, ..);input_hadamard_kernel.encode(&mut base_input,@@ -194,7 +216,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {let mut output = self.base_linear.encode(base_input, batch_dim, encoder)?;{- let mut adapter_kernel = self.adapter_kernel.borrow_mut();+ let mut adapter_kernel = self.adapter_up_kernel.borrow_mut();adapter_kernel.encode(MatmulArguments {a: &intermediate,diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rsindex ddcd964ea..bb32a7b0c 100644--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs@@ -40,7 +40,9 @@ impl<B: Backend> RHTLinearWrapper<B> {input_dimension: usize,output_dimension: usize,has_biases: bool,- data_type: DataType,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,parameter_tree: &ParameterTree<B>,) -> Result<Self, RHTLinearWrapperError<B>> {let weights_tree = parameter_tree.subtree("weights")?;@@ -63,17 +65,22 @@ impl<B: Backend> RHTLinearWrapper<B> {.leaf("incoherence_signs.output_signs")?.validate(&[output_dimension], DataType::I32)?.read_allocation()?;- let input_hadamard_kernel =- <B::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type, HadamardTransformOrder::Input)- .map_err(RHTLinearWrapperError::BackendError)?;- let inner_linear = <dyn Linear<B>>::new_with_output_hadamard(+ let input_hadamard_kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(+ context,+ input_data_type,+ HadamardTransformOrder::Input,+ )+ .map_err(RHTLinearWrapperError::BackendError)?;+ let inner_linear = <dyn Linear<B>>::new_with_output_hadamard_mixed_precision(context,parameter_tree,output_factors,input_dimension,output_dimension,has_biases,- data_type,+ weights_data_type,+ input_data_type,+ output_data_type,)?;Ok(Self {diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rsindex fe17cdac3..541bd6a70 100644--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs@@ -100,8 +100,10 @@ impl<B: Backend> FishAudioSemanticBridge<B> {) -> Result<Self, Error> {let embedding_rows_sum = <B::Kernels as Kernels>::EmbeddingRowsSumKernel::new(context, data_type).map_err(unable_to_create_context)?;- let projection = <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)- .map_err(unable_to_create_context)?;+ let projection = <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(+ context, data_type, data_type, data_type,+ )+ .map_err(unable_to_create_context)?;let codebook_token_indices = context.create_array_zeros(&[num_codebooks], DataType::U32).into_allocation();Ok(Self {diff --git a/crates/backend-uzu/src/utils/dispatch_dtype.rs b/crates/backend-uzu/src/utils/dispatch_dtype.rsnew file mode 100644index 000000000..792d31f14--- /dev/null+++ b/crates/backend-uzu/src/utils/dispatch_dtype.rs@@ -0,0 +1,12 @@+#[macro_export]+macro_rules! dispatch_dtype {+ (|()| $body:expr) => { $body };+ (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {+ match $dt {+ $crate::DataType::F16 => { type $T = half::f16; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }+ $crate::DataType::BF16 => { type $T = half::bf16; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }+ $crate::DataType::F32 => { type $T = f32; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }+ other => panic!("unsupported dtype in test: {:?}", other),+ }+ };+}diff --git a/crates/backend-uzu/src/utils/mod.rs b/crates/backend-uzu/src/utils/mod.rsindex f002e61cb..4e523cabc 100644--- a/crates/backend-uzu/src/utils/mod.rs+++ b/crates/backend-uzu/src/utils/mod.rs@@ -1,3 +1,4 @@+pub mod dispatch_dtype;pub mod env_utils;pub mod fs;pub mod memory;diff --git a/crates/backend-uzu/tests/common/matmul/harness.rs b/crates/backend-uzu/tests/common/matmul/harness.rsindex 105f169db..f2e96bc80 100644--- a/crates/backend-uzu/tests/common/matmul/harness.rs+++ b/crates/backend-uzu/tests/common/matmul/harness.rs@@ -142,8 +142,13 @@ fn run<B: Backend, T: ArrayElement + Float>(pub fn cpu_reference<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let context = <Cpu as Backend>::Context::new().expect("CPU context");- let mut kernel = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())- .expect("CPU MatmulKernel");+ let mut kernel = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ &context,+ T::data_type(),+ T::data_type(),+ T::data_type(),+ )+ .expect("CPU MatmulKernel");run::<Cpu, T>(&context, &mut kernel, input, |kernel, args, encoder| {kernel.encode(args, encoder).expect("encode failed");})diff --git a/crates/backend-uzu/tests/common/matmul/quant.rs b/crates/backend-uzu/tests/common/matmul/quant.rsindex c6cc2bdfa..bad1def0d 100644--- a/crates/backend-uzu/tests/common/matmul/quant.rs+++ b/crates/backend-uzu/tests/common/matmul/quant.rs@@ -155,8 +155,13 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(pub fn run_quant_cpu<T: ArrayElement + Float>(input: &QuantInput<T>) -> Vec<T> {let context = <Cpu as Backend>::Context::new().expect("Cpu context");let mut buffers = QuantBuffers::<Cpu, T>::allocate(&context, input);- let mut matmul = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())- .expect("MatmulCpuKernel");+ let mut matmul = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ &context,+ T::data_type(),+ T::data_type(),+ T::data_type(),+ )+ .expect("MatmulCpuKernel");let mut encoder = Encoder::<Cpu>::new(&context).expect("encoder");matmul.encode(quant_arguments(&mut buffers, input), &mut encoder).expect("encode cpu quant");encoder.end_encoding().submit().wait_until_completed().unwrap();@@ -170,8 +175,13 @@ pub fn run_quant_metal<T: ArrayElement + Float>(path: Option<GemmDispatchPath>,) -> Vec<T> {let mut buffers = QuantBuffers::<Metal, T>::allocate(context, input);- let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type())- .expect("MatmulMetalKernel");+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ context,+ T::data_type(),+ T::data_type(),+ T::data_type(),+ )+ .expect("MatmulMetalKernel");let mut encoder = Encoder::<Metal>::new(context).expect("encoder");let args = quant_arguments(&mut buffers, input);match path {diff --git a/crates/backend-uzu/tests/common/mod.rs b/crates/backend-uzu/tests/common/mod.rsindex ada88165e..4a6372df1 100644--- a/crates/backend-uzu/tests/common/mod.rs+++ b/crates/backend-uzu/tests/common/mod.rs@@ -8,7 +8,7 @@ pub mod path;pub mod perf;pub mod proptest;-pub(crate) use proptest::{dispatch_dtype, for_each_context};+pub(crate) use proptest::for_each_context;pub fn type_short_name<T>() -> &'static str {std::any::type_name::<T>().rsplit("::").next().unwrap()diff --git a/crates/backend-uzu/tests/common/proptest.rs b/crates/backend-uzu/tests/common/proptest.rsindex c30ce6138..a5538dca8 100644--- a/crates/backend-uzu/tests/common/proptest.rs+++ b/crates/backend-uzu/tests/common/proptest.rs@@ -74,16 +74,3 @@ impl<T: ComparableTestResults> TestResults<T> {Ok(())}}--macro_rules! dispatch_dtype {- (|()| $body:expr) => { $body };- (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {- match $dt {- backend_uzu::DataType::F16 => { type $T = half::f16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }- backend_uzu::DataType::BF16 => { type $T = half::bf16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }- backend_uzu::DataType::F32 => { type $T = f32; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }- other => panic!("unsupported dtype in test: {:?}", other),- }- };-}-pub(crate) use dispatch_dtype;diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rsindex 6c091c87e..797c04223 100644--- a/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs+++ b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs@@ -28,7 +28,12 @@ use crate::{fn bench_gemm(c: &mut Criterion) {let context = MetalContext::new().expect("Metal context");let mut kernel =- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ ).expect("MatmulKernel");let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rsindex 55920ac09..b6674cecd 100644--- a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs@@ -36,7 +36,13 @@ fn bench_unified_quant_typed<T: ArrayElement + Float>(let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);let mut matmul =- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type()).unwrap();+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ context,+ T::data_type(),+ T::data_type(),+ T::data_type(),+ )+ .unwrap();let mut group =c.benchmark_group(format!("{}/Kernel/UnifiedQuantizedGemm/{}", type_short_name::<Metal>(), label));diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rsindex 41e15db09..e40fae33a 100644--- a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs@@ -34,6 +34,8 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(context,T::data_type(),+ T::data_type(),+ T::data_type(),group_size,bits,quant_method,diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rsindex 5fe879f7f..d6770cb57 100644--- a/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs+++ b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs@@ -38,7 +38,13 @@ fn bench_qwen3_layers_typed<T: ArrayElement + Float>(let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);let mut matmul =- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type()).unwrap();+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ context,+ T::data_type(),+ T::data_type(),+ T::data_type(),+ )+ .unwrap();group.throughput(Throughput::Elements((m * n * k) as u64));group.bench_function(BenchmarkId::from_parameter(format!("{layer}_{shape}")), |b| {diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rsindex 1bc168176..f71ae2dea 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs@@ -75,7 +75,9 @@ fn pipeline_output<B: Backend>(q: &[bf16], k: &[bf16], v: &[bf16], scale: f32) -.unwrap();let scatter_values =<<B as Backend>::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(&ctx, DataType::BF16).unwrap();- let mut matmul = <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&ctx, DataType::BF16).unwrap();+ let mut matmul =+ <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&ctx, DataType::BF16, DataType::BF16, DataType::BF16)+ .unwrap();let qa = alloc_allocation_with_data::<B, bf16>(&ctx, q);let ka = alloc_allocation_with_data::<B, bf16>(&ctx, k);diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rsindex eb03ae323..13b82d78e 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs@@ -49,7 +49,12 @@ fn run_matrix<T: ArrayElement + Float + Debug + Display>(tolerance: f32,) {let context = MetalContext::new().expect("Metal context");- let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())+ let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ &context,+ T::data_type(),+ T::data_type(),+ T::data_type(),+ ).expect("MatmulKernel");for path in gemm_paths_for_hw(&context) {for shape in all_correctness_shapes() {diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rsindex 948e06985..49fb81038 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs@@ -63,7 +63,12 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {.create_allocation(input.m * input.n * std::mem::size_of::<T>(), AllocationType::Global).expect("Failed to create allocation");- let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())+ let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(+ &context,+ T::data_type(),+ T::data_type(),+ T::data_type(),+ ).expect("Failed to create MatmulKernel");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rsindex aa37cb960..66b5f3a41 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs@@ -135,7 +135,12 @@ fn parity_bf16_gs32_4bit_mlx_with_bias() {let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);let bias_pp_buf =crate::common::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);- let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ ).expect("MatmulMetalKernel");let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");@@ -168,7 +173,12 @@ fn quant_gemm_nonzero_b_offset_returns_unsupported_layout() {let context = MetalContext::new().expect("Metal context");let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);- let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ ).expect("MatmulMetalKernel");let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");@@ -192,7 +202,12 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {let context = MetalContext::new().expect("Metal context");let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);- let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ ).expect("MatmulMetalKernel");let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rsindex 24e497835..a5d7a53e3 100644--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs@@ -34,6 +34,8 @@ fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let kernel = <<Cpu as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(&context,T::data_type(),+ T::data_type(),+ T::data_type(),input.group_size,input.bits,input.quant_method,@@ -72,6 +74,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(&context,T::data_type(),+ T::data_type(),+ T::data_type(),input.group_size,input.bits,input.quant_method,diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rsindex 97c109cea..f80e165d9 100644--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs@@ -35,6 +35,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(&context,T::data_type(),+ T::data_type(),+ T::data_type(),input.group_size,input.bits,input.quant_method,diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rsindex e74b751d4..1bf6e5bb6 100644--- a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs+++ b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs@@ -5,7 +5,7 @@ use proptest::prelude::*;use rand::{RngExt, SeedableRng, rngs::SmallRng};use rand_distr::Normal;use backend_uzu::{- ArrayContextExt, ArrayElement, DataType,+ ArrayContextExt, ArrayElement, DataType, dispatch_dtype,backends::common::{AllocationType, Backend, Context, Encoder, Kernels,gpu_types::ArgmaxPair,@@ -15,7 +15,7 @@ use backend_uzu::{use crate::{common::{- dispatch_dtype, for_each_context,+ for_each_context,proptest::{ComparableTestResults, TestContextes, kernel_data_type},},uzu_test,@@ -113,4 +113,3 @@ fn test_argmax_prop() {});});}-diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rsindex 7a499c6ab..11be8260e 100644--- a/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs@@ -60,8 +60,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>() -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(&context, T::data_type(), in_place)- .expect("Failed to create TensorAddBiasKernel");+ let kernel =+ <<B as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(&context, T::data_type(), T::data_type(), in_place)+ .expect("Failed to create TensorAddBiasKernel");let size = input.length as usize;let input_allocation = (!in_place).then(|| alloc_allocation_with_data::<B, T>(&context, &input.input));