Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Mixed precision matmul (#451)

uuuvndb8613d5754c
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
index 152799751..b9cb93d36 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs
@@ -9,7 +9,9 @@ pub trait MatmulKernel: Sized {
fn new(
context: &<Self::Backend as Backend>::Context,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
) -> Result<Self, <Self::Backend as Backend>::Error>;
fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Self::Backend>>>(
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
index cbafef074..bfe3410e2 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
@@ -6,11 +6,11 @@ use crate::{
};
#[inline(always)]
-unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(
- a: SendPtr<T>,
- b: SendPtr<T>,
+unsafe fn matmul_gemm_compute_row<TA: ArrayElement + Float, TB: ArrayElement + Float, TD: ArrayElement + Float>(
+ a: SendPtr<TA>,
+ b: SendPtr<TB>,
ab_scale: f32,
- d: SendPtrMut<T>,
+ d: SendPtrMut<TD>,
row: usize,
n: usize,
k: usize,
@@ -37,16 +37,16 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(
if is_accumulate {
acc += (*d_element).to_f32().unwrap();
}
- *d_element = T::from(acc).unwrap();
+ *d_element = TD::from(acc).unwrap();
}
}
}
-pub fn matmul_gemm_impl<T: ArrayElement + Float>(
- a: *const T,
- b: *const T,
+pub fn matmul_gemm_impl<TA: ArrayElement + Float, TB: ArrayElement + Float, TD: ArrayElement + Float>(
+ a: *const TA,
+ b: *const TB,
ab_scale: f32,
- d: *mut T,
+ d: *mut TD,
m: usize,
n: usize,
k: usize,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
index 63b3cf00b..f306865e0 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
@@ -11,13 +11,13 @@ use crate::{ArrayElement, backends::common::gpu_types::QuantizationMethod};
/// where groups are along the K dimension.
///
/// Computes y[i, j] = sum_l( x[i, l] * dequant(w[j, l]) ) for each batch i and output j.
-pub fn qmv<T: ArrayElement + Float>(
+pub fn qmv<WeightT: ArrayElement + Float, InputT: ArrayElement + Float, OutputT: ArrayElement + Float>(
weights: *const u32,
- scales: *const T,
+ scales: *const WeightT,
zero_points: Option<*const u8>,
- biases: Option<*const T>,
- input: *const T,
- output: *mut T,
+ biases: Option<*const WeightT>,
+ input: *const InputT,
+ output: *mut OutputT,
in_vec_size: usize,
out_vec_size: usize,
batch_size: usize,
@@ -90,29 +90,37 @@ pub fn qmv<T: ArrayElement + Float>(
acc += val_a * (scale * val_q + bias);
}
- *output.add(i * out_vec_size + j) = T::from(acc).unwrap();
+ *output.add(i * out_vec_size + j) = OutputT::from(acc).unwrap();
}
}
}
}
#[kernel(QuantizedMatmulQmv)]
-#[variants(T, f32, f16, bf16)]
+#[variants(WeightT, f32, f16, bf16)]
+#[variants(InputT, f32, f16, bf16)]
+#[variants(OutputT, f32, f16, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
+pub fn quantized_matmul_qmv<
+ WeightT: ArrayElement + Float,
+ InputT: ArrayElement + Float,
+ OutputT: ArrayElement + Float,
+ const GROUP_SIZE: u32,
+ const BITS: u32,
+>(
weights: *const u32,
- scales: *const T,
+ scales: *const WeightT,
#[optional(quant_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,
- #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const T>,
- input: *const T,
- output: *mut T,
+ #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const WeightT>,
+ input: *const InputT,
+ output: *mut OutputT,
in_vec_size: u32,
out_vec_size: u32,
batch_size: u32,
#[specialize] quant_method: QuantizationMethod,
) {
- qmv::<T>(
+ qmv::<WeightT, InputT, OutputT>(
weights,
scales,
zero_points,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
index 0a42788ff..dee1f9c6a 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
@@ -8,16 +8,24 @@ use crate::{
};
#[kernel(QuantizedMatmulQmvFast)]
-#[variants(T, f32, f16, bf16)]
+#[variants(WeightT, f32, f16, bf16)]
+#[variants(InputT, f32, f16, bf16)]
+#[variants(OutputT, f32, f16, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
+pub fn quantized_matmul_qmv_fast<
+ WeightT: ArrayElement + Float,
+ InputT: ArrayElement + Float,
+ OutputT: ArrayElement + Float,
+ const GROUP_SIZE: u32,
+ const BITS: u32,
+>(
weights: *const u32,
- scales: *const T,
+ scales: *const WeightT,
#[optional(quant_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,
- #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const T>,
- input: *const T,
- output: *mut T,
+ #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const WeightT>,
+ input: *const InputT,
+ output: *mut OutputT,
#[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
in_vec_size: u32,
out_vec_size: u32,
@@ -28,7 +36,7 @@ pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: u32,
if use_hadamard {
unimplemented!("not supported yet");
}
- qmv::<T>(
+ qmv::<WeightT, InputT, OutputT>(
weights,
scales,
zero_points,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 178acc315..85916fa0e 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -1,5 +1,3 @@
-use half::{bf16, f16};
-
use crate::{
DataType,
backends::{
@@ -15,11 +13,14 @@ use crate::{
BufferDowncastExt, Cpu, context::CpuContext, error::CpuError, kernel::matmul::quant::encode_quantized_gemm,
},
},
+ dispatch_dtype,
utils::pointers::{SendPtr, SendPtrMut},
};
pub struct MatmulCpuKernel {
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
hadamard: <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel,
bias_add: <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel,
}
@@ -29,19 +30,30 @@ impl MatmulKernel for MatmulCpuKernel {
fn new(
context: &CpuContext,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
) -> Result<Self, CpuError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());
+ for data_type in [weights_data_type, input_data_type, output_data_type] {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());
+ }
}
let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(
context,
- data_type,
+ output_data_type,
HadamardTransformOrder::Output,
)?;
- let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, true)?;
+ let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(
+ context,
+ output_data_type,
+ weights_data_type,
+ true,
+ )?;
Ok(Self {
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
hadamard,
bias_add,
})
@@ -114,7 +126,9 @@ impl MatmulCpuKernel {
n_u
});
let ldd = n_u;
- let data_type = self.data_type;
+ let weights_data_type = self.weights_data_type;
+ let input_data_type = self.input_data_type;
+ let output_data_type = self.output_data_type;
let a_buffer_range = a.as_buffer_range_ref();
let b_buffer_range = weights.as_buffer_range_ref();
let d_buffer_range = d.as_buffer_range_mut();
@@ -134,32 +148,13 @@ impl MatmulCpuKernel {
SendPtr(unsafe { &*bias_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(bias_range.start))
});
- command_buffer.push_command(move || match data_type {
- DataType::F32 => {
- super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f32>(
- a_ptr.as_ptr() as *const f32,
- b_ptr.as_ptr() as *const f32,
- ab_scale,
- d_ptr.as_ptr() as *mut f32,
- m_u,
- n_u,
- k_u,
- lda,
- ldb,
- ldd,
- b_transpose,
- is_accumulate,
- );
- if let Some(bias) = bias_ptr {
- apply_bias::<f32>(d_ptr.as_ptr() as *mut f32, bias.as_ptr() as *const f32, m_u, n_u);
- }
- },
- DataType::F16 => {
- super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f16>(
- a_ptr.as_ptr() as *const f16,
- b_ptr.as_ptr() as *const f16,
+ command_buffer.push_command(move || {
+ dispatch_dtype!(|(TW: weights_data_type, TA: input_data_type, TD: output_data_type)| {
+ super::gemm::shaders::steel_gemm::matmul_gemm_impl::<TA, TW, TD>(
+ a_ptr.as_ptr() as *const TA,
+ b_ptr.as_ptr() as *const TW,
ab_scale,
- d_ptr.as_ptr() as *mut f16,
+ d_ptr.as_ptr() as *mut TD,
m_u,
n_u,
k_u,
@@ -170,29 +165,13 @@ impl MatmulCpuKernel {
is_accumulate,
);
if let Some(bias) = bias_ptr {
- apply_bias::<f16>(d_ptr.as_ptr() as *mut f16, bias.as_ptr() as *const f16, m_u, n_u);
+ assert_eq!(
+ weights_data_type, output_data_type,
+ "mixed precision matmul with bias is not supported until bias dtype ownership is settled",
+ );
+ apply_bias::<TD>(d_ptr.as_ptr() as *mut TD, bias.as_ptr() as *const TD, m_u, n_u);
}
- },
- DataType::BF16 => {
- super::gemm::shaders::steel_gemm::matmul_gemm_impl::<bf16>(
- a_ptr.as_ptr() as *const bf16,
- b_ptr.as_ptr() as *const bf16,
- ab_scale,
- d_ptr.as_ptr() as *mut bf16,
- m_u,
- n_u,
- k_u,
- lda,
- ldb,
- ldd,
- b_transpose,
- is_accumulate,
- );
- if let Some(bias) = bias_ptr {
- apply_bias::<bf16>(d_ptr.as_ptr() as *mut bf16, bias.as_ptr() as *const bf16, m_u, n_u);
- }
- },
- _ => unreachable!(),
+ })
});
Ok(())
@@ -243,6 +222,14 @@ impl MatmulCpuKernel {
let post_bias = arguments.d_transform.bias;
if arguments.m >= 5 && arguments.n > 1 {
+ assert_eq!(
+ self.input_data_type, self.weights_data_type,
+ "mixed precision quantized GEMM input dtype is not supported yet",
+ );
+ assert_eq!(
+ self.output_data_type, self.weights_data_type,
+ "mixed precision quantized GEMM output dtype is not supported yet",
+ );
assert!(
!(post_bias.is_some() && post_rht.is_some()),
"MatmulCpuKernel/Quant GEMM with both output bias and output RHT is not supported: bias must be applied after RHT",
@@ -276,7 +263,7 @@ impl MatmulCpuKernel {
n,
k,
},
- self.data_type,
+ self.weights_data_type,
);
if let Some(bias) = post_bias {
self.bias_add.encode(None::<&Allocation<Cpu>>, bias, &mut *d, n, m * n, encoder);
@@ -334,7 +321,9 @@ impl MatmulCpuKernel {
let kernel =
<<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
context,
- self.data_type,
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
group_size,
bits,
method,
@@ -364,7 +353,9 @@ impl MatmulCpuKernel {
let kernel =
<<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
context,
- self.data_type,
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
group_size,
bits,
method,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
index bc37cff7d..9f36dc287 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
@@ -6,11 +6,12 @@ use crate::ArrayElement;
#[kernel(TensorAddBias)]
#[variants(T, f32, f16, bf16)]
-pub fn tensor_add_bias<T: ArrayElement + Float>(
+#[variants(BiasT, f32, f16, bf16)]
+pub fn tensor_add_bias<T: ArrayElement + Float, BiasT: ArrayElement + Float>(
#[allow(unused)]
#[optional(!in_place)]
input: Option<*const T>,
- #[allow(unused)] bias: *const T,
+ #[allow(unused)] bias: *const BiasT,
#[allow(unused)] output: *mut T,
#[allow(unused)] num_cols: u32,
#[allow(unused)] length: u32,
@@ -21,11 +22,13 @@ pub fn tensor_add_bias<T: ArrayElement + Float>(
for i in 0usize..(length as usize) {
let bias_position = (i % num_cols as usize);
unsafe {
- if let Some(in_data) = input {
- *output.add(i) = *in_data.add(i) + *bias.add(bias_position);
+ let value: f32 = if let Some(in_data) = input {
+ num_traits::cast(*in_data.add(i)).unwrap()
} else {
- *output.add(i) = *output.add(i) + *bias.add(bias_position);
- }
+ num_traits::cast(*output.add(i)).unwrap()
+ };
+ let bias_value: f32 = num_traits::cast(*bias.add(bias_position)).unwrap();
+ *output.add(i) = num_traits::cast(value + bias_value).unwrap();
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
index 609f3bb40..a62acb567 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
@@ -10,7 +10,8 @@ namespace uzu {
namespace matmul {
template <
- typename T,
+ typename AT,
+ typename BT,
ushort SIMDGROUP_BLOCK_M,
ushort SIMDGROUP_BLOCK_N,
ushort SIMDGROUP_BLOCK_K,
@@ -22,8 +23,8 @@ template <
bool aligned_k,
typename AccumulatorType = float>
METAL_FUNC auto gemm_loop(
- const device T* left_ptr,
- const device T* right_ptr,
+ const device AT* left_ptr,
+ const device BT* right_ptr,
int leading_dimension_a,
int leading_dimension_b,
int K,
@@ -53,10 +54,10 @@ METAL_FUNC auto gemm_loop(
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < BLOCK_K; inner_k += SIMDGROUP_BLOCK_K) {
- Fragment<T, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(
+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(
thread_context
);
- Fragment<T, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(
+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(
thread_context
);
@@ -115,10 +116,10 @@ METAL_FUNC auto gemm_loop(
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < remaining_k; inner_k += SIMDGROUP_BLOCK_K) {
- Fragment<T, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(
+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(
thread_context
);
- Fragment<T, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(
+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(
thread_context
);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
index c07e65a5b..9b64d6989 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
@@ -13,8 +13,9 @@ namespace matmul {
///////////////////////////////////////////////////////////////////////////////
template <
- typename T,
- typename U,
+ typename AT,
+ typename BT,
+ typename DT,
int BLOCK_ROWS,
int BLOCK_COLS,
int BLOCK_DEPTH,
@@ -25,7 +26,7 @@ template <
ushort THREADGROUP_LEADING_DIMENSION_A,
ushort THREADGROUP_LEADING_DIMENSION_B,
typename AccumulatorType = float,
- typename Epilogue = TransformNone<U, AccumulatorType>>
+ typename Epilogue = TransformNone<DT, AccumulatorType>>
struct ThreadgroupTile {
METAL_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;
using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<
@@ -107,8 +108,8 @@ struct ThreadgroupTile {
}
METAL_FUNC void multiply_accumulate(
- const threadgroup T* a_shared,
- const threadgroup T* b_shared
+ const threadgroup AT* a_shared,
+ const threadgroup BT* b_shared
) {
a_shared += a_shared_offset;
b_shared += b_shared_offset;
@@ -119,7 +120,7 @@ struct ThreadgroupTile {
simdgroup_barrier(mem_flags::mem_none);
a_fragment.template load<
- T,
+ AT,
SIMDGROUPS_PER_ROW,
1,
A_STRIDE_ROW,
@@ -128,7 +129,7 @@ struct ThreadgroupTile {
simdgroup_barrier(mem_flags::mem_none);
b_fragment.template load<
- T,
+ BT,
1,
SIMDGROUPS_PER_COLUMN,
B_STRIDE_INNER,
@@ -143,7 +144,7 @@ struct ThreadgroupTile {
}
}
- METAL_FUNC void store_result(device U* D, const int leading_dimension_d) {
+ METAL_FUNC void store_result(device DT* D, const int leading_dimension_d) {
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {
c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);
@@ -151,14 +152,14 @@ struct ThreadgroupTile {
D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;
- c_fragment.template store<U, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
+ c_fragment.template store<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
D,
leading_dimension_d
);
}
METAL_FUNC void store_result_safe(
- device U* D,
+ device DT* D,
const int leading_dimension_d,
short2 destination_tile_dimensions
) {
@@ -176,7 +177,7 @@ struct ThreadgroupTile {
return;
c_fragment
- .template store_safe<U, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
+ .template store_safe<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
D,
leading_dimension_d,
destination_tile_dimensions
@@ -185,13 +186,14 @@ struct ThreadgroupTile {
template <typename EpilogueOp>
METAL_FUNC void apply_epilogue(
- const device U* C,
+ const device DT* C,
const int leading_dimension_c,
const int column_stride_c,
thread const EpilogueOp& epilogue_operation
) {
- const device U* c_pointer = C + simdgroup_row_offset * leading_dimension_c +
- simdgroup_col_offset * column_stride_c;
+ const device DT* c_pointer = C +
+ simdgroup_row_offset * leading_dimension_c +
+ simdgroup_col_offset * column_stride_c;
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < TILE_ROWS; i++) {
@@ -206,7 +208,7 @@ struct ThreadgroupTile {
i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
const ushort col_offset =
j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
- U c_value = c_pointer
+ DT c_value = c_pointer
[row_offset * leading_dimension_c + col_offset * column_stride_c +
k];
block_data[k] = epilogue_operation.apply(
@@ -220,14 +222,15 @@ struct ThreadgroupTile {
template <typename EpilogueOp>
METAL_FUNC void apply_epilogue_safe(
- const device U* C,
+ const device DT* C,
const int leading_dimension_c,
const int column_stride_c,
short2 tile_dimensions,
thread const EpilogueOp& epilogue_operation
) {
- const device U* c_pointer = C + simdgroup_row_offset * leading_dimension_c +
- simdgroup_col_offset * column_stride_c;
+ const device DT* c_pointer = C +
+ simdgroup_row_offset * leading_dimension_c +
+ simdgroup_col_offset * column_stride_c;
tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
METAL_PRAGMA_UNROLL
@@ -244,7 +247,7 @@ struct ThreadgroupTile {
k++) {
if (row_offset < tile_dimensions.y &&
col_offset + k < tile_dimensions.x) {
- U c_value = c_pointer
+ DT c_value = c_pointer
[row_offset * leading_dimension_c +
col_offset * column_stride_c + k];
block_data[k] = epilogue_operation.apply(
@@ -257,8 +260,8 @@ struct ThreadgroupTile {
}
}
- METAL_FUNC void apply_bias(const device U* bias) {
- const device U* bias_pointer = bias + simdgroup_col_offset;
+ METAL_FUNC void apply_bias(const device BT* bias) {
+ const device BT* bias_pointer = bias + simdgroup_col_offset;
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < TILE_ROWS; i++) {
@@ -279,10 +282,10 @@ struct ThreadgroupTile {
}
METAL_FUNC void apply_bias_safe(
- const device U* bias,
+ const device BT* bias,
short2 tile_dimensions
) {
- const device U* bias_pointer = bias + simdgroup_col_offset;
+ const device BT* bias_pointer = bias + simdgroup_col_offset;
tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
METAL_PRAGMA_UNROLL
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index b3dcb051c..cca0830a2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -15,7 +15,12 @@ using namespace metal;
namespace uzu {
namespace gemm {
-template <typename T, GemmTiling GEMM_TILING, bool TRANSPOSE_B>
+template <
+ typename AT,
+ typename BT,
+ typename DT,
+ GemmTiling GEMM_TILING,
+ bool TRANSPOSE_B>
struct MxuMmaCore {
METAL_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
METAL_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
@@ -37,13 +42,13 @@ struct MxuMmaCore {
using AccumulatorType = float;
static METAL_FUNC void run(
- const device T* a,
- const device T* b,
- device T* d,
+ const device AT* a,
+ const device BT* b,
+ device DT* d,
const constant uzu::matmul::GemmParams* params,
GemmAlignment alignment,
GemmDTransform output_transform,
- const device T* output_bias,
+ const device BT* output_bias,
const thread ThreadContext& thread_context
) {
(void)output_bias;
@@ -58,8 +63,8 @@ struct MxuMmaCore {
const size_t block_row = size_t(geometry.block_row_start);
const size_t block_col = size_t(geometry.block_col_start);
- const device T* a_block = a + block_row * params->leading_dimension_a;
- const device T* b_block =
+ const device AT* a_block = a + block_row * params->leading_dimension_a;
+ const device BT* b_block =
b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);
const ushort tile_row_offset =
@@ -69,7 +74,7 @@ struct MxuMmaCore {
SIMDGROUP_BLOCK_N *
(thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
- device T* d_simdgroup =
+ device DT* d_simdgroup =
d + block_row * params->leading_dimension_d + block_col +
tile_row_offset * params->leading_dimension_d + tile_col_offset;
@@ -90,9 +95,9 @@ struct MxuMmaCore {
int(geometry.block_col_start + tile_col_offset))
);
- const device T* a_simdgroup =
+ const device AT* a_simdgroup =
a_block + size_t(tile_row_offset) * params->leading_dimension_a;
- const device T* b_simdgroup =
+ const device BT* b_simdgroup =
b_block + (TRANSPOSE_B ? size_t(tile_col_offset) *
int(params->leading_dimension_b)
: size_t(tile_col_offset));
@@ -113,7 +118,8 @@ struct MxuMmaCore {
(simdgroup_limit_n == SIMDGROUP_BLOCK_N),
[&](auto aligned_n) {
auto accumulator_tile = uzu::matmul::gemm_loop<
- T,
+ AT,
+ BT,
SIMDGROUP_BLOCK_M,
SIMDGROUP_BLOCK_N,
SIMDGROUP_BLOCK_K,
@@ -147,7 +153,7 @@ struct MxuMmaCore {
if (apply_accumulate) {
uzu::matmul::Fragment<
- T,
+ DT,
TILES_M,
TILES_N,
uzu::matmul::MxuFragmentOps>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index c860bc745..4f4626f2c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -19,7 +19,9 @@ namespace uzu {
namespace gemm {
template <
- typename T,
+ typename AT,
+ typename BT,
+ typename DT,
GemmTiling GEMM_TILING,
bool TRANSPOSE_B,
GemmWeightPrologueKind WEIGHT_PROLOGUE =
@@ -34,8 +36,8 @@ struct SimdgroupMmaCore {
gemm_tiling_simdgroups_per_row(GEMM_TILING);
METAL_CONST int SIMDGROUPS_PER_COLUMN =
gemm_tiling_simdgroups_per_column(GEMM_TILING);
- METAL_CONST ushort PADDING_A = 16 / sizeof(T);
- METAL_CONST ushort PADDING_B = 16 / sizeof(T);
+ METAL_CONST ushort PADDING_A = 16 / sizeof(AT);
+ METAL_CONST ushort PADDING_B = 16 / sizeof(BT);
METAL_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;
METAL_CONST ushort SHARED_STRIDE_B =
(TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;
@@ -43,21 +45,21 @@ struct SimdgroupMmaCore {
SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
using ALoader = uzu::matmul::ThreadgroupLoader<
- T,
+ AT,
THREADGROUP_BLOCK_M,
THREADGROUP_BLOCK_K,
SHARED_STRIDE_A,
true,
THREADGROUP_THREADS>;
using BLoaderFp = uzu::matmul::ThreadgroupLoader<
- T,
+ BT,
TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,
TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N,
SHARED_STRIDE_B,
TRANSPOSE_B,
THREADGROUP_THREADS>;
using BLoaderScaleBias = QuantizedBlockLoaderScaleBias<
- T,
+ BT,
THREADGROUP_BLOCK_N,
THREADGROUP_BLOCK_K,
SHARED_STRIDE_B,
@@ -66,7 +68,7 @@ struct SimdgroupMmaCore {
GROUP_SIZE,
BITS>;
using BLoaderScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<
- T,
+ BT,
THREADGROUP_BLOCK_N,
THREADGROUP_BLOCK_K,
SHARED_STRIDE_B,
@@ -75,8 +77,9 @@ struct SimdgroupMmaCore {
GROUP_SIZE,
BITS>;
using TileAccumulator = uzu::matmul::ThreadgroupTile<
- T,
- T,
+ AT,
+ BT,
+ DT,
THREADGROUP_BLOCK_M,
THREADGROUP_BLOCK_N,
THREADGROUP_BLOCK_K,
@@ -87,12 +90,12 @@ struct SimdgroupMmaCore {
SHARED_STRIDE_A,
SHARED_STRIDE_B,
float,
- uzu::matmul::TransformNone<T, float>>;
+ uzu::matmul::TransformNone<DT, float>>;
template <uint GEMM_ALIGNMENT_RAW, typename BLoader>
static METAL_FUNC void k_loop(
- threadgroup T* a_shared,
- threadgroup T* b_shared,
+ threadgroup AT* a_shared,
+ threadgroup BT* b_shared,
const int aligned_k_iterations,
thread ALoader& loader_a,
thread BLoader& loader_b,
@@ -147,14 +150,14 @@ struct SimdgroupMmaCore {
template <uint GEMM_ALIGNMENT_RAW>
static METAL_FUNC void finalize(
thread TileAccumulator& accumulator,
- device T* d,
+ device DT* d,
const constant uzu::matmul::GemmParams* params,
const thread ushort& tile_block_rows,
const thread ushort& tile_block_cols,
const bool needs_epilogue,
const thread uzu::matmul::TransformScaleAccumulate<float, float>&
epilogue,
- const device T* bias_block,
+ const device BT* bias_block,
const bool needs_bias
) {
constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};
@@ -194,18 +197,18 @@ struct SimdgroupMmaCore {
}
static METAL_FUNC void run(
- const device T* a,
+ const device AT* a,
const device uint8_t* b_packed,
- device T* d,
+ device DT* d,
const constant uzu::matmul::GemmParams* params,
GemmAlignment alignment,
GemmDTransform output_transform,
- const device T* scales,
- const device T* biases,
+ const device BT* scales,
+ const device BT* biases,
const device uint8_t* zero_points,
- const device T* output_bias,
- threadgroup T* a_shared,
- threadgroup T* b_shared,
+ const device BT* output_bias,
+ threadgroup AT* a_shared,
+ threadgroup BT* b_shared,
const thread ThreadContext& thread_context
) {
const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);
@@ -247,7 +250,7 @@ struct SimdgroupMmaCore {
const float alpha = needs_scale ? params->ab_scale : 1.0f;
const float beta = needs_accumulate ? 1.0f : 0.0f;
uzu::matmul::TransformScaleAccumulate<float, float> epilogue(alpha, beta);
- const device T* bias_block = output_bias + block_col;
+ const device BT* bias_block = output_bias + block_col;
const bool all_aligned = ((alignment.contains(GemmAlignment::M)) ||
(tile_block_rows == THREADGROUP_BLOCK_M)) &&
@@ -277,7 +280,7 @@ struct SimdgroupMmaCore {
};
if constexpr (WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision) {
- const device T* b = reinterpret_cast<const device T*>(b_packed);
+ const device BT* b = reinterpret_cast<const device BT*>(b_packed);
b += TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col;
thread BLoaderFp
loader_b(b, params->leading_dimension_b, b_shared, thread_context);
@@ -318,8 +321,8 @@ struct SimdgroupMmaCore {
const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
const device uint8_t* weights_block =
b_packed + block_col * weights_row_stride_bytes;
- const device T* scales_offset = scales + block_col * groups_per_row;
- const device T* biases_offset = biases + block_col * groups_per_row;
+ const device BT* scales_offset = scales + block_col * groups_per_row;
+ const device BT* biases_offset = biases + block_col * groups_per_row;
thread BLoaderScaleBias loader_b(
weights_block,
scales_offset,
@@ -368,7 +371,7 @@ struct SimdgroupMmaCore {
const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
const device uint8_t* weights_block =
b_packed + block_col * weights_row_stride_bytes;
- const device T* scales_offset = scales + block_col * groups_per_row;
+ const device BT* scales_offset = scales + block_col * groups_per_row;
const int zero_point_stride_per_row =
(BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;
const device uint8_t* zero_points_row_start =
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 3cffd4945..b8c22eab5 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -13,21 +13,25 @@ using namespace uzu::gemm;
#define GEMM_TGA_ELEMENTS \
(USE_MXU ? 1 \
: (gemm_tiling_block_m(GEMM_TILING) * \
- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(T)))))
+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))
#define GEMM_TGB_ELEMENTS \
(USE_MXU ? 1 \
: (gemm_tiling_block_n(GEMM_TILING) * \
- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(T)))))
+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))
template <
- typename T,
+ typename AT,
+ typename BT,
+ typename DT,
GemmTiling GEMM_TILING,
bool TRANSPOSE_B,
bool USE_MXU,
GemmWeightPrologueKind WEIGHT_PROLOGUE,
uint BITS,
uint GROUP_SIZE>
-VARIANTS(T, half, bfloat, float)
+VARIANTS(AT, half, bfloat, float)
+VARIANTS(BT, half, bfloat, float)
+VARIANTS(DT, half, bfloat, float)
VARIANTS(
GEMM_TILING,
GemmTiling::T8x32x32_1x1,
@@ -71,24 +75,24 @@ CONSTRAINT(
GEMM_TILING != GemmTiling::T64x64x64_2x2 ||
GROUP_SIZE == 64 || GROUP_SIZE == 128)
KERNEL(Gemm)(
- const device T* a,
+ const device AT* a,
const device uint8_t* b_packed,
- device T* d,
- const device T* scales
+ device DT* d,
+ const device BT* scales
OPTIONAL(WEIGHT_PROLOGUE != GemmWeightPrologueKind::FullPrecision),
- const device T* biases
+ const device BT* biases
OPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleBiasDequant),
const device uint8_t* zero_points
OPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleZeroPointDequant),
- const device T* output_bias
+ const device BT* output_bias
OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),
const constant uzu::matmul::GemmParams* params,
const constant uint& group_count_x,
const constant uint& group_count_y,
const GemmDTransform output_transform SPECIALIZE,
const GemmAlignment alignment SPECIALIZE,
- threadgroup T a_shared[GEMM_TGA_ELEMENTS],
- threadgroup T b_shared[GEMM_TGB_ELEMENTS],
+ threadgroup AT a_shared[GEMM_TGA_ELEMENTS],
+ threadgroup BT b_shared[GEMM_TGB_ELEMENTS],
const uint group_x GROUPS(group_count_x),
const uint group_y GROUPS(group_count_y),
const uint thread_x THREADS(METAL_SIMD_SIZE),
@@ -106,8 +110,8 @@ KERNEL(Gemm)(
(void)scales;
(void)biases;
(void)zero_points;
- const device T* b = reinterpret_cast<const device T*>(b_packed);
- MxuMmaCore<T, GEMM_TILING, TRANSPOSE_B>::run(
+ const device BT* b = reinterpret_cast<const device BT*>(b_packed);
+ MxuMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B>::run(
a,
b,
d,
@@ -119,7 +123,9 @@ KERNEL(Gemm)(
);
} else {
SimdgroupMmaCore<
- T,
+ AT,
+ BT,
+ DT,
GEMM_TILING,
TRANSPOSE_B,
WEIGHT_PROLOGUE,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 155d38f64..6be34454c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -32,7 +32,9 @@ pub enum GemmDispatchPath {
}
pub struct GemmKernel {
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
kernels: HashMap<GemmSpecialization, GemmMetalKernel>,
pub bias_add: TensorAddBiasMetalKernel,
pub hadamard: <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel,
@@ -41,21 +43,25 @@ pub struct GemmKernel {
impl GemmKernel {
pub(crate) fn new(
context: &MetalContext,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
) -> Result<Self, MetalError> {
- let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true)?;
+ let bias_add = TensorAddBiasMetalKernel::new(context, output_data_type, weights_data_type, true)?;
let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(
context,
- data_type,
+ output_data_type,
HadamardTransformOrder::Output,
)?;
let mut kernel = Self {
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
kernels: HashMap::new(),
bias_add,
hadamard,
};
- for specialization in GemmSpecialization::precompile_configs(data_type) {
+ for specialization in GemmSpecialization::precompile_configs(weights_data_type) {
kernel.get_or_create(context, specialization)?;
}
Ok(kernel)
@@ -66,7 +72,7 @@ impl GemmKernel {
context: &MetalContext,
combo: MatmulQuantCombo,
) -> Result<(), MetalError> {
- for specialization in GemmSpecialization::quant_combo_specs(self.data_type, combo) {
+ for specialization in GemmSpecialization::quant_combo_specs(self.weights_data_type, combo) {
self.get_or_create(context, specialization)?;
}
Ok(())
@@ -82,7 +88,9 @@ impl GemmKernel {
Entry::Vacant(entry) => {
let kernel = GemmMetalKernel::new(
context,
- specialization.data_type,
+ self.input_data_type,
+ self.weights_data_type,
+ self.output_data_type,
specialization.tiling,
specialization.transpose_b,
specialization.use_mxu,
@@ -103,7 +111,9 @@ impl GemmKernel {
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
let path = if encoder.context().device.supports_mxu()
- && matches!(self.data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ && [self.weights_data_type, self.input_data_type, self.output_data_type]
+ .into_iter()
+ .all(|data_type| matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32))
&& matches!(arguments.b, MatmulB::FullPrecision { .. })
{
GemmDispatchPath::Mxu
@@ -126,9 +136,13 @@ impl GemmKernel {
"GemmDispatchPath::Mxu requested on hardware without MXU support",
);
assert!(
- matches!(self.data_type, DataType::F16 | DataType::BF16 | DataType::F32),
- "GemmDispatchPath::Mxu requires F16, BF16, or F32 data type, got {:?}",
- self.data_type,
+ [self.weights_data_type, self.input_data_type, self.output_data_type]
+ .into_iter()
+ .all(|data_type| matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)),
+ "GemmDispatchPath::Mxu requires F16, BF16, or F32 data types, got weights {:?}, input {:?}, output {:?}",
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
);
assert!(
matches!(arguments.b, MatmulB::FullPrecision { .. }),
@@ -254,7 +268,7 @@ impl GemmKernel {
};
let specialization = GemmSpecialization {
- data_type: self.data_type,
+ weights_data_type: self.weights_data_type,
tiling,
use_mxu,
output_transform,
@@ -310,7 +324,7 @@ impl GemmKernel {
let group_count_y = m.div_ceil(tiling.block_m());
let specialization = GemmSpecialization {
- data_type: self.data_type,
+ weights_data_type: self.weights_data_type,
tiling,
use_mxu,
output_transform,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 43e2b80ff..04985b10f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -12,7 +12,7 @@ use crate::{
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(crate) struct GemmSpecialization {
- pub(crate) data_type: DataType,
+ pub(crate) weights_data_type: DataType,
pub(crate) tiling: GemmTiling,
pub(crate) use_mxu: bool,
pub(crate) output_transform: GemmDTransform,
@@ -44,13 +44,13 @@ impl GemmSpecialization {
Ok(())
}
- pub(crate) fn precompile_configs(data_type: DataType) -> Vec<Self> {
+ pub(crate) fn precompile_configs(weights_data_type: DataType) -> Vec<Self> {
let mut out = Vec::new();
- for &tiling in simdgroup_tiling_set(data_type) {
+ for &tiling in simdgroup_tiling_set(weights_data_type) {
for align_mn in [true, false] {
for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {
out.push(Self {
- data_type,
+ weights_data_type,
tiling,
use_mxu: false,
output_transform,
@@ -64,7 +64,7 @@ impl GemmSpecialization {
}
}
- for &tiling in mxu_tiling_set(data_type) {
+ for &tiling in mxu_tiling_set(weights_data_type) {
for align_m in [true, false] {
for align_n in [true, false] {
for align_k in [true, false] {
@@ -75,7 +75,7 @@ impl GemmSpecialization {
GemmDTransform::SCALE | GemmDTransform::ACCUMULATE,
] {
out.push(Self {
- data_type,
+ weights_data_type,
tiling,
use_mxu: true,
output_transform,
@@ -94,7 +94,7 @@ impl GemmSpecialization {
}
pub(crate) fn quant_combo_specs(
- data_type: DataType,
+ weights_data_type: DataType,
combo: MatmulQuantCombo,
) -> Vec<Self> {
let bits = DataType::from(combo.mode).size_in_bits() as u32;
@@ -104,14 +104,14 @@ impl GemmSpecialization {
QuantizationMethod::ScaleZeroPoint => GemmWeightPrologueKind::ScaleZeroPointDequant,
};
let mut out = Vec::new();
- for &tiling in quant_tiling_set(data_type) {
+ for &tiling in quant_tiling_set(weights_data_type) {
if tiling.block_k() > group_size {
continue;
}
for align_n in [true, false] {
for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {
out.push(Self {
- data_type,
+ weights_data_type,
tiling,
use_mxu: false,
output_transform,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 91362ea91..bbf14ffbc 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -13,13 +13,15 @@ using namespace metal;
// → 16 * (16 + 4) = 320. Unused when tg_simd_cols == 1.
#define GEMV_MAX_THREADGROUP_MEMORY 320
-template <typename T>
-VARIANTS(T, half, bfloat, float)
+template <typename WeightT, typename InputT, typename OutputT>
+VARIANTS(WeightT, half, bfloat, float)
+VARIANTS(InputT, half, bfloat, float)
+VARIANTS(OutputT, half, bfloat, float)
KERNEL(MatmulGemv)(
- const device T* matrix,
- const device T* input_vector,
- const device T* output_bias OPTIONAL(is_bias),
- device T* output_vector,
+ const device WeightT* matrix,
+ const device InputT* input_vector,
+ const device WeightT* output_bias OPTIONAL(is_bias),
+ device OutputT* output_vector,
const constant uint& input_dimension,
const constant uint& output_dimension,
const constant uint& matrix_leading_dimension,
@@ -63,7 +65,7 @@ KERNEL(MatmulGemv)(
// Thread local accumulation results
thread float accumulated_values[4] = {0.0f, 0.0f, 0.0f, 0.0f};
- thread T matrix_values[4];
+ thread WeightT matrix_values[4];
thread float vector_coefficients[4];
const uint thread_row_in_simdgroup =
@@ -112,7 +114,7 @@ KERNEL(MatmulGemv)(
? output_row_start
: output_dimension - uint(thread_out_rows);
- const device T* thread_matrix =
+ const device WeightT* thread_matrix =
matrix + output_row_start * matrix_leading_dimension;
const uniform<uint> input_block_stride =
@@ -128,7 +130,7 @@ KERNEL(MatmulGemv)(
++input_block_index) {
// Load vector coefficients (unchecked)
{
- const device T* input_vector_row =
+ const device InputT* input_vector_row =
input_vector + batch_row * input_dimension;
METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
@@ -170,7 +172,7 @@ KERNEL(MatmulGemv)(
if (remaining_input_columns > 0) {
// Load vector coefficients (checked)
{
- const device T* input_vector_row =
+ const device InputT* input_vector_row =
input_vector + batch_row * input_dimension;
if (input_block_col_offset + uint(thread_out_cols) <=
input_vector_length) {
@@ -221,7 +223,7 @@ KERNEL(MatmulGemv)(
? thread_matrix
[output_row_offset * matrix_leading_dimension +
input_block_col_offset + input_col_offset]
- : T(0);
+ : WeightT(0);
}
}
@@ -292,20 +294,25 @@ KERNEL(MatmulGemv)(
// Write outputs
if (simdgroup_col_thread_base == 0 && thread_col_in_simdgroup == 0) {
- device T* output_row_values = output_vector + batch_row * output_dimension;
+ device OutputT* output_row_values =
+ output_vector + batch_row * output_dimension;
METAL_PRAGMA_UNROLL
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
output_row_offset++) {
- T accumulated_c = static_cast<T>(output_scale) *
- static_cast<T>(accumulated_values[output_row_offset]);
+ float accumulated_c =
+ output_scale * accumulated_values[output_row_offset];
if (is_accumulate) {
- accumulated_c +=
- output_row_values[output_row_start + output_row_offset];
+ accumulated_c += static_cast<float>(
+ output_row_values[output_row_start + output_row_offset]
+ );
}
if (is_bias) {
- accumulated_c += output_bias[output_row_start + output_row_offset];
+ accumulated_c += static_cast<float>(
+ output_bias[output_row_start + output_row_offset]
+ );
}
- output_row_values[output_row_start + output_row_offset] = accumulated_c;
+ output_row_values[output_row_start + output_row_offset] =
+ static_cast<OutputT>(accumulated_c);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 03d67e584..100a59aef 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -10,20 +10,26 @@ use crate::{
};
pub(crate) struct GemvKernel {
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
pipelines: HashMap<GemvSpecialization, MatmulGemvMetalKernel>,
}
impl GemvKernel {
pub(crate) fn new(
context: &MetalContext,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
) -> Result<Self, MatmulError<Metal>> {
let mut kernel = Self {
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
pipelines: HashMap::new(),
};
- for &config in GemvSpecialization::precompile_configs(data_type) {
+ for &config in GemvSpecialization::precompile_configs(weights_data_type) {
kernel.get_or_create(context, config)?;
}
Ok(kernel)
@@ -39,7 +45,9 @@ impl GemvKernel {
Entry::Vacant(entry) => {
let kernel = MatmulGemvMetalKernel::new(
context,
- self.data_type,
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
specialization.threadgroup_rows,
specialization.threadgroup_cols,
specialization.threads_per_simdgroup_row,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
index e376972a9..c1da8180f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
@@ -7,17 +7,24 @@
using namespace uzu::quantization_method;
using namespace uzu::gemm;
-template <typename T, uint GROUP_SIZE, uint BITS>
-VARIANTS(T, float, half, bfloat)
+template <
+ typename WeightT,
+ typename InputT,
+ typename OutputT,
+ uint GROUP_SIZE,
+ uint BITS>
+VARIANTS(WeightT, float, half, bfloat)
+VARIANTS(InputT, float, half, bfloat)
+VARIANTS(OutputT, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
PUBLIC KERNEL(QuantizedMatmulQmv)(
const device uint32_t* weights,
- const device T* scales,
+ const device WeightT* scales,
const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),
- const device T* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),
- const device T* input,
- device T* output,
+ const device WeightT* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),
+ const device InputT* input,
+ device OutputT* output,
const constant uint& in_vec_size,
const constant uint& out_vec_size,
const constant uint& batch_size,
@@ -44,7 +51,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
const uint in_vec_size_g = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;
- const device T* scales_base = scales;
+ const device WeightT* scales_base = scales;
const uint out_row =
out_block_idx * (num_simdgroups * results_per_simdgroup) +
simd_group * results_per_simdgroup;
@@ -62,7 +69,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
const uint zp_stride =
BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;
const device uint8_t* zps_row_base = nullptr;
- const device T* biases_row_base = nullptr;
+ const device WeightT* biases_row_base = nullptr;
if (quant_method == QuantizationMethod::ScaleBias) {
biases_row_base = biases + out_row * in_vec_size_g;
} else {
@@ -74,19 +81,19 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
uint k = 0;
for (; k + block_size <= in_vec_size; k += block_size) {
- U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);
+ U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);
for (uint row = 0;
out_row + row < out_vec_size && row < results_per_simdgroup;
row++) {
auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
const uint row_idx = out_row + row;
- const device T* sr = scales_base + row_idx * in_vec_size_g;
+ const device WeightT* sr = scales_base + row_idx * in_vec_size_g;
uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
U s = static_cast<U>(sr[g]);
if (quant_method == QuantizationMethod::ScaleBias) {
- const device T* bl = biases_row_base + row * in_vec_size_g;
+ const device WeightT* bl = biases_row_base + row * in_vec_size_g;
U b = static_cast<U>(bl[g]);
result[row] +=
qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
@@ -114,7 +121,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
? min(in_vec_size - k - thread_offset, values_per_thread)
: 0;
if (remaining > 0) {
- U sum = load_vector_safe<T, U, values_per_thread, BITS>(
+ U sum = load_vector_safe<InputT, U, values_per_thread, BITS>(
input,
x_thread,
remaining
@@ -125,12 +132,12 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
row++) {
auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
const uint row_idx = out_row + row;
- const device T* sr = scales_base + row_idx * in_vec_size_g;
+ const device WeightT* sr = scales_base + row_idx * in_vec_size_g;
uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
U s = static_cast<U>(sr[g]);
if (quant_method == QuantizationMethod::ScaleBias) {
- const device T* bl = biases_row_base + row * in_vec_size_g;
+ const device WeightT* bl = biases_row_base + row * in_vec_size_g;
U b = static_cast<U>(bl[g]);
result[row] +=
qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
@@ -154,7 +161,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
row++) {
result[row] = simd_sum(result[row]);
if (simd_lane == 0) {
- output[row] = static_cast<T>(result[row]);
+ output[row] = static_cast<OutputT>(result[row]);
}
}
} else {
@@ -165,7 +172,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
const uint zp_stride =
BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;
const device uint8_t* zps_row_base = nullptr;
- const device T* biases_row_base = nullptr;
+ const device WeightT* biases_row_base = nullptr;
if (quant_method == QuantizationMethod::ScaleBias) {
biases_row_base = biases + used_out_row * in_vec_size_g;
} else {
@@ -177,17 +184,17 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
uint k = 0;
for (; k + block_size <= in_vec_size; k += block_size) {
- U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);
+ U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);
for (uint row = 0; row < results_per_simdgroup; row++) {
auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
const uint row_idx = used_out_row + row;
- const device T* sr = scales_base + row_idx * in_vec_size_g;
+ const device WeightT* sr = scales_base + row_idx * in_vec_size_g;
uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
U s = static_cast<U>(sr[g]);
if (quant_method == QuantizationMethod::ScaleBias) {
- const device T* bl = biases_row_base + row * in_vec_size_g;
+ const device WeightT* bl = biases_row_base + row * in_vec_size_g;
U b = static_cast<U>(bl[g]);
result[row] +=
qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
@@ -216,7 +223,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
: 0;
if (remaining > 0) {
- U sum = load_vector_safe<T, U, values_per_thread, BITS>(
+ U sum = load_vector_safe<InputT, U, values_per_thread, BITS>(
input,
x_thread,
remaining
@@ -225,12 +232,12 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
for (uint row = 0; row < results_per_simdgroup; row++) {
auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
const uint row_idx = used_out_row + row;
- const device T* sr = scales_base + row_idx * in_vec_size_g;
+ const device WeightT* sr = scales_base + row_idx * in_vec_size_g;
uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
U s = static_cast<U>(sr[g]);
if (quant_method == QuantizationMethod::ScaleBias) {
- const device T* bl = biases_row_base + row * in_vec_size_g;
+ const device WeightT* bl = biases_row_base + row * in_vec_size_g;
U b = static_cast<U>(bl[g]);
result[row] += qdot_safe<U, values_per_thread, BITS>(
wl,
@@ -258,7 +265,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
for (uint row = 0; row < results_per_simdgroup; row++) {
result[row] = simd_sum(result[row]);
if (simd_lane == 0) {
- output[row] = static_cast<T>(result[row]);
+ output[row] = static_cast<OutputT>(result[row]);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
index 8d0fbfb2c..89859f704 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
@@ -8,17 +8,24 @@
using namespace uzu::quantization_method;
using namespace uzu::gemm;
-template <typename T, uint GROUP_SIZE, uint BITS>
-VARIANTS(T, float, half, bfloat)
+template <
+ typename WeightT,
+ typename InputT,
+ typename OutputT,
+ uint GROUP_SIZE,
+ uint BITS>
+VARIANTS(WeightT, float, half, bfloat)
+VARIANTS(InputT, float, half, bfloat)
+VARIANTS(OutputT, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
PUBLIC KERNEL(QuantizedMatmulQmvFast)(
const device uint32_t* weights,
- const device T* scales,
+ const device WeightT* scales,
const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),
- const device T* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),
- const device T* input,
- device T* output,
+ const device WeightT* biases OPTIONAL(quant_method == QuantizationMethod::ScaleBias),
+ const device InputT* input,
+ device OutputT* output,
const device int32_t* hadamard_factors OPTIONAL(use_hadamard),
const constant uint& in_vec_size,
const constant uint& out_vec_size,
@@ -76,7 +83,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
output += batch_idx * out_vec_size + out_row;
for (uint k = 0; k < in_vec_size; k += block_size) {
- U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);
+ U sum = load_vector<InputT, U, values_per_thread, BITS>(input, x_thread);
{
auto wl0 = (const device uint8_t*)(ws);
@@ -179,7 +186,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
if (global_out_idx < out_vec_size) {
output[simd_lane] = simdgroup_output_random_hadamard_transform(
static_cast<ushort>(simd_lane),
- T(shared_results[simd_lane]),
+ OutputT(shared_results[simd_lane]),
hadamard_factors[global_out_idx]
);
}
@@ -187,7 +194,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
} else {
if (simd_lane == 0) {
for (uint row = 0; row < results_per_simdgroup; row++) {
- output[row] = static_cast<T>(result[row]);
+ output[row] = static_cast<OutputT>(result[row]);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
index 322c016b4..c2e4d6b33 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
@@ -26,7 +26,9 @@ struct QmvKey {
}
pub(crate) struct QuantGemvKernel {
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
qmv: HashMap<QmvKey, <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel>,
qmv_fast: HashMap<
QmvKey,
@@ -37,10 +39,14 @@ pub(crate) struct QuantGemvKernel {
impl QuantGemvKernel {
pub(crate) fn new(
_context: &MetalContext,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
) -> Self {
Self {
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
qmv: HashMap::new(),
qmv_fast: HashMap::new(),
}
@@ -126,7 +132,9 @@ impl QuantGemvKernel {
Entry::Vacant(entry) => {
let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
context,
- self.data_type,
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
group_size,
bits,
method,
@@ -156,7 +164,9 @@ impl QuantGemvKernel {
Entry::Vacant(entry) => {
let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
context,
- self.data_type,
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
group_size,
bits,
method,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
index bd985238f..fac0b723c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/spec.rs
@@ -11,9 +11,9 @@ pub struct GemvSpecialization {
}
impl GemvSpecialization {
- pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {
+ pub fn precompile_configs(weights_data_type: crate::DataType) -> &'static [Self] {
use crate::DataType;
- match data_type {
+ match weights_data_type {
DataType::F32 => &[
Self {
threadgroup_rows: 4,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index b94707ccb..9c8015bf2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -140,15 +140,19 @@ impl MatmulKernel for MatmulMetalKernel {
fn new(
context: &MetalContext,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
) -> Result<Self, MetalError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());
+ for data_type in [weights_data_type, input_data_type, output_data_type] {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MatmulError::<Metal>::UnsupportedDataType(data_type).into());
+ }
}
- let gemm = GemmKernel::new(context, data_type)?;
- let gemv = GemvKernel::new(context, data_type)?;
- let quant_gemv = QuantGemvKernel::new(context, data_type);
+ let gemm = GemmKernel::new(context, weights_data_type, input_data_type, output_data_type)?;
+ let gemv = GemvKernel::new(context, weights_data_type, input_data_type, output_data_type)?;
+ let quant_gemv = QuantGemvKernel::new(context, weights_data_type, input_data_type, output_data_type);
Ok(Self {
gemv,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
index 184350e6f..3809cb350 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
@@ -1,11 +1,12 @@
#include <metal_stdlib>
#include "../common/dsl.h"
-template <typename T>
+template <typename T, typename BiasT>
VARIANTS(T, float, half, bfloat)
+VARIANTS(BiasT, float, half, bfloat)
PUBLIC KERNEL(TensorAddBias)(
const device T* input OPTIONAL(!in_place),
- const device T* bias,
+ const device BiasT* bias,
device T* output,
constant uint& num_cols,
constant uint& length,
@@ -17,5 +18,7 @@ PUBLIC KERNEL(TensorAddBias)(
}
uint col = position % num_cols;
- output[position] = input[position] + bias[col];
-}
\ No newline at end of file
+ output[position] = static_cast<T>(
+ static_cast<float>(input[position]) + static_cast<float>(bias[col])
+ );
+}
diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rs
index ce5e3df68..704f9387b 100644
--- a/crates/backend-uzu/src/encodable_block/attention.rs
+++ b/crates/backend-uzu/src/encodable_block/attention.rs
@@ -235,7 +235,9 @@ impl<B: Backend> Attention<B> {
<B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(context, data_type)
.map_err(AttentionError::BackendError)?;
let matmul_kernel =
- <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).ok().map(RefCell::new);
+ <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type, data_type, data_type)
+ .ok()
+ .map(RefCell::new);
let update_kv_cache_kernel =
<B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, false)
.map_err(AttentionError::BackendError)?;
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 1f5a917e0..3f846f377 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -158,8 +158,13 @@ impl<B: Backend> Embedding<B> {
)
.map_err(EmbeddingError::BackendError)?;
let readout = RefCell::new(
- <B::Kernels as ManualKernels>::MatmulKernel::new(context, model_shape.data_type)
- .map_err(EmbeddingError::BackendError)?,
+ <B::Kernels as ManualKernels>::MatmulKernel::new(
+ context,
+ model_shape.data_type,
+ model_shape.data_type,
+ model_shape.data_type,
+ )
+ .map_err(EmbeddingError::BackendError)?,
);
(
@@ -362,8 +367,13 @@ impl<B: Backend> Embedding<B> {
.validate(&[vocab_size as usize, model_dim as usize], model_shape.data_type)?
.read_allocation()?;
let readout = RefCell::new(
- <B::Kernels as ManualKernels>::MatmulKernel::new(context, model_shape.data_type)
- .map_err(EmbeddingError::BackendError)?,
+ <B::Kernels as ManualKernels>::MatmulKernel::new(
+ context,
+ model_shape.data_type,
+ model_shape.data_type,
+ model_shape.data_type,
+ )
+ .map_err(EmbeddingError::BackendError)?,
);
UntiedEmbeddingReadoutType::FullPrecision {
@@ -730,8 +740,8 @@ fn quantized_readout<B: Backend>(
method: QuantizationMethod,
group_size: usize,
) -> Result<(RefCell<<B::Kernels as ManualKernels>::MatmulKernel>, ReadoutQuantConfig), EmbeddingError<B>> {
- let mut readout =
- <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;
+ let mut readout = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type, data_type, data_type)
+ .map_err(EmbeddingError::BackendError)?;
readout
.preheat_quant_combo(
context,
diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rs
index 2d3bfe7b6..afe203009 100644
--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs
@@ -105,8 +105,8 @@ impl<B: Backend> LayerExecutables<B> {
};
#[cfg(feature = "tracing")]
- let tensor_add =
- TensorAddBiasKernel::new(context, data_type, false).map_err(LayerExecutablesError::BackendError)?;
+ let tensor_add = TensorAddBiasKernel::new(context, data_type, data_type, false)
+ .map_err(LayerExecutablesError::BackendError)?;
let mixer_subtree = parameter_tree.subtree("mixer")?;
diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
index 4fccea9ee..da54a4872 100644
--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
@@ -48,7 +48,7 @@ pub struct LinearMatmul<B: Backend> {
biases: Option<Allocation<B>>,
input_dim: usize,
output_dim: usize,
- data_type: DataType,
+ output_data_type: DataType,
mode: Mode<B>,
}
@@ -58,19 +58,31 @@ impl<B: Backend> LinearMatmul<B> {
input_dim: usize,
output_dim: usize,
has_biases: bool,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
parameter_tree: &ParameterTree<B>,
) -> Result<Self, LinearMatmulError<B>> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(LinearMatmulError::UnsupportedDataType(data_type));
+ for data_type in [weights_data_type, input_data_type, output_data_type] {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(LinearMatmulError::UnsupportedDataType(data_type));
+ }
}
- let weights =
- parameter_tree.leaf("weights.weights")?.validate(&[output_dim, input_dim], data_type)?.read_allocation()?;
- let biases = load_biases(data_type, output_dim, has_biases.then_some(parameter_tree))?;
+ let weights = parameter_tree
+ .leaf("weights.weights")?
+ .validate(&[output_dim, input_dim], weights_data_type)?
+ .read_allocation()?;
+ let biases =
+ load_biases(weights_data_type, output_data_type, output_dim, has_biases.then_some(parameter_tree))?;
- let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
- .map_err(LinearMatmulError::BackendError)?;
+ let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(
+ context,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
+ )
+ .map_err(LinearMatmulError::BackendError)?;
Ok(Self {
kernel: RefCell::new(kernel),
@@ -78,7 +90,7 @@ impl<B: Backend> LinearMatmul<B> {
biases,
input_dim,
output_dim,
- data_type,
+ output_data_type,
mode: Mode::FullPrecision,
})
}
@@ -88,7 +100,9 @@ impl<B: Backend> LinearMatmul<B> {
spec: AnyWeightMatrixSpec,
input_dim: usize,
output_dim: usize,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
weights_tree: &ParameterTree<B>,
bias_tree: Option<&ParameterTree<B>>,
output_hadamard_factors: Option<Allocation<B>>,
@@ -120,8 +134,10 @@ impl<B: Backend> LinearMatmul<B> {
},
};
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(LinearMatmulError::UnsupportedDataType(data_type));
+ for data_type in [weights_data_type, input_data_type, output_data_type] {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(LinearMatmulError::UnsupportedDataType(data_type));
+ }
}
let packing_divisor = weight_quantization_mode.packing_divisor();
@@ -132,10 +148,10 @@ impl<B: Backend> LinearMatmul<B> {
.leaf("weights")?
.validate(&[output_dim, input_dim / packing_divisor], storage_type)?
.read_allocation()?;
- let scales = weights_tree.leaf("scales")?.validate(&[output_dim, k_g], data_type)?.read_allocation()?;
+ let scales = weights_tree.leaf("scales")?.validate(&[output_dim, k_g], weights_data_type)?.read_allocation()?;
let zero_points_or_biases = match quantization_method {
QuantizationMethod::ScaleBias => {
- weights_tree.leaf("biases")?.validate(&[output_dim, k_g], data_type)?.read_allocation()?
+ weights_tree.leaf("biases")?.validate(&[output_dim, k_g], weights_data_type)?.read_allocation()?
},
QuantizationMethod::ScaleZeroPoint => {
let expected_zero_points_entries = k_g.div_ceil(packing_divisor);
@@ -146,10 +162,15 @@ impl<B: Backend> LinearMatmul<B> {
},
};
- let biases = load_biases(data_type, output_dim, bias_tree)?;
+ let biases = load_biases(weights_data_type, output_data_type, output_dim, bias_tree)?;
- let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
- .map_err(LinearMatmulError::BackendError)?;
+ let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(
+ context,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
+ )
+ .map_err(LinearMatmulError::BackendError)?;
kernel
.preheat_quant_combo(
context,
@@ -167,7 +188,7 @@ impl<B: Backend> LinearMatmul<B> {
biases,
input_dim,
output_dim,
- data_type,
+ output_data_type,
mode: Mode::Quantized {
method: quantization_method,
mode: weight_quantization_mode,
@@ -181,12 +202,18 @@ impl<B: Backend> LinearMatmul<B> {
}
fn load_biases<B: Backend>(
- data_type: DataType,
+ weights_data_type: DataType,
+ output_data_type: DataType,
output_dim: usize,
parameter_tree: Option<&ParameterTree<B>>,
) -> Result<Option<Allocation<B>>, LinearMatmulError<B>> {
+ if parameter_tree.is_some() && weights_data_type != output_data_type {
+ return Err(LinearMatmulError::UnsupportedConfiguration(format!(
+ "mixed precision linear with biases is not supported: weights={weights_data_type:?}, output={output_data_type:?}",
+ )));
+ }
Ok(parameter_tree
- .map(|tree| tree.leaf("biases")?.validate(&[output_dim], data_type)?.read_allocation())
+ .map(|tree| tree.leaf("biases")?.validate(&[output_dim], weights_data_type)?.read_allocation())
.transpose()?)
}
@@ -197,7 +224,8 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
batch_dim: usize,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.data_type))?;
+ let mut output =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;
let b = match &self.mode {
Mode::FullPrecision => MatmulB::FullPrecision {
diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rs
index 178f45d0b..6b044aabf 100644
--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs
@@ -54,12 +54,14 @@ pub enum OutputHadamardLinearError<B: Backend> {
}
impl<B: Backend> dyn Linear<B> {
- pub fn new<const N: usize>(
+ pub fn new_mixed_precision<const N: usize>(
input_dimension: usize,
output_dimensions: [usize; N],
has_biases: bool,
context: &B::Context,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
parameter_tree: &ParameterTree<B>,
) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
let output_dimension_sum: usize = output_dimensions.iter().sum();
@@ -75,7 +77,9 @@ impl<B: Backend> dyn Linear<B> {
input_dimension,
output_dimension_sum,
has_biases,
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
parameter_tree,
)?;
Ok(Box::new(block))
@@ -90,7 +94,9 @@ impl<B: Backend> dyn Linear<B> {
input_dimension,
output_dimension_sum,
has_biases,
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
parameter_tree,
)?)),
AnyWeightMatrixSpec::HybridSpec(HybridSpec {
@@ -113,7 +119,9 @@ impl<B: Backend> dyn Linear<B> {
incoherence_processing_mode,
input_dimension,
output_dimension_sum,
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
&weights_tree,
)?))
},
@@ -123,7 +131,9 @@ impl<B: Backend> dyn Linear<B> {
spec,
input_dimension,
output_dimension_sum,
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
&weights_tree,
has_biases.then_some(parameter_tree),
None,
@@ -134,14 +144,36 @@ impl<B: Backend> dyn Linear<B> {
}
}
- pub fn new_with_output_hadamard(
+ pub fn new<const N: usize>(
+ input_dimension: usize,
+ output_dimensions: [usize; N],
+ has_biases: bool,
+ context: &B::Context,
+ data_type: DataType,
+ parameter_tree: &ParameterTree<B>,
+ ) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
+ Self::new_mixed_precision(
+ input_dimension,
+ output_dimensions,
+ has_biases,
+ context,
+ data_type,
+ data_type,
+ data_type,
+ parameter_tree,
+ )
+ }
+
+ pub fn new_with_output_hadamard_mixed_precision(
context: &B::Context,
parameter_tree: &ParameterTree<B>,
output_factors: Allocation<B>,
input_dim: usize,
output_dim: usize,
has_biases: bool,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
) -> Result<Box<dyn Linear<B>>, OutputHadamardLinearError<B>> {
let weights_tree = parameter_tree.subtree("weights")?.subtree("quantized")?;
let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
@@ -152,7 +184,9 @@ impl<B: Backend> dyn Linear<B> {
spec,
input_dim,
output_dim,
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
&weights_tree,
has_biases.then_some(parameter_tree),
Some(output_factors),
@@ -164,12 +198,14 @@ impl<B: Backend> dyn Linear<B> {
}
}
- pub fn new_extracting_input_hadamard<const N: usize>(
+ pub fn new_extracting_input_hadamard_mixed_precision<const N: usize>(
input_dimension: usize,
output_dimensions: [usize; N],
has_biases: bool,
context: &B::Context,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
parameter_tree: &ParameterTree<B>,
) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {
let output_dimension_sum: usize = output_dimensions.iter().sum();
@@ -190,19 +226,50 @@ impl<B: Backend> dyn Linear<B> {
.leaf("incoherence_signs.output_signs")?
.validate(&[output_dimension_sum], DataType::I32)?
.read_allocation()?;
- let inner_linear = Self::new_with_output_hadamard(
+ let inner_linear = Self::new_with_output_hadamard_mixed_precision(
context,
parameter_tree,
output_factors,
input_dimension,
output_dimension_sum,
has_biases,
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
)?;
Ok((inner_linear, Some(input_factors)))
} else {
- let linear = Self::new(input_dimension, output_dimensions, has_biases, context, data_type, parameter_tree)?;
+ let linear = Self::new_mixed_precision(
+ input_dimension,
+ output_dimensions,
+ has_biases,
+ context,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
+ parameter_tree,
+ )?;
Ok((linear, None))
}
}
+
+ pub fn new_extracting_input_hadamard<const N: usize>(
+ input_dimension: usize,
+ output_dimensions: [usize; N],
+ has_biases: bool,
+ context: &B::Context,
+ data_type: DataType,
+ parameter_tree: &ParameterTree<B>,
+ ) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {
+ Self::new_extracting_input_hadamard_mixed_precision(
+ input_dimension,
+ output_dimensions,
+ has_biases,
+ context,
+ data_type,
+ data_type,
+ data_type,
+ parameter_tree,
+ )
+ }
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
index 08bc3aab4..793912a5a 100644
--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -37,13 +37,15 @@ pub struct QLoRALinearWrapper<B: Backend> {
base_linear: LinearMatmul<B>,
input_hadamard: Option<(<B::Kernels as Kernels>::HadamardTransformKernel, Allocation<B>)>,
output_hadamard: Option<(<B::Kernels as Kernels>::HadamardTransformKernel, Allocation<B>)>,
- adapter_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
+ adapter_down_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
+ adapter_up_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
adapter_down: Allocation<B>,
adapter_up: Allocation<B>,
input_dim: usize,
output_dim: usize,
lora_rank: usize,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
}
impl<B: Backend> QLoRALinearWrapper<B> {
@@ -55,7 +57,9 @@ impl<B: Backend> QLoRALinearWrapper<B> {
incoherence_processing_mode: IncoherenceProcessingMode,
input_dim: usize,
output_dim: usize,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
weights_tree: &ParameterTree<B>,
) -> Result<Self, QLoRALinearWrapperError<B>> {
let use_incoherence_signs = match (incoherence_block_size, incoherence_processing_mode) {
@@ -74,7 +78,9 @@ impl<B: Backend> QLoRALinearWrapper<B> {
quantization_spec,
input_dim,
output_dim,
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
&quantized_tree,
None,
None,
@@ -93,7 +99,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
Some((
<B::Kernels as Kernels>::HadamardTransformKernel::new(
context,
- data_type,
+ input_data_type,
HadamardTransformOrder::Input,
)
.map_err(QLoRALinearWrapperError::BackendError)?,
@@ -102,7 +108,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
Some((
<B::Kernels as Kernels>::HadamardTransformKernel::new(
context,
- data_type,
+ output_data_type,
HadamardTransformOrder::Output,
)
.map_err(QLoRALinearWrapperError::BackendError)?,
@@ -113,32 +119,48 @@ impl<B: Backend> QLoRALinearWrapper<B> {
(None, None)
};
- let adapter_kernel = RefCell::new(
- <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)
- .map_err(QLoRALinearWrapperError::BackendError)?,
+ let adapter_down_kernel = RefCell::new(
+ <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(
+ context,
+ weights_data_type,
+ input_data_type,
+ weights_data_type,
+ )
+ .map_err(QLoRALinearWrapperError::BackendError)?,
+ );
+ let adapter_up_kernel = RefCell::new(
+ <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(
+ context,
+ weights_data_type,
+ weights_data_type,
+ output_data_type,
+ )
+ .map_err(QLoRALinearWrapperError::BackendError)?,
);
let adapter_down = weights_tree
.leaf("adapter.down_projection")?
- .validate(&[adapter_spec.rank, input_dim], data_type)?
+ .validate(&[adapter_spec.rank, input_dim], weights_data_type)?
.read_allocation()?;
let adapter_up = weights_tree
.leaf("adapter.up_projection")?
- .validate(&[output_dim, adapter_spec.rank], data_type)?
+ .validate(&[output_dim, adapter_spec.rank], weights_data_type)?
.read_allocation()?;
Ok(Self {
base_linear,
input_hadamard,
output_hadamard,
- adapter_kernel,
+ adapter_down_kernel,
+ adapter_up_kernel,
adapter_down,
adapter_up,
input_dim,
output_dim,
lora_rank: adapter_spec.rank,
- data_type,
+ weights_data_type,
+ input_data_type,
})
}
}
@@ -151,10 +173,10 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let mut intermediate =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.data_type))?;
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type))?;
{
- let mut adapter_kernel = self.adapter_kernel.borrow_mut();
+ let mut adapter_kernel = self.adapter_down_kernel.borrow_mut();
adapter_kernel.encode(
MatmulArguments {
a: &input,
@@ -177,7 +199,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
let base_input = if let Some((input_hadamard_kernel, input_factors)) = &self.input_hadamard {
let mut base_input =
- encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dim], self.data_type))?;
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dim], self.input_data_type))?;
encoder.encode_copy(&input, .., &mut base_input, ..);
input_hadamard_kernel.encode(
&mut base_input,
@@ -194,7 +216,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
let mut output = self.base_linear.encode(base_input, batch_dim, encoder)?;
{
- let mut adapter_kernel = self.adapter_kernel.borrow_mut();
+ let mut adapter_kernel = self.adapter_up_kernel.borrow_mut();
adapter_kernel.encode(
MatmulArguments {
a: &intermediate,
diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
index ddcd964ea..bb32a7b0c 100644
--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
@@ -40,7 +40,9 @@ impl<B: Backend> RHTLinearWrapper<B> {
input_dimension: usize,
output_dimension: usize,
has_biases: bool,
- data_type: DataType,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
parameter_tree: &ParameterTree<B>,
) -> Result<Self, RHTLinearWrapperError<B>> {
let weights_tree = parameter_tree.subtree("weights")?;
@@ -63,17 +65,22 @@ impl<B: Backend> RHTLinearWrapper<B> {
.leaf("incoherence_signs.output_signs")?
.validate(&[output_dimension], DataType::I32)?
.read_allocation()?;
- let input_hadamard_kernel =
- <B::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type, HadamardTransformOrder::Input)
- .map_err(RHTLinearWrapperError::BackendError)?;
- let inner_linear = <dyn Linear<B>>::new_with_output_hadamard(
+ let input_hadamard_kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(
+ context,
+ input_data_type,
+ HadamardTransformOrder::Input,
+ )
+ .map_err(RHTLinearWrapperError::BackendError)?;
+ let inner_linear = <dyn Linear<B>>::new_with_output_hadamard_mixed_precision(
context,
parameter_tree,
output_factors,
input_dimension,
output_dimension,
has_biases,
- data_type,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
)?;
Ok(Self {
diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
index fe17cdac3..541bd6a70 100644
--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs
+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
@@ -100,8 +100,10 @@ impl<B: Backend> FishAudioSemanticBridge<B> {
) -> Result<Self, Error> {
let embedding_rows_sum = <B::Kernels as Kernels>::EmbeddingRowsSumKernel::new(context, data_type)
.map_err(unable_to_create_context)?;
- let projection = <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)
- .map_err(unable_to_create_context)?;
+ let projection = <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(
+ context, data_type, data_type, data_type,
+ )
+ .map_err(unable_to_create_context)?;
let codebook_token_indices = context.create_array_zeros(&[num_codebooks], DataType::U32).into_allocation();
Ok(Self {
diff --git a/crates/backend-uzu/src/utils/dispatch_dtype.rs b/crates/backend-uzu/src/utils/dispatch_dtype.rs
new file mode 100644
index 000000000..792d31f14
--- /dev/null
+++ b/crates/backend-uzu/src/utils/dispatch_dtype.rs
@@ -0,0 +1,12 @@
+#[macro_export]
+macro_rules! dispatch_dtype {
+ (|()| $body:expr) => { $body };
+ (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {
+ match $dt {
+ $crate::DataType::F16 => { type $T = half::f16; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
+ $crate::DataType::BF16 => { type $T = half::bf16; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
+ $crate::DataType::F32 => { type $T = f32; $crate::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
+ other => panic!("unsupported dtype in test: {:?}", other),
+ }
+ };
+}
diff --git a/crates/backend-uzu/src/utils/mod.rs b/crates/backend-uzu/src/utils/mod.rs
index f002e61cb..4e523cabc 100644
--- a/crates/backend-uzu/src/utils/mod.rs
+++ b/crates/backend-uzu/src/utils/mod.rs
@@ -1,3 +1,4 @@
+pub mod dispatch_dtype;
pub mod env_utils;
pub mod fs;
pub mod memory;
diff --git a/crates/backend-uzu/tests/common/matmul/harness.rs b/crates/backend-uzu/tests/common/matmul/harness.rs
index 105f169db..f2e96bc80 100644
--- a/crates/backend-uzu/tests/common/matmul/harness.rs
+++ b/crates/backend-uzu/tests/common/matmul/harness.rs
@@ -142,8 +142,13 @@ fn run<B: Backend, T: ArrayElement + Float>(
pub fn cpu_reference<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let context = <Cpu as Backend>::Context::new().expect("CPU context");
- let mut kernel = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
- .expect("CPU MatmulKernel");
+ let mut kernel = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ &context,
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ )
+ .expect("CPU MatmulKernel");
run::<Cpu, T>(&context, &mut kernel, input, |kernel, args, encoder| {
kernel.encode(args, encoder).expect("encode failed");
})
diff --git a/crates/backend-uzu/tests/common/matmul/quant.rs b/crates/backend-uzu/tests/common/matmul/quant.rs
index c6cc2bdfa..bad1def0d 100644
--- a/crates/backend-uzu/tests/common/matmul/quant.rs
+++ b/crates/backend-uzu/tests/common/matmul/quant.rs
@@ -155,8 +155,13 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(
pub fn run_quant_cpu<T: ArrayElement + Float>(input: &QuantInput<T>) -> Vec<T> {
let context = <Cpu as Backend>::Context::new().expect("Cpu context");
let mut buffers = QuantBuffers::<Cpu, T>::allocate(&context, input);
- let mut matmul = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
- .expect("MatmulCpuKernel");
+ let mut matmul = <<Cpu as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ &context,
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ )
+ .expect("MatmulCpuKernel");
let mut encoder = Encoder::<Cpu>::new(&context).expect("encoder");
matmul.encode(quant_arguments(&mut buffers, input), &mut encoder).expect("encode cpu quant");
encoder.end_encoding().submit().wait_until_completed().unwrap();
@@ -170,8 +175,13 @@ pub fn run_quant_metal<T: ArrayElement + Float>(
path: Option<GemmDispatchPath>,
) -> Vec<T> {
let mut buffers = QuantBuffers::<Metal, T>::allocate(context, input);
- let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type())
- .expect("MatmulMetalKernel");
+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ context,
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ )
+ .expect("MatmulMetalKernel");
let mut encoder = Encoder::<Metal>::new(context).expect("encoder");
let args = quant_arguments(&mut buffers, input);
match path {
diff --git a/crates/backend-uzu/tests/common/mod.rs b/crates/backend-uzu/tests/common/mod.rs
index ada88165e..4a6372df1 100644
--- a/crates/backend-uzu/tests/common/mod.rs
+++ b/crates/backend-uzu/tests/common/mod.rs
@@ -8,7 +8,7 @@ pub mod path;
pub mod perf;
pub mod proptest;
-pub(crate) use proptest::{dispatch_dtype, for_each_context};
+pub(crate) use proptest::for_each_context;
pub fn type_short_name<T>() -> &'static str {
std::any::type_name::<T>().rsplit("::").next().unwrap()
diff --git a/crates/backend-uzu/tests/common/proptest.rs b/crates/backend-uzu/tests/common/proptest.rs
index c30ce6138..a5538dca8 100644
--- a/crates/backend-uzu/tests/common/proptest.rs
+++ b/crates/backend-uzu/tests/common/proptest.rs
@@ -74,16 +74,3 @@ impl<T: ComparableTestResults> TestResults<T> {
Ok(())
}
}
-
-macro_rules! dispatch_dtype {
- (|()| $body:expr) => { $body };
- (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {
- match $dt {
- backend_uzu::DataType::F16 => { type $T = half::f16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
- backend_uzu::DataType::BF16 => { type $T = half::bf16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
- backend_uzu::DataType::F32 => { type $T = f32; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
- other => panic!("unsupported dtype in test: {:?}", other),
- }
- };
-}
-pub(crate) use dispatch_dtype;
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs
index 6c091c87e..797c04223 100644
--- a/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs
@@ -28,7 +28,12 @@ use crate::{
fn bench_gemm(c: &mut Criterion) {
let context = MetalContext::new().expect("Metal context");
let mut kernel =
- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())
+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
.expect("MatmulKernel");
let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs
index 55920ac09..b6674cecd 100644
--- a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs
@@ -36,7 +36,13 @@ fn bench_unified_quant_typed<T: ArrayElement + Float>(
let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);
let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);
let mut matmul =
- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type()).unwrap();
+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ context,
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ )
+ .unwrap();
let mut group =
c.benchmark_group(format!("{}/Kernel/UnifiedQuantizedGemm/{}", type_short_name::<Metal>(), label));
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
index 41e15db09..e40fae33a 100644
--- a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
@@ -34,6 +34,8 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
context,
T::data_type(),
+ T::data_type(),
+ T::data_type(),
group_size,
bits,
quant_method,
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs
index 5fe879f7f..d6770cb57 100644
--- a/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs
@@ -38,7 +38,13 @@ fn bench_qwen3_layers_typed<T: ArrayElement + Float>(
let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);
let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);
let mut matmul =
- <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, T::data_type()).unwrap();
+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ context,
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ )
+ .unwrap();
group.throughput(Throughput::Elements((m * n * k) as u64));
group.bench_function(BenchmarkId::from_parameter(format!("{layer}_{shape}")), |b| {
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs
index 1bc168176..f71ae2dea 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs
@@ -75,7 +75,9 @@ fn pipeline_output<B: Backend>(q: &[bf16], k: &[bf16], v: &[bf16], scale: f32) -
.unwrap();
let scatter_values =
<<B as Backend>::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(&ctx, DataType::BF16).unwrap();
- let mut matmul = <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&ctx, DataType::BF16).unwrap();
+ let mut matmul =
+ <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&ctx, DataType::BF16, DataType::BF16, DataType::BF16)
+ .unwrap();
let qa = alloc_allocation_with_data::<B, bf16>(&ctx, q);
let ka = alloc_allocation_with_data::<B, bf16>(&ctx, k);
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
index eb03ae323..13b82d78e 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
@@ -49,7 +49,12 @@ fn run_matrix<T: ArrayElement + Float + Debug + Display>(
tolerance: f32,
) {
let context = MetalContext::new().expect("Metal context");
- let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
+ let mut kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ &context,
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ )
.expect("MatmulKernel");
for path in gemm_paths_for_hw(&context) {
for shape in all_correctness_shapes() {
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
index 948e06985..49fb81038 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
@@ -63,7 +63,12 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
.create_allocation(input.m * input.n * std::mem::size_of::<T>(), AllocationType::Global)
.expect("Failed to create allocation");
- let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
+ let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(
+ &context,
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ )
.expect("Failed to create MatmulKernel");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
index aa37cb960..66b5f3a41 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
@@ -135,7 +135,12 @@ fn parity_bf16_gs32_4bit_mlx_with_bias() {
let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
let bias_pp_buf =
crate::common::helpers::alloc_allocation_with_data::<Metal, bf16>(&context, &bias_t);
- let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())
+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
.expect("MatmulMetalKernel");
let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
@@ -168,7 +173,12 @@ fn quant_gemm_nonzero_b_offset_returns_unsupported_layout() {
let context = MetalContext::new().expect("Metal context");
let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
- let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())
+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
.expect("MatmulMetalKernel");
let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
@@ -192,7 +202,12 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {
let context = MetalContext::new().expect("Metal context");
let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
- let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&context, bf16::data_type())
+ let mut matmul = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
.expect("MatmulMetalKernel");
let mut encoder = Encoder::<Metal>::new(&context).expect("encoder");
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
index 24e497835..a5d7a53e3 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
@@ -34,6 +34,8 @@ fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let kernel = <<Cpu as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
&context,
T::data_type(),
+ T::data_type(),
+ T::data_type(),
input.group_size,
input.bits,
input.quant_method,
@@ -72,6 +74,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
&context,
T::data_type(),
+ T::data_type(),
+ T::data_type(),
input.group_size,
input.bits,
input.quant_method,
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
index 97c109cea..f80e165d9 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
@@ -35,6 +35,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
&context,
T::data_type(),
+ T::data_type(),
+ T::data_type(),
input.group_size,
input.bits,
input.quant_method,
diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs
index e74b751d4..1bf6e5bb6 100644
--- a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs
+++ b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs
@@ -5,7 +5,7 @@ use proptest::prelude::*;
use rand::{RngExt, SeedableRng, rngs::SmallRng};
use rand_distr::Normal;
use backend_uzu::{
- ArrayContextExt, ArrayElement, DataType,
+ ArrayContextExt, ArrayElement, DataType, dispatch_dtype,
backends::common::{
AllocationType, Backend, Context, Encoder, Kernels,
gpu_types::ArgmaxPair,
@@ -15,7 +15,7 @@ use backend_uzu::{
use crate::{
common::{
- dispatch_dtype, for_each_context,
+ for_each_context,
proptest::{ComparableTestResults, TestContextes, kernel_data_type},
},
uzu_test,
@@ -113,4 +113,3 @@ fn test_argmax_prop() {
});
});
}
-
diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs
index 7a499c6ab..11be8260e 100644
--- a/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs
@@ -60,8 +60,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(&context, T::data_type(), in_place)
- .expect("Failed to create TensorAddBiasKernel");
+ let kernel =
+ <<B as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(&context, T::data_type(), T::data_type(), in_place)
+ .expect("Failed to create TensorAddBiasKernel");
let size = input.length as usize;
let input_allocation = (!in_place).then(|| alloc_allocation_with_data::<B, T>(&context, &input.input));