Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

qm(v/m) refactor (#313)

uuuvnede84c7acfe4
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
index f56caf374..63f31070e 100644
--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
@@ -1,14 +1,11 @@
-use std::collections::HashMap;
-
use crate::{
DataType,
backends::common::{
Backend, Encoder, Kernels,
gpu_types::QuantizationMode,
kernel::{
- QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,
+ QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,
QuantizedMatmulQmmTransposedWideKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel,
- QuantizedMatmulQvmKernel,
},
},
};
@@ -21,20 +18,6 @@ pub enum QuantizedMatmulError<B: Backend> {
UnsupportedDataType(DataType),
#[error("Unsupported group size: {0}")]
UnsupportedGroupSize(usize),
- #[error("Unsupported bits: {0}")]
- UnsupportedBits(usize),
- #[error("Value `{name}` does not fit i32: {value}")]
- ValueOutOfRange {
- name: &'static str,
- value: usize,
- },
- #[error("Quantization type mismatch: kernel={kernel:?}, args={args:?}")]
- QuantizationTypeMismatch {
- kernel: QuantizedMatmulType,
- args: QuantizedMatmulType,
- },
- #[error("Missing kernel for key: {0}")]
- MissingKernel(&'static str),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -51,7 +34,6 @@ pub struct QuantizedMatmulConfiguration {
pub output_dim: usize,
pub mode: QuantizationMode,
pub quantization_type: QuantizedMatmulType,
- pub weights_transposed: bool,
}
pub struct QuantizedMatmulArguments<'a, B: Backend> {
@@ -61,102 +43,124 @@ pub struct QuantizedMatmulArguments<'a, B: Backend> {
pub scales_buffer: &'a B::Buffer,
pub zero_points_or_biases_buffer: &'a B::Buffer,
pub output_buffer: &'a mut B::Buffer,
- pub batch: usize,
- pub input_dim: usize,
- pub output_dim: usize,
- pub quantization_type: QuantizedMatmulType,
+ pub batch_dim: usize,
}
pub struct QuantizedMatmulKernelEncodable<B: Backend> {
- kernels: HashMap<KernelKey, EncodableVariant<B>>,
- matrix_vector_key: KernelKey,
- matrix_matrix_key: KernelKey,
+ matrix_vector: MatrixVectorKernel<B>,
+ matrix_matrix: MatrixMatrixKernel<B>,
+ input_dim: usize,
output_dim: usize,
quantization_type: QuantizedMatmulType,
}
-enum EncodableVariant<B: Backend> {
+enum MatrixVectorKernel<B: Backend> {
Qmv(<B::Kernels as Kernels>::QuantizedMatmulQmvKernel),
QmvFast(<B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel),
- Qvm(<B::Kernels as Kernels>::QuantizedMatmulQvmKernel),
- Qmm(<B::Kernels as Kernels>::QuantizedMatmulQmmKernel),
+}
+
+enum MatrixMatrixKernel<B: Backend> {
QmmTransposed(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel),
QmmTransposed64x64(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel),
QmmTransposedWide(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel),
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-enum KernelKey {
- MatrixVector(MatrixVectorFamily),
- MatrixMatrix(MatrixMatrixFamily),
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-enum MatrixVectorFamily {
- Qmv,
- QmvFast,
- Qvm,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-enum MatrixMatrixFamily {
- QmmAlignedK,
- QmmUnalignedK,
- QmmTransposedAlignedN,
- QmmTransposedUnalignedN,
- QmmTransposed64x64,
- QmmTransposedWide,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum RuntimeVariant {
- MatrixVector,
- MatrixMatrix,
-}
-
impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
pub fn new(
context: &B::Context,
configuration: QuantizedMatmulConfiguration,
) -> Result<Self, QuantizedMatmulError<B>> {
- validate_configuration(&configuration)?;
+ if !matches!(configuration.data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(QuantizedMatmulError::UnsupportedDataType(configuration.data_type));
+ }
- let bits = quant_bits(configuration.mode)?;
- let use_mlx_quant = matches!(configuration.quantization_type, QuantizedMatmulType::Mlx);
+ if !matches!(configuration.group_size, 32 | 64 | 128) {
+ return Err(QuantizedMatmulError::UnsupportedGroupSize(configuration.group_size));
+ }
- let matrix_vector_family = select_matrix_vector_family(&configuration);
- let matrix_matrix_family = select_matrix_matrix_family(&configuration, bits);
- let matrix_vector_key = KernelKey::MatrixVector(matrix_vector_family);
- let matrix_matrix_key = KernelKey::MatrixMatrix(matrix_matrix_family);
+ let bits = match configuration.mode {
+ QuantizationMode::UINT4 => 4,
+ QuantizationMode::INT8 | QuantizationMode::UINT8 => 8,
+ };
+ let group_size = configuration.group_size as u32;
+ let use_mlx_quant = matches!(configuration.quantization_type, QuantizedMatmulType::Mlx);
+ let use_zero_points = !use_mlx_quant;
+
+ // Matrix-vector
+ let matrix_vector = if configuration.output_dim % 8 == 0 && configuration.input_dim % 512 == 0 {
+ MatrixVectorKernel::QmvFast(
+ <B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
+ context,
+ configuration.data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )
+ .map_err(QuantizedMatmulError::BackendError)?,
+ )
+ } else {
+ MatrixVectorKernel::Qmv(
+ <B::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
+ context,
+ configuration.data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )
+ .map_err(QuantizedMatmulError::BackendError)?,
+ )
+ };
- let mut kernels = HashMap::new();
- kernels.insert(
- matrix_vector_key,
- create_matrix_vector_kernel(
- context,
- configuration.data_type,
- configuration.group_size,
- bits,
- use_mlx_quant,
- matrix_vector_family,
- )?,
- );
- kernels.insert(
- matrix_matrix_key,
- create_matrix_matrix_kernel(
- context,
- configuration.data_type,
- configuration.group_size,
- bits,
- use_mlx_quant,
- matrix_matrix_family,
- )?,
- );
+ // Matrix-matrix
+ let aligned_n_64 = configuration.output_dim % 64 == 0;
+ let is_bf16 = configuration.data_type == DataType::BF16;
+
+ let matrix_matrix = if aligned_n_64 && is_bf16 && matches!(configuration.group_size, 64 | 128) {
+ MatrixMatrixKernel::QmmTransposed64x64(
+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(
+ context,
+ configuration.data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )
+ .map_err(QuantizedMatmulError::BackendError)?,
+ )
+ } else if aligned_n_64 && is_bf16 {
+ MatrixMatrixKernel::QmmTransposedWide(
+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(
+ context,
+ configuration.data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )
+ .map_err(QuantizedMatmulError::BackendError)?,
+ )
+ } else {
+ let aligned_n = configuration.output_dim % 32 == 0;
+ MatrixMatrixKernel::QmmTransposed(
+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
+ context,
+ configuration.data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ aligned_n,
+ )
+ .map_err(QuantizedMatmulError::BackendError)?,
+ )
+ };
Ok(Self {
- kernels,
- matrix_vector_key,
- matrix_matrix_key,
+ matrix_vector,
+ matrix_matrix,
+ input_dim: configuration.input_dim,
output_dim: configuration.output_dim,
quantization_type: configuration.quantization_type,
})
@@ -167,380 +171,42 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
encoder: &mut Encoder<B>,
arguments: QuantizedMatmulArguments<B>,
) -> Result<(), QuantizedMatmulError<B>> {
- if arguments.quantization_type != self.quantization_type {
- return Err(QuantizedMatmulError::QuantizationTypeMismatch {
- kernel: self.quantization_type,
- args: arguments.quantization_type,
- });
- }
-
- let key = match self.select_runtime_variant(arguments.batch) {
- RuntimeVariant::MatrixVector => self.matrix_vector_key,
- RuntimeVariant::MatrixMatrix => self.matrix_matrix_key,
+ let (zero_points, biases) = match self.quantization_type {
+ QuantizedMatmulType::ZeroPoint => (Some(arguments.zero_points_or_biases_buffer), None),
+ QuantizedMatmulType::Mlx => (None, Some(arguments.zero_points_or_biases_buffer)),
};
- let kernel = self.kernels.get(&key).ok_or(QuantizedMatmulError::MissingKernel(kernel_key_name(key)))?;
- let k = to_i32("input_dim", arguments.input_dim)?;
- let n = to_i32("output_dim", arguments.output_dim)?;
- let m = to_i32("batch", arguments.batch)?;
- let (zero_points, biases) = quant_buffers::<B>(arguments.zero_points_or_biases_buffer, self.quantization_type);
- let a_with_offset = (arguments.a_buffer, arguments.a_offset);
-
- match kernel {
- EncodableVariant::Qmv(kernel) => {
- kernel.encode(
- arguments.b_buffer,
- arguments.scales_buffer,
- zero_points,
- biases,
- a_with_offset,
- arguments.output_buffer,
- k,
- n,
- m,
- encoder,
- );
- },
- EncodableVariant::QmvFast(kernel) => {
- kernel.encode(
- arguments.b_buffer,
- arguments.scales_buffer,
- zero_points,
- biases,
- a_with_offset,
- arguments.output_buffer,
- k,
- n,
- m,
- encoder,
- );
- },
- EncodableVariant::Qvm(kernel) => {
- kernel.encode(
- arguments.b_buffer,
- arguments.scales_buffer,
- zero_points,
- biases,
- a_with_offset,
- arguments.output_buffer,
- k,
- n,
- m,
- encoder,
- );
- },
- EncodableVariant::Qmm(kernel) => {
- kernel.encode(
- arguments.b_buffer,
- arguments.scales_buffer,
- zero_points,
- biases,
- a_with_offset,
- arguments.output_buffer,
- k,
- n,
- m,
- encoder,
- );
- },
- EncodableVariant::QmmTransposed(kernel) => {
- kernel.encode(
- arguments.b_buffer,
- arguments.scales_buffer,
- zero_points,
- biases,
- a_with_offset,
- arguments.output_buffer,
- k,
- n,
- m,
- encoder,
- );
- },
- EncodableVariant::QmmTransposed64x64(kernel) => {
- kernel.encode(
- arguments.b_buffer,
- arguments.scales_buffer,
- zero_points,
- biases,
- a_with_offset,
- arguments.output_buffer,
- k,
- n,
- m,
- encoder,
- );
- },
- EncodableVariant::QmmTransposedWide(kernel) => {
- kernel.encode(
+ macro_rules! encode_kernel {
+ ($kernel:expr) => {
+ $kernel.encode(
arguments.b_buffer,
arguments.scales_buffer,
zero_points,
biases,
- a_with_offset,
+ (arguments.a_buffer, arguments.a_offset),
arguments.output_buffer,
- k,
- n,
- m,
+ self.input_dim as u32,
+ self.output_dim as u32,
+ arguments.batch_dim as u32,
encoder,
- );
- },
+ )
+ };
}
- Ok(())
- }
-
- fn select_runtime_variant(
- &self,
- batch: usize,
- ) -> RuntimeVariant {
- if batch < 32 || self.output_dim == 1 {
- RuntimeVariant::MatrixVector
+ if arguments.batch_dim < 32 || self.output_dim == 1 {
+ match &self.matrix_vector {
+ MatrixVectorKernel::Qmv(k) => encode_kernel!(k),
+ MatrixVectorKernel::QmvFast(k) => encode_kernel!(k),
+ }
} else {
- RuntimeVariant::MatrixMatrix
+ match &self.matrix_matrix {
+ MatrixMatrixKernel::QmmTransposed(k) => encode_kernel!(k),
+ MatrixMatrixKernel::QmmTransposed64x64(k) => encode_kernel!(k),
+ MatrixMatrixKernel::QmmTransposedWide(k) => encode_kernel!(k),
+ }
}
- }
-}
-
-fn create_matrix_vector_kernel<B: Backend>(
- context: &B::Context,
- data_type: DataType,
- group_size: usize,
- bits: usize,
- use_mlx_quant: bool,
- family: MatrixVectorFamily,
-) -> Result<EncodableVariant<B>, QuantizedMatmulError<B>> {
- let group_size = to_i32("group_size", group_size)?;
- let bits = to_i32("bits", bits)?;
- let use_zero_points = !use_mlx_quant;
-
- let kernel = match family {
- MatrixVectorFamily::Qmv => EncodableVariant::Qmv(
- <B::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
- context,
- data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- ),
- MatrixVectorFamily::QmvFast => EncodableVariant::QmvFast(
- <B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
- context,
- data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- ),
- MatrixVectorFamily::Qvm => EncodableVariant::Qvm(
- <B::Kernels as Kernels>::QuantizedMatmulQvmKernel::new(
- context,
- data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- ),
- };
-
- Ok(kernel)
-}
-
-fn create_matrix_matrix_kernel<B: Backend>(
- context: &B::Context,
- data_type: DataType,
- group_size: usize,
- bits: usize,
- use_mlx_quant: bool,
- family: MatrixMatrixFamily,
-) -> Result<EncodableVariant<B>, QuantizedMatmulError<B>> {
- let group_size = to_i32("group_size", group_size)?;
- let bits = to_i32("bits", bits)?;
- let use_zero_points = !use_mlx_quant;
-
- let kernel = match family {
- MatrixMatrixFamily::QmmAlignedK => EncodableVariant::Qmm(
- <B::Kernels as Kernels>::QuantizedMatmulQmmKernel::new(
- context,
- data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- true,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- ),
- MatrixMatrixFamily::QmmUnalignedK => EncodableVariant::Qmm(
- <B::Kernels as Kernels>::QuantizedMatmulQmmKernel::new(
- context,
- data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- false,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- ),
- MatrixMatrixFamily::QmmTransposedAlignedN => EncodableVariant::QmmTransposed(
- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
- context,
- data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- true,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- ),
- MatrixMatrixFamily::QmmTransposedUnalignedN => EncodableVariant::QmmTransposed(
- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
- context,
- data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- false,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- ),
- MatrixMatrixFamily::QmmTransposed64x64 => EncodableVariant::QmmTransposed64x64(
- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(
- context,
- data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- ),
- MatrixMatrixFamily::QmmTransposedWide => EncodableVariant::QmmTransposedWide(
- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(
- context,
- data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- ),
- };
- Ok(kernel)
-}
-
-fn validate_configuration<B: Backend>(
- configuration: &QuantizedMatmulConfiguration
-) -> Result<(), QuantizedMatmulError<B>> {
- if !matches!(configuration.data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(QuantizedMatmulError::UnsupportedDataType(configuration.data_type));
- }
-
- if !matches!(configuration.group_size, 32 | 64 | 128) {
- return Err(QuantizedMatmulError::UnsupportedGroupSize(configuration.group_size));
- }
-
- let _ = quant_bits(configuration.mode)?;
- Ok(())
-}
-
-fn select_matrix_vector_family(configuration: &QuantizedMatmulConfiguration) -> MatrixVectorFamily {
- if configuration.weights_transposed {
- if configuration.output_dim % 8 == 0 && configuration.input_dim % 512 == 0 {
- MatrixVectorFamily::QmvFast
- } else {
- MatrixVectorFamily::Qmv
- }
- } else {
- MatrixVectorFamily::Qvm
- }
-}
-
-fn select_matrix_matrix_family(
- configuration: &QuantizedMatmulConfiguration,
- bits: usize,
-) -> MatrixMatrixFamily {
- if configuration.weights_transposed {
- let aligned_n = configuration.output_dim % 32 == 0;
- let aligned_n_64 = configuration.output_dim % 64 == 0;
- let use_64x64 = aligned_n_64
- && configuration.data_type == DataType::BF16
- && matches!(configuration.group_size, 64 | 128)
- && matches!(bits, 4 | 8);
- let use_wide = aligned_n_64 && configuration.data_type == DataType::BF16 && matches!(bits, 4 | 8) && !use_64x64;
- if use_64x64 {
- MatrixMatrixFamily::QmmTransposed64x64
- } else if use_wide {
- MatrixMatrixFamily::QmmTransposedWide
- } else if aligned_n {
- MatrixMatrixFamily::QmmTransposedAlignedN
- } else {
- MatrixMatrixFamily::QmmTransposedUnalignedN
- }
- } else if configuration.input_dim % 32 == 0 {
- MatrixMatrixFamily::QmmAlignedK
- } else {
- MatrixMatrixFamily::QmmUnalignedK
- }
-}
-
-fn quant_bits<B: Backend>(mode: QuantizationMode) -> Result<usize, QuantizedMatmulError<B>> {
- let bits = match mode {
- QuantizationMode::UINT4 => 4,
- QuantizationMode::INT8 | QuantizationMode::UINT8 => 8,
- };
- if matches!(bits, 4 | 8) {
- Ok(bits)
- } else {
- Err(QuantizedMatmulError::UnsupportedBits(bits))
- }
-}
-
-fn to_i32<B: Backend>(
- name: &'static str,
- value: usize,
-) -> Result<i32, QuantizedMatmulError<B>> {
- i32::try_from(value).map_err(|_| QuantizedMatmulError::ValueOutOfRange {
- name,
- value,
- })
-}
-
-fn quant_buffers<'a, B: Backend>(
- buffer: &'a B::Buffer,
- quantization_type: QuantizedMatmulType,
-) -> (Option<&'a B::Buffer>, Option<&'a B::Buffer>) {
- match quantization_type {
- QuantizedMatmulType::ZeroPoint => (Some(buffer), None),
- QuantizedMatmulType::Mlx => (None, Some(buffer)),
- }
-}
-
-fn kernel_key_name(key: KernelKey) -> &'static str {
- match key {
- KernelKey::MatrixVector(MatrixVectorFamily::Qmv) => "matrix_vector_qmv",
- KernelKey::MatrixVector(MatrixVectorFamily::QmvFast) => "matrix_vector_qmv_fast",
- KernelKey::MatrixVector(MatrixVectorFamily::Qvm) => "matrix_vector_qvm",
- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmAlignedK) => "matrix_matrix_qmm_aligned_k",
- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmUnalignedK) => "matrix_matrix_qmm_unaligned_k",
- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedAlignedN) => "matrix_matrix_qmm_transposed_aligned_n",
- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedUnalignedN) => {
- "matrix_matrix_qmm_transposed_unaligned_n"
- },
- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposed64x64) => "matrix_matrix_qmm_transposed_64x64",
- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedWide) => "matrix_matrix_qmm_transposed_wide",
+ Ok(())
}
}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
index 330ce71e9..84dcec8b9 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
@@ -1,7 +1,5 @@
-pub mod qmm;
pub mod qmm_transposed;
pub mod qmm_transposed_64x64;
pub mod qmm_transposed_wide;
pub mod qmv;
pub mod qmv_fast;
-pub mod qvm;
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rs
deleted file mode 100644
index 823301f3d..000000000
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rs
+++ /dev/null
@@ -1,36 +0,0 @@
-use dsl::kernel;
-use half::{bf16, f16};
-use num_traits::Float;
-
-use crate::ArrayElement;
-
-#[kernel(QuantizedMatmulQmm)]
-#[variants(T, f32, f16, bf16)]
-#[variants(GROUP_SIZE, 32, 64, 128)]
-#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmm<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
- #[allow(unused)] w: *const u32,
- #[allow(unused)] scales: *const T,
- #[allow(unused)]
- #[optional(use_zero_points)]
- zero_points: Option<*const u8>,
- #[allow(unused)]
- #[optional(use_mlx_quant)]
- biases: Option<*const T>,
- #[allow(unused)] x: *const T,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] k: i32,
- #[allow(unused)] n: i32,
- #[allow(unused)] m: i32,
- #[allow(unused)]
- #[specialize]
- use_zero_points: bool,
- #[allow(unused)]
- #[specialize]
- use_mlx_quant: bool,
- #[allow(unused)]
- #[specialize]
- aligned_k: bool,
-) {
- todo!()
-}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
index 62f3f9456..4306fc711 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
@@ -8,8 +8,8 @@ use crate::ArrayElement;
#[variants(T, f32, f16, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
- #[allow(unused)] w: *const u32,
+pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
+ #[allow(unused)] weights: *const u32,
#[allow(unused)] scales: *const T,
#[allow(unused)]
#[optional(use_zero_points)]
@@ -17,11 +17,11 @@ pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE
#[allow(unused)]
#[optional(use_mlx_quant)]
biases: Option<*const T>,
- #[allow(unused)] x: *const T,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] k: i32,
- #[allow(unused)] n: i32,
- #[allow(unused)] m: i32,
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] in_vec_size: u32,
+ #[allow(unused)] out_vec_size: u32,
+ #[allow(unused)] batch_size: u32,
#[allow(unused)]
#[specialize]
use_zero_points: bool,
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs
index e252f69bc..aee61e512 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs
@@ -8,8 +8,8 @@ use crate::ArrayElement;
#[variants(T, bf16)]
#[variants(GROUP_SIZE, 64, 128)]
#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
- #[allow(unused)] w: *const u32,
+pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
+ #[allow(unused)] weights: *const u32,
#[allow(unused)] scales: *const T,
#[allow(unused)]
#[optional(use_zero_points)]
@@ -17,11 +17,11 @@ pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP
#[allow(unused)]
#[optional(use_mlx_quant)]
biases: Option<*const T>,
- #[allow(unused)] x: *const T,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] k: i32,
- #[allow(unused)] n: i32,
- #[allow(unused)] m: i32,
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] in_vec_size: u32,
+ #[allow(unused)] out_vec_size: u32,
+ #[allow(unused)] batch_size: u32,
#[allow(unused)]
#[specialize]
use_zero_points: bool,
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs
index f9cabc631..aa64ed350 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs
@@ -8,8 +8,8 @@ use crate::ArrayElement;
#[variants(T, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
- #[allow(unused)] w: *const u32,
+pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
+ #[allow(unused)] weights: *const u32,
#[allow(unused)] scales: *const T,
#[allow(unused)]
#[optional(use_zero_points)]
@@ -17,11 +17,11 @@ pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP
#[allow(unused)]
#[optional(use_mlx_quant)]
biases: Option<*const T>,
- #[allow(unused)] x: *const T,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] k: i32,
- #[allow(unused)] n: i32,
- #[allow(unused)] m: i32,
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] in_vec_size: u32,
+ #[allow(unused)] out_vec_size: u32,
+ #[allow(unused)] batch_size: u32,
#[allow(unused)]
#[specialize]
use_zero_points: bool,
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs
index 5ae376ea9..7341a22e6 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs
@@ -8,8 +8,8 @@ use crate::ArrayElement;
#[variants(T, f32, f16, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
- #[allow(unused)] w: *const u32,
+pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
+ #[allow(unused)] weights: *const u32,
#[allow(unused)] scales: *const T,
#[allow(unused)]
#[optional(use_zero_points)]
@@ -17,11 +17,11 @@ pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: i32, cons
#[allow(unused)]
#[optional(use_mlx_quant)]
biases: Option<*const T>,
- #[allow(unused)] x: *const T,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] k: i32,
- #[allow(unused)] n: i32,
- #[allow(unused)] m: i32,
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] in_vec_size: u32,
+ #[allow(unused)] out_vec_size: u32,
+ #[allow(unused)] batch_size: u32,
#[allow(unused)]
#[specialize]
use_zero_points: bool,
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
index 33f16dbcc..d5c53856d 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
@@ -8,8 +8,8 @@ use crate::ArrayElement;
#[variants(T, f32, f16, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
- #[allow(unused)] w: *const u32,
+pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
+ #[allow(unused)] weights: *const u32,
#[allow(unused)] scales: *const T,
#[allow(unused)]
#[optional(use_zero_points)]
@@ -17,11 +17,11 @@ pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: i32,
#[allow(unused)]
#[optional(use_mlx_quant)]
biases: Option<*const T>,
- #[allow(unused)] x: *const T,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] k: i32,
- #[allow(unused)] n: i32,
- #[allow(unused)] m: i32,
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] in_vec_size: u32,
+ #[allow(unused)] out_vec_size: u32,
+ #[allow(unused)] batch_size: u32,
#[allow(unused)]
#[specialize]
use_zero_points: bool,
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rs
deleted file mode 100644
index 6eeba6605..000000000
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-use dsl::kernel;
-use half::{bf16, f16};
-use num_traits::Float;
-
-use crate::ArrayElement;
-
-#[kernel(QuantizedMatmulQvm)]
-#[variants(T, f32, f16, bf16)]
-#[variants(GROUP_SIZE, 32, 64, 128)]
-#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qvm<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(
- #[allow(unused)] w: *const u32,
- #[allow(unused)] scales: *const T,
- #[allow(unused)]
- #[optional(use_zero_points)]
- zero_points: Option<*const u8>,
- #[allow(unused)]
- #[optional(use_mlx_quant)]
- biases: Option<*const T>,
- #[allow(unused)] x: *const T,
- #[allow(unused)] y: *mut T,
- #[allow(unused)] k: i32,
- #[allow(unused)] n: i32,
- #[allow(unused)] m: i32,
- #[allow(unused)]
- #[specialize]
- use_zero_points: bool,
- #[allow(unused)]
- #[specialize]
- use_mlx_quant: bool,
-) {
- todo!()
-}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
deleted file mode 100644
index bcdc3b970..000000000
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
+++ /dev/null
@@ -1,113 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-#include "quant_matmul.h"
-
-template <typename T, int GROUP_SIZE, int BITS>
-VARIANTS(T, float, half, bfloat)
-VARIANTS(GROUP_SIZE, 32, 64, 128)
-VARIANTS(BITS, 4, 8)
-PUBLIC KERNEL(QuantizedMatmulQmm)(
- const device uint32_t* w,
- const device T* scales,
- const device uint8_t* zero_points OPTIONAL(use_zero_points),
- const device T* biases OPTIONAL(use_mlx_quant),
- const device T* x,
- device T* y,
- const constant int& k,
- const constant int& n,
- const constant int& m,
- threadgroup T Xs[32 * (32 + 16 / sizeof(T))],
- threadgroup T Ws[32 * (32 + 16 / sizeof(T))],
- const bool use_zero_points SPECIALIZE,
- const bool use_mlx_quant SPECIALIZE,
- const bool aligned_k SPECIALIZE,
- const uint tgid_x GROUPS((n + 32 - 1) / 32),
- const uint tgid_y GROUPS((m + 32 - 1) / 32),
- const uint tgid_z GROUPS(1),
- const uint tid_x THREADS(32),
- const uint tid_y THREADS(2),
- const uint tid_z THREADS(2)
-) {
- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
- const uint lid = tid_z * 64 + tid_y * 32 + tid_x;
- const uint simd_gid = tid_z * 2 + tid_y;
- const uint simd_lid = tid_x;
-
- if (use_mlx_quant) {
- if (aligned_k) {
- qmm_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, true>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- Xs,
- Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
- } else {
- qmm_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, true>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- Xs,
- Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
- }
- } else {
- if (aligned_k) {
- qmm_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, false>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- Xs,
- Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
- } else {
- qmm_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, false>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- Xs,
- Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
index 84b0932b9..9db3a1769 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
@@ -2,111 +2,104 @@
#include "../common/dsl.h"
#include "quant_matmul.h"
-template <typename T, int GROUP_SIZE, int BITS>
+template <typename T, uint GROUP_SIZE, uint BITS>
VARIANTS(T, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
- const device uint32_t* w,
+ const device uint32_t* weights,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
const device T* biases OPTIONAL(use_mlx_quant),
- const device T* x,
- device T* y,
- const constant int& k,
- const constant int& n,
- const constant int& m,
+ const device T* input,
+ device T* output,
+ const constant uint& in_vec_size,
+ const constant uint& out_vec_size,
+ const constant uint& batch_size,
threadgroup T Xs[32 * (32 + 16 / sizeof(T))],
threadgroup T Ws[32 * (32 + 16 / sizeof(T))],
const bool use_zero_points SPECIALIZE,
const bool use_mlx_quant SPECIALIZE,
const bool aligned_n SPECIALIZE,
- const uint tgid_x GROUPS((n + 32 - 1) / 32),
- const uint tgid_y GROUPS((m + 32 - 1) / 32),
- const uint tgid_z GROUPS(1),
- const uint tid_x THREADS(32),
- const uint tid_y THREADS(2),
- const uint tid_z THREADS(2)
+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),
+ const uint batch_block_idx GROUPS(batch_size.div_ceil(32)),
+ const uint simd_lane THREADS(32),
+ const uint simd_group THREADS(4)
) {
- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
- const uint lid = tid_z * 64 + tid_y * 32 + tid_x;
- const uint simd_gid = tid_z * 2 + tid_y;
- const uint simd_lid = tid_x;
-
if (use_mlx_quant) {
if (aligned_n) {
qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, true>(
- w,
+ weights,
scales,
zero_points,
biases,
- x,
- y,
+ input,
+ output,
Xs,
Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ out_block_idx,
+ batch_block_idx,
+ simd_group,
+ simd_lane
);
} else {
qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, true>(
- w,
+ weights,
scales,
zero_points,
biases,
- x,
- y,
+ input,
+ output,
Xs,
Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ out_block_idx,
+ batch_block_idx,
+ simd_group,
+ simd_lane
);
}
} else {
if (aligned_n) {
qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, false>(
- w,
+ weights,
scales,
zero_points,
biases,
- x,
- y,
+ input,
+ output,
Xs,
Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ out_block_idx,
+ batch_block_idx,
+ simd_group,
+ simd_lane
);
} else {
qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, false>(
- w,
+ weights,
scales,
zero_points,
biases,
- x,
- y,
+ input,
+ output,
Xs,
Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ out_block_idx,
+ batch_block_idx,
+ simd_group,
+ simd_lane
);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
index b4af07b58..a11b6c023 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
@@ -2,71 +2,64 @@
#include "../common/dsl.h"
#include "quant_matmul.h"
-template <typename T, int GROUP_SIZE, int BITS>
+template <typename T, uint GROUP_SIZE, uint BITS>
VARIANTS(T, bfloat)
VARIANTS(GROUP_SIZE, 64, 128)
VARIANTS(BITS, 4, 8)
PUBLIC KERNEL(QuantizedMatmulQmmTransposed64x64)(
- const device uint32_t* w,
+ const device uint32_t* weights,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
const device T* biases OPTIONAL(use_mlx_quant),
- const device T* x,
- device T* y,
- const constant int& k,
- const constant int& n,
- const constant int& m,
+ const device T* input,
+ device T* output,
+ const constant uint& in_vec_size,
+ const constant uint& out_vec_size,
+ const constant uint& batch_size,
threadgroup T Xs[64 * (64 + 16 / sizeof(T))],
threadgroup T Ws[64 * (64 + 16 / sizeof(T))],
const bool use_zero_points SPECIALIZE,
const bool use_mlx_quant SPECIALIZE,
- const uint tgid_x GROUPS((n + 64 - 1) / 64),
- const uint tgid_y GROUPS((m + 64 - 1) / 64),
- const uint tgid_z GROUPS(1),
- const uint tid_x THREADS(32),
- const uint tid_y THREADS(2),
- const uint tid_z THREADS(2)
+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(64)),
+ const uint batch_block_idx GROUPS(batch_size.div_ceil(64)),
+ const uint simd_lane THREADS(32),
+ const uint simd_group THREADS(4)
) {
- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
- const uint lid = tid_z * 64 + tid_y * 32 + tid_x;
- const uint simd_gid = tid_z * 2 + tid_y;
- const uint simd_lid = tid_x;
-
if (use_mlx_quant) {
qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, true>(
- w,
+ weights,
scales,
zero_points,
biases,
- x,
- y,
+ input,
+ output,
Xs,
Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ out_block_idx,
+ batch_block_idx,
+ simd_group,
+ simd_lane
);
} else {
qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, false>(
- w,
+ weights,
scales,
zero_points,
biases,
- x,
- y,
+ input,
+ output,
Xs,
Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ out_block_idx,
+ batch_block_idx,
+ simd_group,
+ simd_lane
);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal
index a066b3022..1c6464ba9 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal
@@ -6,71 +6,64 @@
// Processes 4x more output elements per threadgroup than the 32x32 variant.
// BK=32 satisfies the group_size >= BK constraint for all group sizes (32, 64,
// 128).
-template <typename T, int GROUP_SIZE, int BITS>
+template <typename T, uint GROUP_SIZE, uint BITS>
VARIANTS(T, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
PUBLIC KERNEL(QuantizedMatmulQmmTransposedWide)(
- const device uint32_t* w,
+ const device uint32_t* weights,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
const device T* biases OPTIONAL(use_mlx_quant),
- const device T* x,
- device T* y,
- const constant int& k,
- const constant int& n,
- const constant int& m,
+ const device T* input,
+ device T* output,
+ const constant uint& in_vec_size,
+ const constant uint& out_vec_size,
+ const constant uint& batch_size,
threadgroup T Xs[64 * (32 + 16 / sizeof(T))],
threadgroup T Ws[64 * (32 + 16 / sizeof(T))],
const bool use_zero_points SPECIALIZE,
const bool use_mlx_quant SPECIALIZE,
- const uint tgid_x GROUPS((n + 64 - 1) / 64),
- const uint tgid_y GROUPS((m + 64 - 1) / 64),
- const uint tgid_z GROUPS(1),
- const uint tid_x THREADS(32),
- const uint tid_y THREADS(2),
- const uint tid_z THREADS(2)
+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(64)),
+ const uint batch_block_idx GROUPS(batch_size.div_ceil(64)),
+ const uint simd_lane THREADS(32),
+ const uint simd_group THREADS(4)
) {
- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
- const uint lid = tid_z * 64 + tid_y * 32 + tid_x;
- const uint simd_gid = tid_z * 2 + tid_y;
- const uint simd_lid = tid_x;
-
if (use_mlx_quant) {
qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, true>(
- w,
+ weights,
scales,
zero_points,
biases,
- x,
- y,
+ input,
+ output,
Xs,
Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ out_block_idx,
+ batch_block_idx,
+ simd_group,
+ simd_lane
);
} else {
qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, false>(
- w,
+ weights,
scales,
zero_points,
biases,
- x,
- y,
+ input,
+ output,
Xs,
Ws,
- k,
- n,
- m,
- tid,
- lid,
- simd_gid,
- simd_lid
+ in_vec_size,
+ out_vec_size,
+ batch_size,
+ out_block_idx,
+ batch_block_idx,
+ simd_group,
+ simd_lane
);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
index 527711d3a..a35ed5b9d 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
@@ -2,59 +2,260 @@
#include "../common/dsl.h"
#include "quant_matmul.h"
-template <typename T, int GROUP_SIZE, int BITS>
+template <typename T, uint GROUP_SIZE, uint BITS>
VARIANTS(T, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
PUBLIC KERNEL(QuantizedMatmulQmv)(
- const device uint32_t* w,
+ const device uint32_t* weights,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
const device T* biases OPTIONAL(use_mlx_quant),
- const device T* x,
- device T* y,
- const constant int& k,
- const constant int& n,
- const constant int& m,
+ const device T* input,
+ device T* output,
+ const constant uint& in_vec_size,
+ const constant uint& out_vec_size,
+ const constant uint& batch_size,
const bool use_zero_points SPECIALIZE,
const bool use_mlx_quant SPECIALIZE,
- const uint tgid_x GROUPS(m),
- const uint tgid_y GROUPS((n + 8 - 1) / 8),
- const uint tgid_z GROUPS(1),
- const uint tid_x THREADS(32),
- const uint tid_y THREADS(2)
+ const uint batch_idx GROUPS(batch_size),
+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(8)),
+ const uint simd_lane THREADS(32),
+ const uint simd_group THREADS(2)
) {
- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
- const uint simd_gid = tid_y;
- const uint simd_lid = tid_x;
-
- if (use_mlx_quant) {
- qmv_impl<T, GROUP_SIZE, BITS, true>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- k,
- n,
- tid,
- simd_gid,
- simd_lid
- );
+ constexpr uint num_simdgroups = 2;
+ constexpr uint results_per_simdgroup = 4;
+ constexpr uint packs_per_thread = 1;
+ constexpr uint pack_factor = get_pack_factor<BITS, 32>();
+ constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();
+
+ constexpr uint values_per_thread = pack_factor * packs_per_thread;
+ constexpr uint block_size = values_per_thread * 32;
+ constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;
+
+ const device uint8_t* ws = (const device uint8_t*)weights;
+ typedef float U;
+ thread U x_thread[values_per_thread];
+ thread U result[results_per_simdgroup] = {0};
+
+ const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
+ const uint in_vec_size_g = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;
+ const device T* scales_base = scales;
+ const uint out_row =
+ out_block_idx * (num_simdgroups * results_per_simdgroup) +
+ simd_group * results_per_simdgroup;
+ const uint used_out_row = min(out_vec_size - results_per_simdgroup, out_row);
+
+ if (out_row >= out_vec_size) {
+ return;
+ }
+
+ if (out_vec_size < (num_simdgroups * results_per_simdgroup)) {
+ ws +=
+ out_row * in_vec_size_w + simd_lane * packs_per_thread * bytes_per_pack;
+ scales += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;
+
+ const uint zp_stride =
+ BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;
+ const device uint8_t* zps_row_base = nullptr;
+ const device T* biases_row_base = nullptr;
+ if (use_mlx_quant) {
+ biases_row_base = biases + out_row * in_vec_size_g;
+ } else {
+ zps_row_base = zero_points + out_row * zp_stride;
+ }
+
+ input += batch_idx * in_vec_size + simd_lane * values_per_thread;
+ output += batch_idx * out_vec_size + out_row;
+
+ uint k = 0;
+ for (; k + block_size <= in_vec_size; k += block_size) {
+ U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);
+
+ for (uint row = 0;
+ out_row + row < out_vec_size && row < results_per_simdgroup;
+ row++) {
+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
+ const uint row_idx = out_row + row;
+ const device T* sr = scales_base + row_idx * in_vec_size_g;
+
+ uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
+ U s = static_cast<U>(sr[g]);
+ if (use_mlx_quant) {
+ const device T* bl = biases_row_base + row * in_vec_size_g;
+ U b = static_cast<U>(bl[g]);
+ result[row] +=
+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
+ } else {
+ const device uint8_t* zl = zps_row_base + row * zp_stride;
+ U zp;
+ if (BITS == 4) {
+ uint8_t zp_b = zl[g >> 1];
+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
+ } else {
+ zp = static_cast<U>(zl[g]);
+ }
+ result[row] +=
+ qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);
+ }
+ }
+
+ ws += block_size * bytes_per_pack / pack_factor;
+ scales += block_size / GROUP_SIZE;
+ input += block_size;
+ }
+ const uint thread_offset = simd_lane * values_per_thread;
+ const int remaining =
+ (k + thread_offset < in_vec_size)
+ ? min(in_vec_size - k - thread_offset, values_per_thread)
+ : 0;
+ if (remaining > 0) {
+ U sum = load_vector_safe<T, U, values_per_thread, BITS>(
+ input,
+ x_thread,
+ remaining
+ );
+
+ for (uint row = 0;
+ out_row + row < out_vec_size && row < results_per_simdgroup;
+ row++) {
+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
+ const uint row_idx = out_row + row;
+ const device T* sr = scales_base + row_idx * in_vec_size_g;
+
+ uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
+ U s = static_cast<U>(sr[g]);
+ if (use_mlx_quant) {
+ const device T* bl = biases_row_base + row * in_vec_size_g;
+ U b = static_cast<U>(bl[g]);
+ result[row] +=
+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
+ } else {
+ const device uint8_t* zl = zps_row_base + row * zp_stride;
+ U zp;
+ if (BITS == 4) {
+ uint8_t zp_b = zl[g >> 1];
+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
+ } else {
+ zp = static_cast<U>(zl[g]);
+ }
+ result[row] +=
+ qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);
+ }
+ }
+ }
+
+ for (uint row = 0;
+ out_row + row < out_vec_size && row < results_per_simdgroup;
+ row++) {
+ result[row] = simd_sum(result[row]);
+ if (simd_lane == 0) {
+ output[row] = static_cast<T>(result[row]);
+ }
+ }
} else {
- qmv_impl<T, GROUP_SIZE, BITS, false>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- k,
- n,
- tid,
- simd_gid,
- simd_lid
- );
+ ws += used_out_row * in_vec_size_w +
+ simd_lane * packs_per_thread * bytes_per_pack;
+ scales += used_out_row * in_vec_size_g + simd_lane / scale_step_per_thread;
+
+ const uint zp_stride =
+ BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;
+ const device uint8_t* zps_row_base = nullptr;
+ const device T* biases_row_base = nullptr;
+ if (use_mlx_quant) {
+ biases_row_base = biases + used_out_row * in_vec_size_g;
+ } else {
+ zps_row_base = zero_points + used_out_row * zp_stride;
+ }
+
+ input += batch_idx * in_vec_size + simd_lane * values_per_thread;
+ output += batch_idx * out_vec_size + used_out_row;
+
+ uint k = 0;
+ for (; k + block_size <= in_vec_size; k += block_size) {
+ U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);
+
+ for (uint row = 0; row < results_per_simdgroup; row++) {
+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
+ const uint row_idx = used_out_row + row;
+ const device T* sr = scales_base + row_idx * in_vec_size_g;
+
+ uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
+ U s = static_cast<U>(sr[g]);
+ if (use_mlx_quant) {
+ const device T* bl = biases_row_base + row * in_vec_size_g;
+ U b = static_cast<U>(bl[g]);
+ result[row] +=
+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);
+ } else {
+ const device uint8_t* zl = zps_row_base + row * zp_stride;
+ U zp;
+ if (BITS == 4) {
+ uint8_t zp_b = zl[g >> 1];
+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
+ } else {
+ zp = static_cast<U>(zl[g]);
+ }
+ result[row] +=
+ qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);
+ }
+ }
+
+ ws += block_size * bytes_per_pack / pack_factor;
+ scales += block_size / GROUP_SIZE;
+ input += block_size;
+ }
+ const uint thread_offset = simd_lane * values_per_thread;
+ const int remaining =
+ (k + thread_offset < in_vec_size)
+ ? min(in_vec_size - k - thread_offset, values_per_thread)
+ : 0;
+
+ if (remaining > 0) {
+ U sum = load_vector_safe<T, U, values_per_thread, BITS>(
+ input,
+ x_thread,
+ remaining
+ );
+
+ for (uint row = 0; row < results_per_simdgroup; row++) {
+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
+ const uint row_idx = used_out_row + row;
+ const device T* sr = scales_base + row_idx * in_vec_size_g;
+
+ uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;
+ U s = static_cast<U>(sr[g]);
+ if (use_mlx_quant) {
+ const device T* bl = biases_row_base + row * in_vec_size_g;
+ U b = static_cast<U>(bl[g]);
+ result[row] += qdot_safe<U, values_per_thread, BITS>(
+ wl,
+ x_thread,
+ s,
+ b,
+ sum,
+ remaining
+ );
+ } else {
+ const device uint8_t* zl = zps_row_base + row * zp_stride;
+ U zp;
+ if (BITS == 4) {
+ uint8_t zp_b = zl[g >> 1];
+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
+ } else {
+ zp = static_cast<U>(zl[g]);
+ }
+ result[row] +=
+ qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);
+ }
+ }
+ }
+
+ for (uint row = 0; row < results_per_simdgroup; row++) {
+ result[row] = simd_sum(result[row]);
+ if (simd_lane == 0) {
+ output[row] = static_cast<T>(result[row]);
+ }
+ }
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
index db8eda84f..129af75b7 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
@@ -2,59 +2,150 @@
#include "../common/dsl.h"
#include "quant_matmul.h"
-template <typename T, int GROUP_SIZE, int BITS>
+template <typename T, uint GROUP_SIZE, uint BITS>
VARIANTS(T, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
PUBLIC KERNEL(QuantizedMatmulQmvFast)(
- const device uint32_t* w,
+ const device uint32_t* weights,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
const device T* biases OPTIONAL(use_mlx_quant),
- const device T* x,
- device T* y,
- const constant int& k,
- const constant int& n,
- const constant int& m,
+ const device T* input,
+ device T* output,
+ const constant uint& in_vec_size,
+ const constant uint& out_vec_size,
+ const constant uint& batch_size,
const bool use_zero_points SPECIALIZE,
const bool use_mlx_quant SPECIALIZE,
- const uint tgid_x GROUPS(m),
- const uint tgid_y GROUPS((n + 32 - 1) / 32),
- const uint tgid_z GROUPS(1),
- const uint tid_x THREADS(32),
- const uint tid_y THREADS(8)
+ const uint batch_idx GROUPS(batch_size),
+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),
+ const uint simd_lane THREADS(32),
+ const uint simd_group THREADS(8)
) {
- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
- const uint simd_gid = tid_y;
- const uint simd_lid = tid_x;
+ constexpr uint packs_per_thread = BITS == 2 ? 1 : 2;
+ constexpr uint num_simdgroups = 8;
+ constexpr uint results_per_simdgroup = 4;
+ constexpr uint pack_factor = get_pack_factor<BITS, 32>();
+ constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();
+ constexpr uint values_per_thread = pack_factor * packs_per_thread;
+ constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;
+ constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;
+ const device uint8_t* ws = (const device uint8_t*)weights;
+ typedef float U;
+ thread U x_thread[values_per_thread];
+ thread U result[results_per_simdgroup] = {0};
+
+ const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
+ const uint in_vec_size_g = in_vec_size / GROUP_SIZE;
+ const uint out_row =
+ out_block_idx * (num_simdgroups * results_per_simdgroup) +
+ simd_group * results_per_simdgroup;
+ ws += out_row * in_vec_size_w + simd_lane * packs_per_thread * bytes_per_pack;
+ scales += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;
+
+ uint zp_stride = 0;
+ const device uint8_t* zps = nullptr;
+ bool high_nibble = false;
if (use_mlx_quant) {
- qmv_fast_impl<T, GROUP_SIZE, BITS, true>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- k,
- n,
- tid,
- simd_gid,
- simd_lid
- );
+ biases += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;
} else {
- qmv_fast_impl<T, GROUP_SIZE, BITS, false>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- k,
- n,
- tid,
- simd_gid,
- simd_lid
- );
+ if (BITS == 4) {
+ zp_stride = (in_vec_size_g + 1) / 2;
+ zps = zero_points + out_row * zp_stride;
+ uint g_offset = simd_lane / scale_step_per_thread;
+ zps += g_offset / 2;
+ high_nibble = (g_offset & 1);
+ } else {
+ zp_stride = in_vec_size_g;
+ zps = zero_points + out_row * zp_stride;
+ zps += simd_lane / scale_step_per_thread;
+ }
+ }
+
+ input += batch_idx * in_vec_size + simd_lane * values_per_thread;
+ output += batch_idx * out_vec_size + out_row;
+
+ for (uint k = 0; k < in_vec_size; k += block_size) {
+ U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);
+
+ {
+ auto wl0 = (const device uint8_t*)(ws);
+ auto wl1 = (const device uint8_t*)(ws + in_vec_size_w);
+ auto wl2 = (const device uint8_t*)(ws + 2 * in_vec_size_w);
+ auto wl3 = (const device uint8_t*)(ws + 3 * in_vec_size_w);
+
+ U s0 = static_cast<U>(scales[0]);
+ U s1 = static_cast<U>(scales[in_vec_size_g]);
+ U s2 = static_cast<U>(scales[2 * in_vec_size_g]);
+ U s3 = static_cast<U>(scales[3 * in_vec_size_g]);
+
+ if (use_mlx_quant) {
+ U b0 = static_cast<U>(biases[0]);
+ U b1 = static_cast<U>(biases[in_vec_size_g]);
+ U b2 = static_cast<U>(biases[2 * in_vec_size_g]);
+ U b3 = static_cast<U>(biases[3 * in_vec_size_g]);
+ result[0] +=
+ qdot<U, values_per_thread, BITS>(wl0, x_thread, s0, b0, sum);
+ result[1] +=
+ qdot<U, values_per_thread, BITS>(wl1, x_thread, s1, b1, sum);
+ result[2] +=
+ qdot<U, values_per_thread, BITS>(wl2, x_thread, s2, b2, sum);
+ result[3] +=
+ qdot<U, values_per_thread, BITS>(wl3, x_thread, s3, b3, sum);
+ } else {
+ uint8_t zp_byte0 = zps[0];
+ uint8_t zp_byte1 = zps[zp_stride];
+ uint8_t zp_byte2 = zps[2 * zp_stride];
+ uint8_t zp_byte3 = zps[3 * zp_stride];
+ U zp0 = static_cast<U>(
+ (BITS == 4 && high_nibble) ? (zp_byte0 >> 4) : (zp_byte0 & 0x0F)
+ );
+ U zp1 = static_cast<U>(
+ (BITS == 4 && high_nibble) ? (zp_byte1 >> 4) : (zp_byte1 & 0x0F)
+ );
+ U zp2 = static_cast<U>(
+ (BITS == 4 && high_nibble) ? (zp_byte2 >> 4) : (zp_byte2 & 0x0F)
+ );
+ U zp3 = static_cast<U>(
+ (BITS == 4 && high_nibble) ? (zp_byte3 >> 4) : (zp_byte3 & 0x0F)
+ );
+ if (BITS == 8) {
+ zp0 = static_cast<U>(zp_byte0);
+ zp1 = static_cast<U>(zp_byte1);
+ zp2 = static_cast<U>(zp_byte2);
+ zp3 = static_cast<U>(zp_byte3);
+ }
+ result[0] +=
+ qdot_zero_point<U, values_per_thread, BITS>(wl0, x_thread, s0, zp0);
+ result[1] +=
+ qdot_zero_point<U, values_per_thread, BITS>(wl1, x_thread, s1, zp1);
+ result[2] +=
+ qdot_zero_point<U, values_per_thread, BITS>(wl2, x_thread, s2, zp2);
+ result[3] +=
+ qdot_zero_point<U, values_per_thread, BITS>(wl3, x_thread, s3, zp3);
+ }
+ }
+
+ ws += block_size * bytes_per_pack / pack_factor;
+ scales += block_size / GROUP_SIZE;
+ if (use_mlx_quant) {
+ biases += block_size / GROUP_SIZE;
+ } else {
+ if (BITS == 4) {
+ zps += (block_size / GROUP_SIZE) / 2;
+ } else {
+ zps += block_size / GROUP_SIZE;
+ }
+ }
+ input += block_size;
+ }
+
+ for (uint row = 0; row < results_per_simdgroup; row++) {
+ result[row] = simd_sum(result[row]);
+ if (simd_lane == 0) {
+ output[row] = static_cast<T>(result[row]);
+ }
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
index 6da2bfedf..a27c349e9 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
@@ -1,3 +1,5 @@
+#pragma once
+
#include <metal_simdgroup>
#include <metal_stdlib>
@@ -627,228 +629,6 @@ struct QuantizedBlockLoaderZp {
}
};
-template <
- typename LoaderW,
- typename LoaderX,
- typename Mma,
- typename T,
- const bool aligned_K,
- const int BM,
- const int BK,
- const int BN>
-inline void qmm_core(
- thread LoaderX& loader_x,
- thread LoaderW& loader_w,
- thread Mma& mma_op,
- const short num_els,
- const short num_outs,
- const int K,
- device T* y,
- const constant int& N,
- threadgroup T* Xs,
- threadgroup T* Ws,
- uint lid [[thread_index_in_threadgroup]]
-) {
- if (num_els < BM) {
- if ((K % BK) != 0) {
- const int k_blocks = K / BK;
- for (int k = 0; k < k_blocks; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_safe(short2(BK, num_els));
- loader_w.load_unsafe();
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(Xs, Ws);
- loader_x.next();
- loader_w.next();
- }
- const short num_k = K - k_blocks * BK;
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_safe(short2(num_k, num_els));
- loader_w.load_safe(short2(BN, num_k));
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(Xs, Ws);
- } else {
- for (int k = 0; k < K; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_safe(short2(BK, num_els));
- loader_w.load_unsafe();
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(Xs, Ws);
- loader_x.next();
- loader_w.next();
- }
- }
- } else {
- if ((K % BK) != 0) {
- const int k_blocks = K / BK;
- for (int k = 0; k < k_blocks; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_unsafe();
- loader_w.load_unsafe();
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(Xs, Ws);
- loader_x.next();
- loader_w.next();
- }
- const short num_k = K - k_blocks * BK;
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_safe(short2(num_k, BM));
- loader_w.load_safe(short2(BN, num_k));
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(Xs, Ws);
- } else {
- for (int k = 0; k < K; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_unsafe();
- loader_w.load_unsafe();
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(Xs, Ws);
- loader_x.next();
- loader_w.next();
- }
- }
- }
-
- // Store results to device memory
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (num_els < BM) {
- mma_op.store_result_safe(y, N, short2(BN, num_els));
- } else {
- mma_op.store_result(y, N);
- }
-}
-
-template <
- typename T,
- const int group_size,
- const int bits,
- const bool aligned_K = false,
- const int BM = 32,
- const int BK = 32,
- const int BN = 32,
- const bool use_mlx_quant = false>
-void qmm_impl(
- const device uint32_t* w,
- const device T* scales,
- const device uint8_t* zero_points,
- const device T* biases,
- const device T* x,
- device T* y,
- threadgroup T* Xs,
- threadgroup T* Ws,
- const constant int& K,
- const constant int& N,
- const constant int& M,
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");
- static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");
-
- (void)lid;
-
- constexpr int WM = 2;
- constexpr int WN = 2;
- constexpr int pack_factor = get_pack_factor<bits, 8>();
- constexpr int bytes_per_pack = get_bytes_per_pack<bits>();
-
- constexpr int BK_padded = (BK + 16 / sizeof(T));
- constexpr int BN_padded = (BN + 16 / sizeof(T));
-
- using mma_t = matmul_utils::
- BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;
- using loader_x_t =
- matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;
-
- auto wl = (const device uint8_t*)w;
-
- const int y_row = tid.y * BM;
- const int y_col = tid.x * BN;
- x += y_row * static_cast<int64_t>(K);
- wl += y_col * bytes_per_pack / pack_factor;
- y += y_row * static_cast<int64_t>(N) + y_col;
-
- const device T* scales_base = scales;
- const device T* biases_base = biases;
-
- const short num_els = min(BM, M - y_row);
- const short num_outs = min(BN, N - y_col);
- loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);
- mma_t mma_op(simd_gid, simd_lid);
-
- if (use_mlx_quant) {
- using loader_w_t = QuantizedBlockLoaderMlx<
- T,
- BK,
- BN,
- BN_padded,
- 0,
- WM * WN * 32,
- group_size,
- bits>;
- const device T* scales_mlx = scales_base + (y_col / group_size);
- const device T* biases_mlx = biases_base + (y_col / group_size);
- loader_w_t loader_w(wl, scales_mlx, biases_mlx, N, Ws, simd_gid, simd_lid);
- qmm_core<loader_w_t, loader_x_t, mma_t, T, aligned_K, BM, BK, BN>(
- loader_x,
- loader_w,
- mma_op,
- num_els,
- num_outs,
- K,
- y,
- N,
- Xs,
- Ws,
- lid
- );
- } else {
- const int out_groups_total = (N + group_size - 1) / group_size;
- const int groups_per_row = out_groups_total;
- const int out_group = y_col / group_size;
- const int zp_stride_out =
- (bits == 4) ? ((out_groups_total + 1) / 2) : out_groups_total;
- using loader_w_t = QuantizedBlockLoaderZp<
- T,
- BK,
- BN,
- BN_padded,
- 0,
- WM * WN * 32,
- group_size,
- bits,
- true>;
- loader_w_t loader_w(
- wl,
- scales_base,
- zero_points,
- N,
- groups_per_row,
- Ws,
- simd_gid,
- simd_lid,
- out_group,
- out_groups_total,
- zp_stride_out
- );
- qmm_core<loader_w_t, loader_x_t, mma_t, T, aligned_K, BM, BK, BN>(
- loader_x,
- loader_w,
- mma_op,
- num_els,
- num_outs,
- K,
- y,
- N,
- Xs,
- Ws,
- lid
- );
- }
-}
-
template <
typename LoaderW,
typename LoaderX,
@@ -864,16 +644,15 @@ inline void qmm_transposed_core(
thread Mma& mma_op,
const short num_els,
const short num_outs,
- const int K,
- device T* y,
- const constant int& N,
+ const int in_vec_size,
+ device T* output,
+ const int out_vec_size,
threadgroup T* Xs,
- threadgroup T* Ws,
- uint lid [[thread_index_in_threadgroup]]
+ threadgroup T* Ws
) {
if (num_els < BM) {
if (!aligned_N && num_outs < BN) {
- for (int k = 0; k < K; k += BK) {
+ for (int k = 0; k < in_vec_size; k += BK) {
threadgroup_barrier(mem_flags::mem_threadgroup);
loader_x.load_safe(short2(BK, num_els));
loader_w.load_safe(short2(BK, num_outs));
@@ -883,7 +662,7 @@ inline void qmm_transposed_core(
loader_w.next();
}
} else {
- for (int k = 0; k < K; k += BK) {
+ for (int k = 0; k < in_vec_size; k += BK) {
threadgroup_barrier(mem_flags::mem_threadgroup);
loader_x.load_safe(short2(BK, num_els));
loader_w.load_unsafe();
@@ -895,7 +674,7 @@ inline void qmm_transposed_core(
}
} else {
if (!aligned_N && num_outs < BN) {
- for (int k = 0; k < K; k += BK) {
+ for (int k = 0; k < in_vec_size; k += BK) {
threadgroup_barrier(mem_flags::mem_threadgroup);
loader_x.load_unsafe();
loader_w.load_safe(short2(BK, num_outs));
@@ -905,7 +684,7 @@ inline void qmm_transposed_core(
loader_w.next();
}
} else {
- for (int k = 0; k < K; k += BK) {
+ for (int k = 0; k < in_vec_size; k += BK) {
threadgroup_barrier(mem_flags::mem_threadgroup);
loader_x.load_unsafe();
loader_w.load_unsafe();
@@ -921,37 +700,37 @@ inline void qmm_transposed_core(
// Store results to device memory
threadgroup_barrier(mem_flags::mem_threadgroup);
if (num_els < BM || num_outs < BN) {
- mma_op.store_result_safe(y, N, short2(num_outs, num_els));
+ mma_op.store_result_safe(output, out_vec_size, short2(num_outs, num_els));
} else {
- mma_op.store_result(y, N);
+ mma_op.store_result(output, out_vec_size);
}
}
template <
typename T,
- const int group_size,
- const int bits,
+ const uint group_size,
+ const uint bits,
const bool aligned_N,
const int BM = 32,
const int BK = 32,
const int BN = 32,
const bool use_mlx_quant = false>
void qmm_transposed_impl(
- const device uint32_t* w,
+ const device uint32_t* weights,
const device T* scales,
const device uint8_t* zero_points,
const device T* biases,
- const device T* x,
- device T* y,
+ const device T* input,
+ device T* output,
threadgroup T* Xs,
threadgroup T* Ws,
- const constant int& K,
- const constant int& N,
- const constant int& M,
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
+ const int in_vec_size,
+ const int out_vec_size,
+ const int batch_size,
+ uint out_block_idx,
+ uint batch_block_idx,
+ uint simd_group,
+ uint simd_lane
) {
static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");
static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");
@@ -967,23 +746,24 @@ void qmm_transposed_impl(
using loader_x_t =
matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32>;
- const int K_w = K * bytes_per_pack / pack_factor;
- const int K_g = (K + group_size - 1) / group_size;
- const int y_row = tid.y * BM;
- const int y_col = tid.x * BN;
+ const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
+ const int in_vec_size_g = (in_vec_size + group_size - 1) / group_size;
+ const int out_row = batch_block_idx * BM;
+ const int out_col = out_block_idx * BN;
- auto wl = (const device uint8_t*)w;
+ auto wl = (const device uint8_t*)weights;
- const device T* x_block = x + y_row * static_cast<int64_t>(K);
- const device uint8_t* w_block = wl + y_col * K_w;
- scales += y_col * K_g;
- biases += y_col * K_g;
- device T* y_block = y + y_row * static_cast<int64_t>(N) + y_col;
+ const device T* x_block = input + out_row * static_cast<int64_t>(in_vec_size);
+ const device uint8_t* w_block = wl + out_col * in_vec_size_w;
+ scales += out_col * in_vec_size_g;
+ biases += out_col * in_vec_size_g;
+ device T* y_block =
+ output + out_row * static_cast<int64_t>(out_vec_size) + out_col;
- const short num_els = min(BM, M - y_row);
- const short num_outs = min(BN, N - y_col);
- loader_x_t loader_x(x_block, K, Xs, simd_gid, simd_lid);
- mma_t mma_op(simd_gid, simd_lid);
+ const short num_els = min(BM, batch_size - out_row);
+ const short num_outs = min(BN, out_vec_size - out_col);
+ loader_x_t loader_x(x_block, in_vec_size, Xs, simd_group, simd_lane);
+ mma_t mma_op(simd_group, simd_lane);
if (use_mlx_quant) {
using loader_w_t = QuantizedBlockLoaderMlx<
@@ -995,7 +775,15 @@ void qmm_transposed_impl(
WM * WN * 32,
group_size,
bits>;
- loader_w_t loader_w(w_block, scales, biases, K, Ws, simd_gid, simd_lid);
+ loader_w_t loader_w(
+ w_block,
+ scales,
+ biases,
+ in_vec_size,
+ Ws,
+ simd_group,
+ simd_lane
+ );
qmm_transposed_core<
loader_w_t,
loader_x_t,
@@ -1010,12 +798,11 @@ void qmm_transposed_impl(
mma_op,
num_els,
num_outs,
- K,
+ in_vec_size,
y_block,
- N,
+ out_vec_size,
Xs,
- Ws,
- lid
+ Ws
);
} else {
using loader_w_t = QuantizedBlockLoaderZp<
@@ -1028,16 +815,17 @@ void qmm_transposed_impl(
group_size,
bits>;
const device uint8_t* zero_points_row =
- zero_points + y_col * (bits == 4 ? ((K_g + 1) / 2) : K_g);
+ zero_points +
+ out_col * (bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g);
loader_w_t loader_w(
w_block,
scales,
zero_points_row,
- K,
- K_g,
+ in_vec_size,
+ in_vec_size_g,
Ws,
- simd_gid,
- simd_lid
+ simd_group,
+ simd_lane
);
qmm_transposed_core<
loader_w_t,
@@ -1053,661 +841,11 @@ void qmm_transposed_impl(
mma_op,
num_els,
num_outs,
- K,
+ in_vec_size,
y_block,
- N,
+ out_vec_size,
Xs,
- Ws,
- lid
- );
- }
-}
-
-template <typename T, int group_size, int bits, bool UseMlx>
-void qmv_impl_dispatch(
- const device uint32_t* w,
- const device T* scales,
- const device uint8_t* zero_points,
- const device T* biases,
- const device T* x,
- device T* y,
- const constant int& K, // in_vec_size
- const constant int& N, // out_vec_size
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- constexpr int num_simdgroups = 2;
- constexpr int results_per_simdgroup = 4;
- constexpr int packs_per_thread = 1;
- constexpr int pack_factor = get_pack_factor<bits, 32>();
- constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();
-
- constexpr int values_per_thread = pack_factor * packs_per_thread;
- constexpr int block_size = values_per_thread * 32;
- constexpr int scale_step_per_thread = group_size / values_per_thread;
-
- const device uint8_t* ws = (const device uint8_t*)w;
- typedef float U;
- thread U x_thread[values_per_thread];
- thread U result[results_per_simdgroup] = {0};
-
- const int in_vec_size_w = K * bytes_per_pack / pack_factor;
- const int in_vec_size_g =
- (K + group_size - 1) / group_size; // ceil(K / group_size)
- const device T* scales_base = scales;
- const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) +
- simd_gid * results_per_simdgroup;
- const int used_out_row = min(N - results_per_simdgroup, out_row);
-
- if (out_row >= N) {
- return;
- }
-
- if (N < (num_simdgroups * results_per_simdgroup)) {
- ws +=
- out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;
- scales += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
-
- const int zp_stride = bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;
- const device uint8_t* zps_row_base = nullptr;
- const device T* biases_row_base = nullptr;
- if (UseMlx) {
- biases_row_base = biases + out_row * in_vec_size_g;
- } else {
- zps_row_base = zero_points + out_row * zp_stride;
- }
-
- x += tid.x * K + simd_lid * values_per_thread;
- y += tid.x * N + out_row;
-
- int k = 0;
- for (; k < K - block_size; k += block_size) {
- U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);
-
- for (int row = 0; out_row + row < N; row++) {
- if (row >= results_per_simdgroup)
- break;
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const int row_idx = out_row + row;
- const device T* sr = scales_base + row_idx * in_vec_size_g;
-
- int g = (k + simd_lid * values_per_thread) / group_size;
- U s = static_cast<U>(sr[g]);
- if (UseMlx) {
- const device T* bl = biases_row_base + row * in_vec_size_g;
- U b = static_cast<U>(bl[g]);
- result[row] +=
- qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
- } else {
- const device uint8_t* zl = zps_row_base + row * zp_stride;
- U zp;
- if (bits == 4) {
- uint8_t zp_b = zl[g >> 1];
- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
- } else {
- zp = static_cast<U>(zl[g]);
- }
- result[row] +=
- qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);
- }
- }
-
- ws += block_size * bytes_per_pack / pack_factor;
- scales += block_size / group_size;
- x += block_size;
- }
- const int remaining = clamp(
- static_cast<int>(K - k - simd_lid * values_per_thread),
- 0,
- values_per_thread
- );
- if (remaining > 0) {
- U sum = load_vector_safe<T, U, values_per_thread, bits>(
- x,
- x_thread,
- remaining
- );
-
- for (int row = 0; out_row + row < N; row++) {
- if (row >= results_per_simdgroup)
- break;
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const int row_idx = out_row + row;
- const device T* sr = scales_base + row_idx * in_vec_size_g;
-
- int g = (k + simd_lid * values_per_thread) / group_size;
- U s = static_cast<U>(sr[g]);
- if (UseMlx) {
- const device T* bl = biases_row_base + row * in_vec_size_g;
- U b = static_cast<U>(bl[g]);
- result[row] +=
- qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
- } else {
- const device uint8_t* zl = zps_row_base + row * zp_stride;
- U zp;
- if (bits == 4) {
- uint8_t zp_b = zl[g >> 1];
- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
- } else {
- zp = static_cast<U>(zl[g]);
- }
- result[row] +=
- qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);
- }
- }
- }
-
- for (int row = 0; out_row + row < N; row++) {
- if (row >= results_per_simdgroup)
- break;
- result[row] = simd_sum(result[row]);
- if (simd_lid == 0) {
- y[row] = static_cast<T>(result[row]);
- }
- }
- } else {
- ws += used_out_row * in_vec_size_w +
- simd_lid * packs_per_thread * bytes_per_pack;
- scales += used_out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
-
- const int zp_stride = bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;
- const device uint8_t* zps_row_base = nullptr;
- const device T* biases_row_base = nullptr;
- if (UseMlx) {
- biases_row_base = biases + used_out_row * in_vec_size_g;
- } else {
- zps_row_base = zero_points + used_out_row * zp_stride;
- }
-
- x += tid.x * K + simd_lid * values_per_thread;
- y += tid.x * N + used_out_row;
-
- int k = 0;
- for (; k < K - block_size; k += block_size) {
- U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);
-
- for (int row = 0; row < results_per_simdgroup; row++) {
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const int row_idx = used_out_row + row;
- const device T* sr = scales_base + row_idx * in_vec_size_g;
-
- int g = (k + simd_lid * values_per_thread) / group_size;
- U s = static_cast<U>(sr[g]);
- if (UseMlx) {
- const device T* bl = biases_row_base + row * in_vec_size_g;
- U b = static_cast<U>(bl[g]);
- result[row] +=
- qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
- } else {
- const device uint8_t* zl = zps_row_base + row * zp_stride;
- U zp;
- if (bits == 4) {
- uint8_t zp_b = zl[g >> 1];
- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
- } else {
- zp = static_cast<U>(zl[g]);
- }
- result[row] +=
- qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);
- }
- }
-
- ws += block_size * bytes_per_pack / pack_factor;
- scales += block_size / group_size;
- x += block_size;
- }
- const int remaining = clamp(
- static_cast<int>(K - k - simd_lid * values_per_thread),
- 0,
- values_per_thread
- );
-
- if (remaining > 0) {
- U sum = load_vector_safe<T, U, values_per_thread, bits>(
- x,
- x_thread,
- remaining
- );
-
- for (int row = 0; row < results_per_simdgroup; row++) {
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const int row_idx = used_out_row + row;
- const device T* sr = scales_base + row_idx * in_vec_size_g;
-
- int g = (k + simd_lid * values_per_thread) / group_size;
- U s = static_cast<U>(sr[g]);
- if (UseMlx) {
- const device T* bl = biases_row_base + row * in_vec_size_g;
- U b = static_cast<U>(bl[g]);
- result[row] += qdot_safe<U, values_per_thread, bits>(
- wl,
- x_thread,
- s,
- b,
- sum,
- remaining
- );
- } else {
- const device uint8_t* zl = zps_row_base + row * zp_stride;
- U zp;
- if (bits == 4) {
- uint8_t zp_b = zl[g >> 1];
- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
- } else {
- zp = static_cast<U>(zl[g]);
- }
- result[row] +=
- qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);
- }
- }
- }
-
- for (int row = 0; row < results_per_simdgroup; row++) {
- result[row] = simd_sum(result[row]);
- if (simd_lid == 0) {
- y[row] = static_cast<T>(result[row]);
- }
- }
- }
-}
-
-template <typename T, int group_size, int bits, bool use_mlx_quant>
-void qmv_impl(
- const device uint32_t* w,
- const device T* scales,
- const device uint8_t* zero_points,
- const device T* biases,
- const device T* x,
- device T* y,
- const constant int& K,
- const constant int& N,
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- if (use_mlx_quant) {
- qmv_impl_dispatch<T, group_size, bits, true>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- K,
- N,
- tid,
- simd_gid,
- simd_lid
- );
- } else {
- qmv_impl_dispatch<T, group_size, bits, false>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- K,
- N,
- tid,
- simd_gid,
- simd_lid
- );
- }
-}
-
-template <typename T, int group_size, int bits, bool use_mlx_quant>
-void qmv_fast_impl(
- const device uint32_t* w,
- const device T* scales,
- const device uint8_t* zero_points,
- const device T* biases,
- const device T* x,
- device T* y,
- const constant int& in_vec_size,
- const constant int& out_vec_size,
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- constexpr int packs_per_thread = bits == 2 ? 1 : 2;
- constexpr int num_simdgroups = 8;
- constexpr int results_per_simdgroup = 4;
- constexpr int pack_factor = get_pack_factor<bits, 32>();
- constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();
- constexpr int values_per_thread = pack_factor * packs_per_thread;
- constexpr int block_size = values_per_thread * METAL_SIMD_SIZE;
- constexpr int scale_step_per_thread = group_size / values_per_thread;
- const device uint8_t* ws = (const device uint8_t*)w;
- typedef float U;
- thread U x_thread[values_per_thread];
- thread U result[results_per_simdgroup] = {0};
-
- const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
- const int in_vec_size_g = in_vec_size / group_size;
- const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) +
- simd_gid * results_per_simdgroup;
- ws += out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;
- scales += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
-
- int zp_stride = 0;
- const device uint8_t* zps = nullptr;
- bool high_nibble = false;
-
- if (use_mlx_quant) {
- biases += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
- } else {
- if (bits == 4) {
- zp_stride = (in_vec_size_g + 1) / 2;
- zps = zero_points + out_row * zp_stride;
- int g_offset = simd_lid / scale_step_per_thread;
- zps += g_offset / 2;
- high_nibble = (g_offset & 1);
- } else {
- zp_stride = in_vec_size_g;
- zps = zero_points + out_row * zp_stride;
- zps += simd_lid / scale_step_per_thread;
- }
- }
-
- x += tid.x * in_vec_size + simd_lid * values_per_thread;
- y += tid.x * out_vec_size + out_row;
-
- for (int k = 0; k < in_vec_size; k += block_size) {
- U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);
-
- {
- auto wl0 = (const device uint8_t*)(ws);
- auto wl1 = (const device uint8_t*)(ws + in_vec_size_w);
- auto wl2 = (const device uint8_t*)(ws + 2 * in_vec_size_w);
- auto wl3 = (const device uint8_t*)(ws + 3 * in_vec_size_w);
-
- U s0 = static_cast<U>(scales[0]);
- U s1 = static_cast<U>(scales[in_vec_size_g]);
- U s2 = static_cast<U>(scales[2 * in_vec_size_g]);
- U s3 = static_cast<U>(scales[3 * in_vec_size_g]);
-
- if (use_mlx_quant) {
- U b0 = static_cast<U>(biases[0]);
- U b1 = static_cast<U>(biases[in_vec_size_g]);
- U b2 = static_cast<U>(biases[2 * in_vec_size_g]);
- U b3 = static_cast<U>(biases[3 * in_vec_size_g]);
- result[0] +=
- qdot<U, values_per_thread, bits>(wl0, x_thread, s0, b0, sum);
- result[1] +=
- qdot<U, values_per_thread, bits>(wl1, x_thread, s1, b1, sum);
- result[2] +=
- qdot<U, values_per_thread, bits>(wl2, x_thread, s2, b2, sum);
- result[3] +=
- qdot<U, values_per_thread, bits>(wl3, x_thread, s3, b3, sum);
- } else {
- uint8_t zp_byte0 = zps[0];
- uint8_t zp_byte1 = zps[zp_stride];
- uint8_t zp_byte2 = zps[2 * zp_stride];
- uint8_t zp_byte3 = zps[3 * zp_stride];
- U zp0 = static_cast<U>(
- (bits == 4 && high_nibble) ? (zp_byte0 >> 4) : (zp_byte0 & 0x0F)
- );
- U zp1 = static_cast<U>(
- (bits == 4 && high_nibble) ? (zp_byte1 >> 4) : (zp_byte1 & 0x0F)
- );
- U zp2 = static_cast<U>(
- (bits == 4 && high_nibble) ? (zp_byte2 >> 4) : (zp_byte2 & 0x0F)
- );
- U zp3 = static_cast<U>(
- (bits == 4 && high_nibble) ? (zp_byte3 >> 4) : (zp_byte3 & 0x0F)
- );
- if (bits == 8) {
- zp0 = static_cast<U>(zp_byte0);
- zp1 = static_cast<U>(zp_byte1);
- zp2 = static_cast<U>(zp_byte2);
- zp3 = static_cast<U>(zp_byte3);
- }
- result[0] +=
- qdot_zero_point<U, values_per_thread, bits>(wl0, x_thread, s0, zp0);
- result[1] +=
- qdot_zero_point<U, values_per_thread, bits>(wl1, x_thread, s1, zp1);
- result[2] +=
- qdot_zero_point<U, values_per_thread, bits>(wl2, x_thread, s2, zp2);
- result[3] +=
- qdot_zero_point<U, values_per_thread, bits>(wl3, x_thread, s3, zp3);
- }
- }
-
- ws += block_size * bytes_per_pack / pack_factor;
- scales += block_size / group_size;
- if (use_mlx_quant) {
- biases += block_size / group_size;
- } else {
- if (bits == 4) {
- zps += (block_size / group_size) / 2;
- } else {
- zps += block_size / group_size;
- }
- }
- x += block_size;
- }
-
- for (int row = 0; row < results_per_simdgroup; row++) {
- result[row] = simd_sum(result[row]);
- if (simd_lid == 0) {
- y[row] = static_cast<T>(result[row]);
- }
- }
-}
-
-template <typename T, int group_size, int bits, bool use_zero_points>
-void qvm_impl_core(
- const device uint32_t* ws,
- const device T* scales,
- const device T* biases,
- const device uint8_t* zero_points,
- const device T* x,
- device T* y,
- const int in_vec_size,
- const int out_vec_size,
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- constexpr int num_simdgroups = 2;
- constexpr int pack_factor = get_pack_factor<bits, 32>();
- constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();
- constexpr int tn = 32 / pack_factor;
- constexpr int block_size = 32;
-
- typedef float U;
- typedef struct {
- uint8_t wi[tn * bytes_per_pack];
- } vec_w;
-
- thread vec_w w_local;
- thread U result[tn * pack_factor] = {0};
-
- const int out_vec_size_w = out_vec_size * bytes_per_pack / pack_factor;
- const int out_vec_size_g = out_vec_size / group_size;
- const int out_col = pack_factor * tn * (tid.y * num_simdgroups + simd_gid);
-
- if (out_col >= out_vec_size) {
- return;
- }
-
- const int out_group = out_col / group_size;
- const int zp_row_stride =
- use_zero_points
- ? ((bits == 4) ? ((out_vec_size_g + 1) / 2) : out_vec_size_g)
- : 0;
-
- const device uint8_t* ws_ptr = (const device uint8_t*)ws +
- out_col * bytes_per_pack / pack_factor +
- simd_lid * out_vec_size_w;
- const device T* scales_base = scales;
- const device T* biases_base = biases;
- x += tid.x * in_vec_size + simd_lid;
- y += tid.x * out_vec_size + out_col;
-
- for (int k_base = 0; k_base < in_vec_size; k_base += block_size) {
- const int k_index = k_base + simd_lid;
- const bool active = k_index < in_vec_size;
-
- U x_local = active ? static_cast<U>(*x) : U(0);
- U scale =
- active
- ? static_cast<U>(scales_base[k_index * out_vec_size_g + out_group])
- : U(0);
- U bias;
-
- if (use_zero_points && active) {
- U zp;
- if (bits == 4) {
- const device uint8_t* row_ptr = zero_points + k_index * zp_row_stride;
- const device uint8_t* zp_ptr = row_ptr + (out_group >> 1);
- uint8_t zp_byte = *zp_ptr;
- bool high_nibble = (out_group & 1) != 0;
- zp = static_cast<U>(
- high_nibble ? ((zp_byte >> 4) & 0x0F) : (zp_byte & 0x0F)
- );
- } else {
- const device uint8_t* row_ptr = zero_points + k_index * zp_row_stride;
- const device uint8_t* zp_ptr = row_ptr + out_group;
- zp = static_cast<U>(*zp_ptr);
- }
- bias = -scale * zp;
- } else if (active) {
- bias = static_cast<U>(biases_base[k_index * out_vec_size_g + out_group]);
- } else {
- bias = U(0);
- }
-
- if (active) {
- w_local = *((device vec_w*)ws_ptr);
- }
-
- qouter<U, tn * pack_factor, bits>(
- (thread uint8_t*)&w_local,
- x_local,
- scale,
- bias,
- result
- );
-
- x += block_size;
- scales += block_size * out_vec_size_g;
- if (!use_zero_points) {
- biases += block_size * out_vec_size_g;
- }
- ws_ptr += block_size * out_vec_size_w;
- }
-
-#pragma clang loop unroll(full)
- for (int k = 0; k < tn * pack_factor; k++) {
- result[k] = simd_sum(result[k]);
- }
-
- if (simd_lid == 0) {
-#pragma clang loop unroll(full)
- for (int k = 0; k < tn * pack_factor; k++) {
- y[k] = static_cast<T>(result[k]);
- }
- }
-}
-
-template <typename T, int group_size, int bits>
-void qvm_impl_mlx(
- const device uint32_t* ws,
- const device T* scales,
- const device T* biases,
- const device T* x,
- device T* y,
- const int in_vec_size,
- const int out_vec_size,
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- qvm_impl_core<T, group_size, bits, false>(
- ws,
- scales,
- biases,
- nullptr,
- x,
- y,
- in_vec_size,
- out_vec_size,
- tid,
- simd_gid,
- simd_lid
- );
-}
-
-template <typename T, int group_size, int bits>
-void qvm_impl_zeropoint(
- const device uint32_t* w,
- const device T* scales,
- const device uint8_t* zero_points,
- const device T* x,
- device T* y,
- const int in_vec_size,
- const int out_vec_size,
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- qvm_impl_core<T, group_size, bits, true>(
- w,
- scales,
- nullptr,
- zero_points,
- x,
- y,
- in_vec_size,
- out_vec_size,
- tid,
- simd_gid,
- simd_lid
- );
-}
-
-template <typename T, int group_size, int bits, bool use_mlx_quant>
-void qvm_impl(
- const device uint32_t* w,
- const device T* scales,
- const device uint8_t* zero_points,
- const device T* biases,
- const device T* x,
- device T* y,
- const constant int& K, // in_vec_size
- const constant int& N, // out_vec_size
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- if (use_mlx_quant) {
- qvm_impl_mlx<T, group_size, bits>(
- w,
- scales,
- biases,
- x,
- y,
- K,
- N,
- tid,
- simd_gid,
- simd_lid
- );
- } else {
- qvm_impl_zeropoint<T, group_size, bits>(
- w,
- scales,
- zero_points,
- x,
- y,
- K,
- N,
- tid,
- simd_gid,
- simd_lid
+ Ws
);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
deleted file mode 100644
index 27356c4a1..000000000
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
+++ /dev/null
@@ -1,60 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-#include "quant_matmul.h"
-
-template <typename T, int GROUP_SIZE, int BITS>
-VARIANTS(T, float, half, bfloat)
-VARIANTS(GROUP_SIZE, 32, 64, 128)
-VARIANTS(BITS, 4, 8)
-PUBLIC KERNEL(QuantizedMatmulQvm)(
- const device uint32_t* w,
- const device T* scales,
- const device uint8_t* zero_points OPTIONAL(use_zero_points),
- const device T* biases OPTIONAL(use_mlx_quant),
- const device T* x,
- device T* y,
- const constant int& k,
- const constant int& n,
- const constant int& m,
- const bool use_zero_points SPECIALIZE,
- const bool use_mlx_quant SPECIALIZE,
- const uint tgid_x GROUPS(m),
- const uint tgid_y GROUPS((n + 64 - 1) / 64),
- const uint tgid_z GROUPS(1),
- const uint tid_x THREADS(32),
- const uint tid_y THREADS(2)
-) {
- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
- const uint simd_gid = tid_y;
- const uint simd_lid = tid_x;
-
- if (use_mlx_quant) {
- qvm_impl<T, GROUP_SIZE, BITS, true>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- k,
- n,
- tid,
- simd_gid,
- simd_lid
- );
- } else {
- qvm_impl<T, GROUP_SIZE, BITS, false>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- k,
- n,
- tid,
- simd_gid,
- simd_lid
- );
- }
-}
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
index 1204b63e9..a03e223be 100644
--- a/crates/uzu/src/encodable_block/embedding.rs
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -297,7 +297,6 @@ impl<B: Backend> Embedding<B> {
output_dim: vocab_size as usize,
mode: *embedding_quantization_mode,
quantization_type: QuantizedMatmulType::Mlx,
- weights_transposed: true,
},
)?;
@@ -377,7 +376,6 @@ impl<B: Backend> Embedding<B> {
output_dim: vocab_size as usize,
mode: *embedding_quantization_mode,
quantization_type: QuantizedMatmulType::Mlx,
- weights_transposed: true,
},
)?;
@@ -447,7 +445,6 @@ impl<B: Backend> Embedding<B> {
output_dim: vocab_size as usize,
mode: *embedding_quantization_mode,
quantization_type: QuantizedMatmulType::Mlx,
- weights_transposed: true,
},
)?;
@@ -654,10 +651,7 @@ impl<B: Backend> Embedding<B> {
scales_buffer: scales,
zero_points_or_biases_buffer: biases,
output_buffer: output,
- batch: batch_dim,
- input_dim: self.model_dim as usize,
- output_dim: self.vocab_size as usize,
- quantization_type: QuantizedMatmulType::Mlx,
+ batch_dim,
},
)?;
},
diff --git a/crates/uzu/src/encodable_block/linear/quantized.rs b/crates/uzu/src/encodable_block/linear/quantized.rs
index 6450399ba..0d235415e 100644
--- a/crates/uzu/src/encodable_block/linear/quantized.rs
+++ b/crates/uzu/src/encodable_block/linear/quantized.rs
@@ -92,8 +92,6 @@ pub struct QuantizedLinear<B: Backend> {
weights_buffer: Rc<RefCell<B::Buffer>>,
scales_buffer: Rc<RefCell<B::Buffer>>,
zero_points_or_biases_buffer: Rc<RefCell<B::Buffer>>,
- quantization_type: QuantizedMatmulType,
- input_dim: usize,
output_dim: usize,
input_array_id: ArrayId,
output_array_id: ArrayId,
@@ -135,8 +133,6 @@ impl<B: Backend> QuantizedLinear<B> {
let k_g = (input_dim + config.group_size - 1) / config.group_size;
let weights_shape = weights.shape().to_vec();
let scales_shape = scales.shape().to_vec();
- let weights_transposed = weights_shape[0] == output_dim;
-
let (quantization_type, zero_points_or_biases_buffer) = match parameter_tree.leaf_array("deq_biases") {
Ok(deq_biases) => {
let deq_biases_shape = deq_biases.shape().to_vec();
@@ -224,7 +220,6 @@ impl<B: Backend> QuantizedLinear<B> {
output_dim,
mode: config.weight_quantization_mode,
quantization_type,
- weights_transposed,
},
)
.map_err(QuantizedLinearError::QuantizedMatmulError)?;
@@ -236,8 +231,6 @@ impl<B: Backend> QuantizedLinear<B> {
weights_buffer: weights.buffer(),
scales_buffer: scales.buffer(),
zero_points_or_biases_buffer,
- quantization_type,
- input_dim,
output_dim,
input_array_id,
output_array_id,
@@ -268,10 +261,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {
scales_buffer: self.scales_buffer.borrow().deref(),
zero_points_or_biases_buffer: self.zero_points_or_biases_buffer.borrow().deref(),
output_buffer: output_buf_borrow.deref_mut(),
- batch: batch_dim,
- input_dim: self.input_dim,
- output_dim: self.output_dim,
- quantization_type: self.quantization_type,
+ batch_dim,
},
)
.expect("Failed to encode quantized matmul");
diff --git a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
index 38e8c653f..a2a42b2a9 100644
--- a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
+++ b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
@@ -62,34 +62,19 @@ struct ExecutionResult {
fn create_test_weights(
output_dim: usize,
input_dim: usize,
- weights_transposed: bool,
bits: usize,
) -> Vec<u8> {
let mut weights_quant: Vec<u8> = Vec::with_capacity(output_dim * input_dim);
- if weights_transposed {
- // Weights stored as [output_dim × input_dim]
- for row in 0..output_dim {
- for _col in 0..input_dim {
- let v = if bits == 4 {
- ((row + 1) & 0x0F) as u8
- } else {
- ((row + 1) & 0xFF) as u8
- };
- weights_quant.push(v);
- }
- }
- } else {
- // Weights stored as [input_dim × output_dim]
- for _row in 0..input_dim {
- for col in 0..output_dim {
- let v = if bits == 4 {
- ((col + 1) & 0x0F) as u8
- } else {
- (col & 0xFF) as u8 // Allow 0..255 range
- };
- weights_quant.push(v);
- }
+ // Weights stored as [output_dim × input_dim]
+ for row in 0..output_dim {
+ for _col in 0..input_dim {
+ let v = if bits == 4 {
+ ((row + 1) & 0x0F) as u8
+ } else {
+ ((row + 1) & 0xFF) as u8
+ };
+ weights_quant.push(v);
}
}
@@ -154,7 +139,6 @@ fn cpu_reference(
b_quant: &[u8],
scales: &[f32],
biases: &[f32],
- weights_transposed: bool,
group_size: usize,
dtype: DataType,
bits: usize,
@@ -162,26 +146,27 @@ fn cpu_reference(
zero_points: &[u8],
zero_points_stride: usize,
) -> Vec<f32> {
- let num_groups_k = (input_dim + group_size - 1) / group_size;
- let num_groups_n = (output_dim + group_size - 1) / group_size;
- let num_groups = if !weights_transposed {
- num_groups_n
- } else {
- num_groups_k
- };
+ let num_groups = (input_dim + group_size - 1) / group_size;
let mut y = vec![0.0f32; batch * output_dim];
for i in 0..batch {
for j in 0..output_dim {
let mut acc = 0.0f32;
- if !weights_transposed {
- let group_idx = j / group_size;
- for l in 0..input_dim {
- let weight_linear_idx = if weights_transposed {
- j * input_dim + l
- } else {
- l * output_dim + j
- };
+ for g in 0..num_groups {
+ let scale = scales[j * num_groups + g];
+ let bias = match quantization_type {
+ QuantizedMatmulType::ZeroPoint => {
+ let zp = get_zp_value(zero_points, zero_points_stride, j, g, bits);
+ -scale * zp
+ },
+ QuantizedMatmulType::Mlx => biases[j * num_groups + g],
+ };
+ let l_start = g * group_size;
+ let l_end = (l_start + group_size).min(input_dim);
+ let mut group_acc = 0.0f32;
+ let mut group_sum = 0.0f32;
+ for l in l_start..l_end {
+ let weight_linear_idx = j * input_dim + l;
let val_q = if bits == 4 {
get_4bit_value(b_quant, weight_linear_idx)
@@ -190,48 +175,10 @@ fn cpu_reference(
};
let val_a = a[i * input_dim + l];
- let scale = scales[l * num_groups + group_idx];
- let bias = if quantization_type == QuantizedMatmulType::ZeroPoint {
- let zp_val_qvm = get_zp_value(zero_points, zero_points_stride, l, group_idx, bits);
- -scale * zp_val_qvm
- } else {
- biases[l * num_groups + group_idx]
- };
- acc += val_a * (scale * val_q + bias);
- }
- } else {
- for g in 0..num_groups {
- let scale = scales[j * num_groups + g];
- let bias = match quantization_type {
- QuantizedMatmulType::ZeroPoint => {
- let zp = get_zp_value(zero_points, zero_points_stride, j, g, bits);
- -scale * zp
- },
- QuantizedMatmulType::Mlx => biases[j * num_groups + g],
- };
- let l_start = g * group_size;
- let l_end = (l_start + group_size).min(input_dim);
- let mut group_acc = 0.0f32;
- let mut group_sum = 0.0f32;
- for l in l_start..l_end {
- let weight_linear_idx = if weights_transposed {
- j * input_dim + l
- } else {
- l * output_dim + j
- };
-
- let val_q = if bits == 4 {
- get_4bit_value(b_quant, weight_linear_idx)
- } else {
- b_quant[weight_linear_idx] as f32
- };
-
- let val_a = a[i * input_dim + l];
- group_acc += val_a * val_q;
- group_sum += val_a;
- }
- acc += scale * group_acc + bias * group_sum;
+ group_acc += val_a * val_q;
+ group_sum += val_a;
}
+ acc += scale * group_acc + bias * group_sum;
}
y[i * output_dim + j] = quantize_value(acc, dtype);
}
@@ -254,23 +201,11 @@ fn generate_test_quant_params(
data_type: DataType,
bits: usize,
quantization_type: QuantizedMatmulType,
- weights_transposed: bool,
randomize_zp: bool,
) -> TestQuantParams {
- let num_groups_k = (input_dim + group_size - 1) / group_size;
- let num_groups_n = (output_dim + group_size - 1) / group_size;
- let num_groups = if !weights_transposed {
- num_groups_n
- } else {
- num_groups_k
- };
- let primary_dim = if !weights_transposed {
- input_dim
- } else {
- output_dim
- };
+ let num_groups = (input_dim + group_size - 1) / group_size;
- let len = primary_dim * num_groups;
+ let len = output_dim * num_groups;
let scales = vec![1.0; len];
let scales_quant = quantize_slice(&scales, data_type);
let mut biases = vec![0.0; len];
@@ -280,104 +215,43 @@ fn generate_test_quant_params(
} else {
num_groups
};
- let zero_points_len = if !weights_transposed {
- input_dim * zero_points_stride
- } else {
- output_dim * zero_points_stride
- };
- let mut zero_points = vec![0u8; zero_points_len];
+ let mut zero_points = vec![0u8; output_dim * zero_points_stride];
if quantization_type == QuantizedMatmulType::ZeroPoint && randomize_zp {
- if !weights_transposed {
- // GMM/QVM non-transposed: Use [K][N_groups] layout for generation
- for k in 0..input_dim {
- for g in 0..num_groups {
- let k_eff = k / group_size;
- let base_val = ((k_eff * 5 + g * 7) & 0xFF) as u8;
- let zp_val_u8 = if bits == 4 {
- base_val & 0x0F
- } else {
- base_val
- };
- if bits == 4 {
- let byte_index = k * zero_points_stride + (g >> 1);
- if (g & 1) == 0 {
- zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val_u8 & 0x0F);
- } else {
- zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val_u8 & 0x0F) << 4);
- }
- } else {
- zero_points[k * zero_points_stride + g] = zp_val_u8;
- }
-
- // Note: scales are [input_dim][num_groups] in this case?
- // No, scales are 1.0.
- // Biases logic needs to use same indexing!
- // But biases is flat buffer.
- // Index: k * num_groups + g.
- let s = scales_quant[k * num_groups + g];
- let zp_val = if bits == 4 {
- (zp_val_u8 & 0x0F) as f32
- } else {
- zp_val_u8 as f32
- };
- biases[k * num_groups + g] = quantize_value(-s * zp_val, data_type);
- }
- }
- } else {
- for j in 0..output_dim {
- for g in 0..num_groups {
- let base_val = if weights_transposed {
- j + 3 * g
- } else {
- 3 * g
- };
- let zp_val: u8 = if bits == 4 {
- (base_val as u8) & 0x0F
- } else {
- base_val as u8
- };
+ for j in 0..output_dim {
+ for g in 0..num_groups {
+ let base_val = j + 3 * g;
+ let zp_val: u8 = if bits == 4 {
+ (base_val as u8) & 0x0F
+ } else {
+ base_val as u8
+ };
- if bits == 4 {
- let byte_index = j * zero_points_stride + (g >> 1);
- if (g & 1) == 0 {
- zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val & 0x0F);
- } else {
- zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val & 0x0F) << 4);
- }
+ if bits == 4 {
+ let byte_index = j * zero_points_stride + (g >> 1);
+ if (g & 1) == 0 {
+ zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val & 0x0F);
} else {
- let byte_index = j * zero_points_stride + g;
- zero_points[byte_index] = zp_val;
+ zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val & 0x0F) << 4);
}
-
- let s = scales_quant[j * num_groups + g];
- let b = quantize_value(-s * (zp_val as f32), data_type);
- biases[j * num_groups + g] = b;
+ } else {
+ let byte_index = j * zero_points_stride + g;
+ zero_points[byte_index] = zp_val;
}
+
+ let s = scales_quant[j * num_groups + g];
+ let b = quantize_value(-s * (zp_val as f32), data_type);
+ biases[j * num_groups + g] = b;
}
}
}
if quantization_type == QuantizedMatmulType::Mlx {
- if !weights_transposed {
- for k in 0..input_dim {
- for g in 0..num_groups {
- let base_val = g * 3;
- let bias_val = (base_val % 19) as f32 * 0.125;
- biases[k * num_groups + g] = quantize_value(bias_val, data_type);
- }
- }
- } else {
- for j in 0..output_dim {
- for g in 0..num_groups {
- let base_val = if weights_transposed {
- j * 7 + g * 3
- } else {
- g * 3
- };
- let bias_val = (base_val % 19) as f32 * 0.125;
- biases[j * num_groups + g] = quantize_value(bias_val, data_type);
- }
+ for j in 0..output_dim {
+ for g in 0..num_groups {
+ let base_val = j * 7 + g * 3;
+ let bias_val = (base_val % 19) as f32 * 0.125;
+ biases[j * num_groups + g] = quantize_value(bias_val, data_type);
}
}
}
@@ -424,7 +298,6 @@ fn execute_quantized_matmul(
batch: usize,
input_dim: usize,
output_dim: usize,
- weights_transposed: bool,
iterations: usize,
validate: bool,
quantization_type: QuantizedMatmulType,
@@ -433,23 +306,15 @@ fn execute_quantized_matmul(
data_type: DataType,
bits: usize,
) -> ExecutionResult {
- let weights_quant = create_test_weights(output_dim, input_dim, weights_transposed, bits);
+ let weights_quant = create_test_weights(output_dim, input_dim, bits);
let weights_packed = if bits == 4 {
pack_u4_weights(&weights_quant)
} else {
weights_quant.clone()
};
- let params = generate_test_quant_params(
- output_dim,
- input_dim,
- group_size,
- data_type,
- bits,
- quantization_type,
- weights_transposed,
- randomize_zp,
- );
+ let params =
+ generate_test_quant_params(output_dim, input_dim, group_size, data_type, bits, quantization_type, randomize_zp);
// X is batch × input_dim
let x_f32: Vec<f32> = if batch == 1 {
// Vector case: 1 × input_dim
@@ -494,7 +359,6 @@ fn execute_quantized_matmul(
_ => panic!("Unsupported bits: {}", bits),
},
quantization_type,
- weights_transposed,
},
)
.unwrap();
@@ -508,10 +372,7 @@ fn execute_quantized_matmul(
scales_buffer: &s_buf,
zero_points_or_biases_buffer: &b_buf,
output_buffer: &mut y_buf,
- batch,
- input_dim,
- output_dim,
- quantization_type,
+ batch_dim: batch,
};
let mut encoder = Encoder::new(ctx).unwrap();
kernel.encode(&mut encoder, args).unwrap();
@@ -528,10 +389,7 @@ fn execute_quantized_matmul(
scales_buffer: &s_buf,
zero_points_or_biases_buffer: &b_buf,
output_buffer: &mut y_buf,
- batch,
- input_dim,
- output_dim,
- quantization_type,
+ batch_dim: batch,
};
let mut encoder = Encoder::new(ctx).unwrap();
kernel.encode(&mut encoder, args).unwrap();
@@ -548,7 +406,6 @@ fn execute_quantized_matmul(
&weights_packed,
&params.scales_quant,
&params.biases,
- weights_transposed,
group_size,
data_type,
bits,
@@ -629,7 +486,6 @@ struct TestConfig {
}
const QMV_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];
-const QVM_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];
const QMM_DIMS: &[(usize, usize, usize)] = &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];
fn run_kernel_test(
@@ -637,7 +493,6 @@ fn run_kernel_test(
batch: usize,
output_dim: usize,
input_dim: usize,
- weights_transposed: bool,
config: &TestConfig,
validate: bool,
iterations: usize,
@@ -649,7 +504,6 @@ fn run_kernel_test(
batch,
input_dim,
output_dim,
- weights_transposed,
iterations,
validate,
config.quant_type,
@@ -699,95 +553,7 @@ fn test_quant_gmv() {
for config in &configs {
for &(output_dim, input_dim) in QMV_DIMS {
- run_kernel_test(&ctx, 1, output_dim, input_dim, true, config, true, 1);
- }
- }
-}
-
-#[test]
-fn test_quant_qvm() {
- let ctx = match <Metal as Backend>::Context::new() {
- Ok(c) => c,
- Err(_) => {
- println!("Metal not available — skipping QVM test");
- return;
- },
- };
-
- let configs = vec![
- TestConfig {
- quant_type: QuantizedMatmulType::ZeroPoint,
- bits: 4,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_type: QuantizedMatmulType::ZeroPoint,
- bits: 8,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_type: QuantizedMatmulType::Mlx,
- bits: 4,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_type: QuantizedMatmulType::Mlx,
- bits: 8,
- data_type: DataType::F32,
- group_size: 64,
- },
- ];
-
- for config in &configs {
- for &(output_dim, input_dim) in QVM_DIMS {
- run_kernel_test(&ctx, 1, output_dim, input_dim, false, config, true, 1);
- }
- }
-}
-
-#[test]
-fn test_quant_gmm() {
- let ctx = match <Metal as Backend>::Context::new() {
- Ok(c) => c,
- Err(_) => {
- println!("Metal not available — skipping QMM test");
- return;
- },
- };
-
- let configs = vec![
- TestConfig {
- quant_type: QuantizedMatmulType::ZeroPoint,
- bits: 4,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_type: QuantizedMatmulType::ZeroPoint,
- bits: 8,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_type: QuantizedMatmulType::Mlx,
- bits: 4,
- data_type: DataType::F32,
- group_size: 64,
- },
- TestConfig {
- quant_type: QuantizedMatmulType::Mlx,
- bits: 8,
- data_type: DataType::F32,
- group_size: 64,
- },
- ];
-
- for config in &configs {
- for &(batch, output_dim, input_dim) in QMM_DIMS {
- run_kernel_test(&ctx, batch, output_dim, input_dim, false, config, true, 1);
+ run_kernel_test(&ctx, 1, output_dim, input_dim, config, true, 1);
}
}
}
@@ -831,7 +597,7 @@ fn test_quant_gmm_transposed() {
for config in &configs {
for &(batch, output_dim, input_dim) in QMM_DIMS {
- run_kernel_test(&ctx, batch, output_dim, input_dim, true, config, true, 1);
+ run_kernel_test(&ctx, batch, output_dim, input_dim, config, true, 1);
}
}
}
@@ -896,7 +662,7 @@ fn test_quant_matmul_perf() {
for config in &configs {
for &(batch, output_dim, input_dim) in &shapes {
- let result = run_kernel_test(&ctx, batch, output_dim, input_dim, false, config, false, 20);
+ let result = run_kernel_test(&ctx, batch, output_dim, input_dim, config, false, 20);
let avg = result.elapsed / 20.0 * 1000.0; // ms
println!(