A high-performance inference engine for AI models
qm(v/m) refactor (#313)
uuuvnede84c7acfe4
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rsindex f56caf374..63f31070e 100644--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs@@ -1,14 +1,11 @@-use std::collections::HashMap;-use crate::{DataType,backends::common::{Backend, Encoder, Kernels,gpu_types::QuantizationMode,kernel::{- QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,+ QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,QuantizedMatmulQmmTransposedWideKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel,- QuantizedMatmulQvmKernel,},},};@@ -21,20 +18,6 @@ pub enum QuantizedMatmulError<B: Backend> {UnsupportedDataType(DataType),#[error("Unsupported group size: {0}")]UnsupportedGroupSize(usize),- #[error("Unsupported bits: {0}")]- UnsupportedBits(usize),- #[error("Value `{name}` does not fit i32: {value}")]- ValueOutOfRange {- name: &'static str,- value: usize,- },- #[error("Quantization type mismatch: kernel={kernel:?}, args={args:?}")]- QuantizationTypeMismatch {- kernel: QuantizedMatmulType,- args: QuantizedMatmulType,- },- #[error("Missing kernel for key: {0}")]- MissingKernel(&'static str),}#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -51,7 +34,6 @@ pub struct QuantizedMatmulConfiguration {pub output_dim: usize,pub mode: QuantizationMode,pub quantization_type: QuantizedMatmulType,- pub weights_transposed: bool,}pub struct QuantizedMatmulArguments<'a, B: Backend> {@@ -61,102 +43,124 @@ pub struct QuantizedMatmulArguments<'a, B: Backend> {pub scales_buffer: &'a B::Buffer,pub zero_points_or_biases_buffer: &'a B::Buffer,pub output_buffer: &'a mut B::Buffer,- pub batch: usize,- pub input_dim: usize,- pub output_dim: usize,- pub quantization_type: QuantizedMatmulType,+ pub batch_dim: usize,}pub struct QuantizedMatmulKernelEncodable<B: Backend> {- kernels: HashMap<KernelKey, EncodableVariant<B>>,- matrix_vector_key: KernelKey,- matrix_matrix_key: KernelKey,+ matrix_vector: MatrixVectorKernel<B>,+ matrix_matrix: MatrixMatrixKernel<B>,+ input_dim: usize,output_dim: usize,quantization_type: QuantizedMatmulType,}-enum EncodableVariant<B: Backend> {+enum MatrixVectorKernel<B: Backend> {Qmv(<B::Kernels as Kernels>::QuantizedMatmulQmvKernel),QmvFast(<B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel),- Qvm(<B::Kernels as Kernels>::QuantizedMatmulQvmKernel),- Qmm(<B::Kernels as Kernels>::QuantizedMatmulQmmKernel),+}++enum MatrixMatrixKernel<B: Backend> {QmmTransposed(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel),QmmTransposed64x64(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel),QmmTransposedWide(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel),}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-enum KernelKey {- MatrixVector(MatrixVectorFamily),- MatrixMatrix(MatrixMatrixFamily),-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-enum MatrixVectorFamily {- Qmv,- QmvFast,- Qvm,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-enum MatrixMatrixFamily {- QmmAlignedK,- QmmUnalignedK,- QmmTransposedAlignedN,- QmmTransposedUnalignedN,- QmmTransposed64x64,- QmmTransposedWide,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-enum RuntimeVariant {- MatrixVector,- MatrixMatrix,-}-impl<B: Backend> QuantizedMatmulKernelEncodable<B> {pub fn new(context: &B::Context,configuration: QuantizedMatmulConfiguration,) -> Result<Self, QuantizedMatmulError<B>> {- validate_configuration(&configuration)?;+ if !matches!(configuration.data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(QuantizedMatmulError::UnsupportedDataType(configuration.data_type));+ }- let bits = quant_bits(configuration.mode)?;- let use_mlx_quant = matches!(configuration.quantization_type, QuantizedMatmulType::Mlx);+ if !matches!(configuration.group_size, 32 | 64 | 128) {+ return Err(QuantizedMatmulError::UnsupportedGroupSize(configuration.group_size));+ }- let matrix_vector_family = select_matrix_vector_family(&configuration);- let matrix_matrix_family = select_matrix_matrix_family(&configuration, bits);- let matrix_vector_key = KernelKey::MatrixVector(matrix_vector_family);- let matrix_matrix_key = KernelKey::MatrixMatrix(matrix_matrix_family);+ let bits = match configuration.mode {+ QuantizationMode::UINT4 => 4,+ QuantizationMode::INT8 | QuantizationMode::UINT8 => 8,+ };+ let group_size = configuration.group_size as u32;+ let use_mlx_quant = matches!(configuration.quantization_type, QuantizedMatmulType::Mlx);+ let use_zero_points = !use_mlx_quant;++ // Matrix-vector+ let matrix_vector = if configuration.output_dim % 8 == 0 && configuration.input_dim % 512 == 0 {+ MatrixVectorKernel::QmvFast(+ <B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(+ context,+ configuration.data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )+ .map_err(QuantizedMatmulError::BackendError)?,+ )+ } else {+ MatrixVectorKernel::Qmv(+ <B::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(+ context,+ configuration.data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )+ .map_err(QuantizedMatmulError::BackendError)?,+ )+ };- let mut kernels = HashMap::new();- kernels.insert(- matrix_vector_key,- create_matrix_vector_kernel(- context,- configuration.data_type,- configuration.group_size,- bits,- use_mlx_quant,- matrix_vector_family,- )?,- );- kernels.insert(- matrix_matrix_key,- create_matrix_matrix_kernel(- context,- configuration.data_type,- configuration.group_size,- bits,- use_mlx_quant,- matrix_matrix_family,- )?,- );+ // Matrix-matrix+ let aligned_n_64 = configuration.output_dim % 64 == 0;+ let is_bf16 = configuration.data_type == DataType::BF16;++ let matrix_matrix = if aligned_n_64 && is_bf16 && matches!(configuration.group_size, 64 | 128) {+ MatrixMatrixKernel::QmmTransposed64x64(+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(+ context,+ configuration.data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )+ .map_err(QuantizedMatmulError::BackendError)?,+ )+ } else if aligned_n_64 && is_bf16 {+ MatrixMatrixKernel::QmmTransposedWide(+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(+ context,+ configuration.data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )+ .map_err(QuantizedMatmulError::BackendError)?,+ )+ } else {+ let aligned_n = configuration.output_dim % 32 == 0;+ MatrixMatrixKernel::QmmTransposed(+ <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(+ context,+ configuration.data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ aligned_n,+ )+ .map_err(QuantizedMatmulError::BackendError)?,+ )+ };Ok(Self {- kernels,- matrix_vector_key,- matrix_matrix_key,+ matrix_vector,+ matrix_matrix,+ input_dim: configuration.input_dim,output_dim: configuration.output_dim,quantization_type: configuration.quantization_type,})@@ -167,380 +171,42 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {encoder: &mut Encoder<B>,arguments: QuantizedMatmulArguments<B>,) -> Result<(), QuantizedMatmulError<B>> {- if arguments.quantization_type != self.quantization_type {- return Err(QuantizedMatmulError::QuantizationTypeMismatch {- kernel: self.quantization_type,- args: arguments.quantization_type,- });- }-- let key = match self.select_runtime_variant(arguments.batch) {- RuntimeVariant::MatrixVector => self.matrix_vector_key,- RuntimeVariant::MatrixMatrix => self.matrix_matrix_key,+ let (zero_points, biases) = match self.quantization_type {+ QuantizedMatmulType::ZeroPoint => (Some(arguments.zero_points_or_biases_buffer), None),+ QuantizedMatmulType::Mlx => (None, Some(arguments.zero_points_or_biases_buffer)),};- let kernel = self.kernels.get(&key).ok_or(QuantizedMatmulError::MissingKernel(kernel_key_name(key)))?;- let k = to_i32("input_dim", arguments.input_dim)?;- let n = to_i32("output_dim", arguments.output_dim)?;- let m = to_i32("batch", arguments.batch)?;- let (zero_points, biases) = quant_buffers::<B>(arguments.zero_points_or_biases_buffer, self.quantization_type);- let a_with_offset = (arguments.a_buffer, arguments.a_offset);-- match kernel {- EncodableVariant::Qmv(kernel) => {- kernel.encode(- arguments.b_buffer,- arguments.scales_buffer,- zero_points,- biases,- a_with_offset,- arguments.output_buffer,- k,- n,- m,- encoder,- );- },- EncodableVariant::QmvFast(kernel) => {- kernel.encode(- arguments.b_buffer,- arguments.scales_buffer,- zero_points,- biases,- a_with_offset,- arguments.output_buffer,- k,- n,- m,- encoder,- );- },- EncodableVariant::Qvm(kernel) => {- kernel.encode(- arguments.b_buffer,- arguments.scales_buffer,- zero_points,- biases,- a_with_offset,- arguments.output_buffer,- k,- n,- m,- encoder,- );- },- EncodableVariant::Qmm(kernel) => {- kernel.encode(- arguments.b_buffer,- arguments.scales_buffer,- zero_points,- biases,- a_with_offset,- arguments.output_buffer,- k,- n,- m,- encoder,- );- },- EncodableVariant::QmmTransposed(kernel) => {- kernel.encode(- arguments.b_buffer,- arguments.scales_buffer,- zero_points,- biases,- a_with_offset,- arguments.output_buffer,- k,- n,- m,- encoder,- );- },- EncodableVariant::QmmTransposed64x64(kernel) => {- kernel.encode(- arguments.b_buffer,- arguments.scales_buffer,- zero_points,- biases,- a_with_offset,- arguments.output_buffer,- k,- n,- m,- encoder,- );- },- EncodableVariant::QmmTransposedWide(kernel) => {- kernel.encode(+ macro_rules! encode_kernel {+ ($kernel:expr) => {+ $kernel.encode(arguments.b_buffer,arguments.scales_buffer,zero_points,biases,- a_with_offset,+ (arguments.a_buffer, arguments.a_offset),arguments.output_buffer,- k,- n,- m,+ self.input_dim as u32,+ self.output_dim as u32,+ arguments.batch_dim as u32,encoder,- );- },+ )+ };}- Ok(())- }-- fn select_runtime_variant(- &self,- batch: usize,- ) -> RuntimeVariant {- if batch < 32 || self.output_dim == 1 {- RuntimeVariant::MatrixVector+ if arguments.batch_dim < 32 || self.output_dim == 1 {+ match &self.matrix_vector {+ MatrixVectorKernel::Qmv(k) => encode_kernel!(k),+ MatrixVectorKernel::QmvFast(k) => encode_kernel!(k),+ }} else {- RuntimeVariant::MatrixMatrix+ match &self.matrix_matrix {+ MatrixMatrixKernel::QmmTransposed(k) => encode_kernel!(k),+ MatrixMatrixKernel::QmmTransposed64x64(k) => encode_kernel!(k),+ MatrixMatrixKernel::QmmTransposedWide(k) => encode_kernel!(k),+ }}- }-}--fn create_matrix_vector_kernel<B: Backend>(- context: &B::Context,- data_type: DataType,- group_size: usize,- bits: usize,- use_mlx_quant: bool,- family: MatrixVectorFamily,-) -> Result<EncodableVariant<B>, QuantizedMatmulError<B>> {- let group_size = to_i32("group_size", group_size)?;- let bits = to_i32("bits", bits)?;- let use_zero_points = !use_mlx_quant;-- let kernel = match family {- MatrixVectorFamily::Qmv => EncodableVariant::Qmv(- <B::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(- context,- data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- )- .map_err(QuantizedMatmulError::BackendError)?,- ),- MatrixVectorFamily::QmvFast => EncodableVariant::QmvFast(- <B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(- context,- data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- )- .map_err(QuantizedMatmulError::BackendError)?,- ),- MatrixVectorFamily::Qvm => EncodableVariant::Qvm(- <B::Kernels as Kernels>::QuantizedMatmulQvmKernel::new(- context,- data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- )- .map_err(QuantizedMatmulError::BackendError)?,- ),- };-- Ok(kernel)-}--fn create_matrix_matrix_kernel<B: Backend>(- context: &B::Context,- data_type: DataType,- group_size: usize,- bits: usize,- use_mlx_quant: bool,- family: MatrixMatrixFamily,-) -> Result<EncodableVariant<B>, QuantizedMatmulError<B>> {- let group_size = to_i32("group_size", group_size)?;- let bits = to_i32("bits", bits)?;- let use_zero_points = !use_mlx_quant;-- let kernel = match family {- MatrixMatrixFamily::QmmAlignedK => EncodableVariant::Qmm(- <B::Kernels as Kernels>::QuantizedMatmulQmmKernel::new(- context,- data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- true,- )- .map_err(QuantizedMatmulError::BackendError)?,- ),- MatrixMatrixFamily::QmmUnalignedK => EncodableVariant::Qmm(- <B::Kernels as Kernels>::QuantizedMatmulQmmKernel::new(- context,- data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- false,- )- .map_err(QuantizedMatmulError::BackendError)?,- ),- MatrixMatrixFamily::QmmTransposedAlignedN => EncodableVariant::QmmTransposed(- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(- context,- data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- true,- )- .map_err(QuantizedMatmulError::BackendError)?,- ),- MatrixMatrixFamily::QmmTransposedUnalignedN => EncodableVariant::QmmTransposed(- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(- context,- data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- false,- )- .map_err(QuantizedMatmulError::BackendError)?,- ),- MatrixMatrixFamily::QmmTransposed64x64 => EncodableVariant::QmmTransposed64x64(- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(- context,- data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- )- .map_err(QuantizedMatmulError::BackendError)?,- ),- MatrixMatrixFamily::QmmTransposedWide => EncodableVariant::QmmTransposedWide(- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(- context,- data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- )- .map_err(QuantizedMatmulError::BackendError)?,- ),- };- Ok(kernel)-}--fn validate_configuration<B: Backend>(- configuration: &QuantizedMatmulConfiguration-) -> Result<(), QuantizedMatmulError<B>> {- if !matches!(configuration.data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(QuantizedMatmulError::UnsupportedDataType(configuration.data_type));- }-- if !matches!(configuration.group_size, 32 | 64 | 128) {- return Err(QuantizedMatmulError::UnsupportedGroupSize(configuration.group_size));- }-- let _ = quant_bits(configuration.mode)?;- Ok(())-}--fn select_matrix_vector_family(configuration: &QuantizedMatmulConfiguration) -> MatrixVectorFamily {- if configuration.weights_transposed {- if configuration.output_dim % 8 == 0 && configuration.input_dim % 512 == 0 {- MatrixVectorFamily::QmvFast- } else {- MatrixVectorFamily::Qmv- }- } else {- MatrixVectorFamily::Qvm- }-}--fn select_matrix_matrix_family(- configuration: &QuantizedMatmulConfiguration,- bits: usize,-) -> MatrixMatrixFamily {- if configuration.weights_transposed {- let aligned_n = configuration.output_dim % 32 == 0;- let aligned_n_64 = configuration.output_dim % 64 == 0;- let use_64x64 = aligned_n_64- && configuration.data_type == DataType::BF16- && matches!(configuration.group_size, 64 | 128)- && matches!(bits, 4 | 8);- let use_wide = aligned_n_64 && configuration.data_type == DataType::BF16 && matches!(bits, 4 | 8) && !use_64x64;- if use_64x64 {- MatrixMatrixFamily::QmmTransposed64x64- } else if use_wide {- MatrixMatrixFamily::QmmTransposedWide- } else if aligned_n {- MatrixMatrixFamily::QmmTransposedAlignedN- } else {- MatrixMatrixFamily::QmmTransposedUnalignedN- }- } else if configuration.input_dim % 32 == 0 {- MatrixMatrixFamily::QmmAlignedK- } else {- MatrixMatrixFamily::QmmUnalignedK- }-}--fn quant_bits<B: Backend>(mode: QuantizationMode) -> Result<usize, QuantizedMatmulError<B>> {- let bits = match mode {- QuantizationMode::UINT4 => 4,- QuantizationMode::INT8 | QuantizationMode::UINT8 => 8,- };- if matches!(bits, 4 | 8) {- Ok(bits)- } else {- Err(QuantizedMatmulError::UnsupportedBits(bits))- }-}--fn to_i32<B: Backend>(- name: &'static str,- value: usize,-) -> Result<i32, QuantizedMatmulError<B>> {- i32::try_from(value).map_err(|_| QuantizedMatmulError::ValueOutOfRange {- name,- value,- })-}--fn quant_buffers<'a, B: Backend>(- buffer: &'a B::Buffer,- quantization_type: QuantizedMatmulType,-) -> (Option<&'a B::Buffer>, Option<&'a B::Buffer>) {- match quantization_type {- QuantizedMatmulType::ZeroPoint => (Some(buffer), None),- QuantizedMatmulType::Mlx => (None, Some(buffer)),- }-}--fn kernel_key_name(key: KernelKey) -> &'static str {- match key {- KernelKey::MatrixVector(MatrixVectorFamily::Qmv) => "matrix_vector_qmv",- KernelKey::MatrixVector(MatrixVectorFamily::QmvFast) => "matrix_vector_qmv_fast",- KernelKey::MatrixVector(MatrixVectorFamily::Qvm) => "matrix_vector_qvm",- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmAlignedK) => "matrix_matrix_qmm_aligned_k",- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmUnalignedK) => "matrix_matrix_qmm_unaligned_k",- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedAlignedN) => "matrix_matrix_qmm_transposed_aligned_n",- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedUnalignedN) => {- "matrix_matrix_qmm_transposed_unaligned_n"- },- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposed64x64) => "matrix_matrix_qmm_transposed_64x64",- KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedWide) => "matrix_matrix_qmm_transposed_wide",+ Ok(())}}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rsindex 330ce71e9..84dcec8b9 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs@@ -1,7 +1,5 @@-pub mod qmm;pub mod qmm_transposed;pub mod qmm_transposed_64x64;pub mod qmm_transposed_wide;pub mod qmv;pub mod qmv_fast;-pub mod qvm;diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rsdeleted file mode 100644index 823301f3d..000000000--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm.rs+++ /dev/null@@ -1,36 +0,0 @@-use dsl::kernel;-use half::{bf16, f16};-use num_traits::Float;--use crate::ArrayElement;--#[kernel(QuantizedMatmulQmm)]-#[variants(T, f32, f16, bf16)]-#[variants(GROUP_SIZE, 32, 64, 128)]-#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmm<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(- #[allow(unused)] w: *const u32,- #[allow(unused)] scales: *const T,- #[allow(unused)]- #[optional(use_zero_points)]- zero_points: Option<*const u8>,- #[allow(unused)]- #[optional(use_mlx_quant)]- biases: Option<*const T>,- #[allow(unused)] x: *const T,- #[allow(unused)] y: *mut T,- #[allow(unused)] k: i32,- #[allow(unused)] n: i32,- #[allow(unused)] m: i32,- #[allow(unused)]- #[specialize]- use_zero_points: bool,- #[allow(unused)]- #[specialize]- use_mlx_quant: bool,- #[allow(unused)]- #[specialize]- aligned_k: bool,-) {- todo!()-}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rsindex 62f3f9456..4306fc711 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs@@ -8,8 +8,8 @@ use crate::ArrayElement;#[variants(T, f32, f16, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(- #[allow(unused)] w: *const u32,+pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(+ #[allow(unused)] weights: *const u32,#[allow(unused)] scales: *const T,#[allow(unused)]#[optional(use_zero_points)]@@ -17,11 +17,11 @@ pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE#[allow(unused)]#[optional(use_mlx_quant)]biases: Option<*const T>,- #[allow(unused)] x: *const T,- #[allow(unused)] y: *mut T,- #[allow(unused)] k: i32,- #[allow(unused)] n: i32,- #[allow(unused)] m: i32,+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] in_vec_size: u32,+ #[allow(unused)] out_vec_size: u32,+ #[allow(unused)] batch_size: u32,#[allow(unused)]#[specialize]use_zero_points: bool,diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rsindex e252f69bc..aee61e512 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs@@ -8,8 +8,8 @@ use crate::ArrayElement;#[variants(T, bf16)]#[variants(GROUP_SIZE, 64, 128)]#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(- #[allow(unused)] w: *const u32,+pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(+ #[allow(unused)] weights: *const u32,#[allow(unused)] scales: *const T,#[allow(unused)]#[optional(use_zero_points)]@@ -17,11 +17,11 @@ pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP#[allow(unused)]#[optional(use_mlx_quant)]biases: Option<*const T>,- #[allow(unused)] x: *const T,- #[allow(unused)] y: *mut T,- #[allow(unused)] k: i32,- #[allow(unused)] n: i32,- #[allow(unused)] m: i32,+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] in_vec_size: u32,+ #[allow(unused)] out_vec_size: u32,+ #[allow(unused)] batch_size: u32,#[allow(unused)]#[specialize]use_zero_points: bool,diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rsindex f9cabc631..aa64ed350 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs@@ -8,8 +8,8 @@ use crate::ArrayElement;#[variants(T, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(- #[allow(unused)] w: *const u32,+pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(+ #[allow(unused)] weights: *const u32,#[allow(unused)] scales: *const T,#[allow(unused)]#[optional(use_zero_points)]@@ -17,11 +17,11 @@ pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP#[allow(unused)]#[optional(use_mlx_quant)]biases: Option<*const T>,- #[allow(unused)] x: *const T,- #[allow(unused)] y: *mut T,- #[allow(unused)] k: i32,- #[allow(unused)] n: i32,- #[allow(unused)] m: i32,+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] in_vec_size: u32,+ #[allow(unused)] out_vec_size: u32,+ #[allow(unused)] batch_size: u32,#[allow(unused)]#[specialize]use_zero_points: bool,diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rsindex 5ae376ea9..7341a22e6 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs@@ -8,8 +8,8 @@ use crate::ArrayElement;#[variants(T, f32, f16, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(- #[allow(unused)] w: *const u32,+pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(+ #[allow(unused)] weights: *const u32,#[allow(unused)] scales: *const T,#[allow(unused)]#[optional(use_zero_points)]@@ -17,11 +17,11 @@ pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: i32, cons#[allow(unused)]#[optional(use_mlx_quant)]biases: Option<*const T>,- #[allow(unused)] x: *const T,- #[allow(unused)] y: *mut T,- #[allow(unused)] k: i32,- #[allow(unused)] n: i32,- #[allow(unused)] m: i32,+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] in_vec_size: u32,+ #[allow(unused)] out_vec_size: u32,+ #[allow(unused)] batch_size: u32,#[allow(unused)]#[specialize]use_zero_points: bool,diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rsindex 33f16dbcc..d5c53856d 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs@@ -8,8 +8,8 @@ use crate::ArrayElement;#[variants(T, f32, f16, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(- #[allow(unused)] w: *const u32,+pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(+ #[allow(unused)] weights: *const u32,#[allow(unused)] scales: *const T,#[allow(unused)]#[optional(use_zero_points)]@@ -17,11 +17,11 @@ pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: i32,#[allow(unused)]#[optional(use_mlx_quant)]biases: Option<*const T>,- #[allow(unused)] x: *const T,- #[allow(unused)] y: *mut T,- #[allow(unused)] k: i32,- #[allow(unused)] n: i32,- #[allow(unused)] m: i32,+ #[allow(unused)] input: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] in_vec_size: u32,+ #[allow(unused)] out_vec_size: u32,+ #[allow(unused)] batch_size: u32,#[allow(unused)]#[specialize]use_zero_points: bool,diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rsdeleted file mode 100644index 6eeba6605..000000000--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qvm.rs+++ /dev/null@@ -1,33 +0,0 @@-use dsl::kernel;-use half::{bf16, f16};-use num_traits::Float;--use crate::ArrayElement;--#[kernel(QuantizedMatmulQvm)]-#[variants(T, f32, f16, bf16)]-#[variants(GROUP_SIZE, 32, 64, 128)]-#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qvm<T: ArrayElement + Float, const GROUP_SIZE: i32, const BITS: i32>(- #[allow(unused)] w: *const u32,- #[allow(unused)] scales: *const T,- #[allow(unused)]- #[optional(use_zero_points)]- zero_points: Option<*const u8>,- #[allow(unused)]- #[optional(use_mlx_quant)]- biases: Option<*const T>,- #[allow(unused)] x: *const T,- #[allow(unused)] y: *mut T,- #[allow(unused)] k: i32,- #[allow(unused)] n: i32,- #[allow(unused)] m: i32,- #[allow(unused)]- #[specialize]- use_zero_points: bool,- #[allow(unused)]- #[specialize]- use_mlx_quant: bool,-) {- todo!()-}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metaldeleted file mode 100644index bcdc3b970..000000000--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal+++ /dev/null@@ -1,113 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"-#include "quant_matmul.h"--template <typename T, int GROUP_SIZE, int BITS>-VARIANTS(T, float, half, bfloat)-VARIANTS(GROUP_SIZE, 32, 64, 128)-VARIANTS(BITS, 4, 8)-PUBLIC KERNEL(QuantizedMatmulQmm)(- const device uint32_t* w,- const device T* scales,- const device uint8_t* zero_points OPTIONAL(use_zero_points),- const device T* biases OPTIONAL(use_mlx_quant),- const device T* x,- device T* y,- const constant int& k,- const constant int& n,- const constant int& m,- threadgroup T Xs[32 * (32 + 16 / sizeof(T))],- threadgroup T Ws[32 * (32 + 16 / sizeof(T))],- const bool use_zero_points SPECIALIZE,- const bool use_mlx_quant SPECIALIZE,- const bool aligned_k SPECIALIZE,- const uint tgid_x GROUPS((n + 32 - 1) / 32),- const uint tgid_y GROUPS((m + 32 - 1) / 32),- const uint tgid_z GROUPS(1),- const uint tid_x THREADS(32),- const uint tid_y THREADS(2),- const uint tid_z THREADS(2)-) {- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);- const uint lid = tid_z * 64 + tid_y * 32 + tid_x;- const uint simd_gid = tid_z * 2 + tid_y;- const uint simd_lid = tid_x;-- if (use_mlx_quant) {- if (aligned_k) {- qmm_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, true>(- w,- scales,- zero_points,- biases,- x,- y,- Xs,- Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid- );- } else {- qmm_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, true>(- w,- scales,- zero_points,- biases,- x,- y,- Xs,- Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid- );- }- } else {- if (aligned_k) {- qmm_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, false>(- w,- scales,- zero_points,- biases,- x,- y,- Xs,- Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid- );- } else {- qmm_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, false>(- w,- scales,- zero_points,- biases,- x,- y,- Xs,- Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid- );- }- }-}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metalindex 84b0932b9..9db3a1769 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal@@ -2,111 +2,104 @@#include "../common/dsl.h"#include "quant_matmul.h"-template <typename T, int GROUP_SIZE, int BITS>+template <typename T, uint GROUP_SIZE, uint BITS>VARIANTS(T, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(- const device uint32_t* w,+ const device uint32_t* weights,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),const device T* biases OPTIONAL(use_mlx_quant),- const device T* x,- device T* y,- const constant int& k,- const constant int& n,- const constant int& m,+ const device T* input,+ device T* output,+ const constant uint& in_vec_size,+ const constant uint& out_vec_size,+ const constant uint& batch_size,threadgroup T Xs[32 * (32 + 16 / sizeof(T))],threadgroup T Ws[32 * (32 + 16 / sizeof(T))],const bool use_zero_points SPECIALIZE,const bool use_mlx_quant SPECIALIZE,const bool aligned_n SPECIALIZE,- const uint tgid_x GROUPS((n + 32 - 1) / 32),- const uint tgid_y GROUPS((m + 32 - 1) / 32),- const uint tgid_z GROUPS(1),- const uint tid_x THREADS(32),- const uint tid_y THREADS(2),- const uint tid_z THREADS(2)+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),+ const uint batch_block_idx GROUPS(batch_size.div_ceil(32)),+ const uint simd_lane THREADS(32),+ const uint simd_group THREADS(4)) {- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);- const uint lid = tid_z * 64 + tid_y * 32 + tid_x;- const uint simd_gid = tid_z * 2 + tid_y;- const uint simd_lid = tid_x;-if (use_mlx_quant) {if (aligned_n) {qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, true>(- w,+ weights,scales,zero_points,biases,- x,- y,+ input,+ output,Xs,Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid+ in_vec_size,+ out_vec_size,+ batch_size,+ out_block_idx,+ batch_block_idx,+ simd_group,+ simd_lane);} else {qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, true>(- w,+ weights,scales,zero_points,biases,- x,- y,+ input,+ output,Xs,Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid+ in_vec_size,+ out_vec_size,+ batch_size,+ out_block_idx,+ batch_block_idx,+ simd_group,+ simd_lane);}} else {if (aligned_n) {qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, false>(- w,+ weights,scales,zero_points,biases,- x,- y,+ input,+ output,Xs,Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid+ in_vec_size,+ out_vec_size,+ batch_size,+ out_block_idx,+ batch_block_idx,+ simd_group,+ simd_lane);} else {qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, false>(- w,+ weights,scales,zero_points,biases,- x,- y,+ input,+ output,Xs,Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid+ in_vec_size,+ out_vec_size,+ batch_size,+ out_block_idx,+ batch_block_idx,+ simd_group,+ simd_lane);}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metalindex b4af07b58..a11b6c023 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal@@ -2,71 +2,64 @@#include "../common/dsl.h"#include "quant_matmul.h"-template <typename T, int GROUP_SIZE, int BITS>+template <typename T, uint GROUP_SIZE, uint BITS>VARIANTS(T, bfloat)VARIANTS(GROUP_SIZE, 64, 128)VARIANTS(BITS, 4, 8)PUBLIC KERNEL(QuantizedMatmulQmmTransposed64x64)(- const device uint32_t* w,+ const device uint32_t* weights,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),const device T* biases OPTIONAL(use_mlx_quant),- const device T* x,- device T* y,- const constant int& k,- const constant int& n,- const constant int& m,+ const device T* input,+ device T* output,+ const constant uint& in_vec_size,+ const constant uint& out_vec_size,+ const constant uint& batch_size,threadgroup T Xs[64 * (64 + 16 / sizeof(T))],threadgroup T Ws[64 * (64 + 16 / sizeof(T))],const bool use_zero_points SPECIALIZE,const bool use_mlx_quant SPECIALIZE,- const uint tgid_x GROUPS((n + 64 - 1) / 64),- const uint tgid_y GROUPS((m + 64 - 1) / 64),- const uint tgid_z GROUPS(1),- const uint tid_x THREADS(32),- const uint tid_y THREADS(2),- const uint tid_z THREADS(2)+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(64)),+ const uint batch_block_idx GROUPS(batch_size.div_ceil(64)),+ const uint simd_lane THREADS(32),+ const uint simd_group THREADS(4)) {- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);- const uint lid = tid_z * 64 + tid_y * 32 + tid_x;- const uint simd_gid = tid_z * 2 + tid_y;- const uint simd_lid = tid_x;-if (use_mlx_quant) {qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, true>(- w,+ weights,scales,zero_points,biases,- x,- y,+ input,+ output,Xs,Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid+ in_vec_size,+ out_vec_size,+ batch_size,+ out_block_idx,+ batch_block_idx,+ simd_group,+ simd_lane);} else {qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, false>(- w,+ weights,scales,zero_points,biases,- x,- y,+ input,+ output,Xs,Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid+ in_vec_size,+ out_vec_size,+ batch_size,+ out_block_idx,+ batch_block_idx,+ simd_group,+ simd_lane);}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metalindex a066b3022..1c6464ba9 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal@@ -6,71 +6,64 @@// Processes 4x more output elements per threadgroup than the 32x32 variant.// BK=32 satisfies the group_size >= BK constraint for all group sizes (32, 64,// 128).-template <typename T, int GROUP_SIZE, int BITS>+template <typename T, uint GROUP_SIZE, uint BITS>VARIANTS(T, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)PUBLIC KERNEL(QuantizedMatmulQmmTransposedWide)(- const device uint32_t* w,+ const device uint32_t* weights,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),const device T* biases OPTIONAL(use_mlx_quant),- const device T* x,- device T* y,- const constant int& k,- const constant int& n,- const constant int& m,+ const device T* input,+ device T* output,+ const constant uint& in_vec_size,+ const constant uint& out_vec_size,+ const constant uint& batch_size,threadgroup T Xs[64 * (32 + 16 / sizeof(T))],threadgroup T Ws[64 * (32 + 16 / sizeof(T))],const bool use_zero_points SPECIALIZE,const bool use_mlx_quant SPECIALIZE,- const uint tgid_x GROUPS((n + 64 - 1) / 64),- const uint tgid_y GROUPS((m + 64 - 1) / 64),- const uint tgid_z GROUPS(1),- const uint tid_x THREADS(32),- const uint tid_y THREADS(2),- const uint tid_z THREADS(2)+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(64)),+ const uint batch_block_idx GROUPS(batch_size.div_ceil(64)),+ const uint simd_lane THREADS(32),+ const uint simd_group THREADS(4)) {- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);- const uint lid = tid_z * 64 + tid_y * 32 + tid_x;- const uint simd_gid = tid_z * 2 + tid_y;- const uint simd_lid = tid_x;-if (use_mlx_quant) {qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, true>(- w,+ weights,scales,zero_points,biases,- x,- y,+ input,+ output,Xs,Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid+ in_vec_size,+ out_vec_size,+ batch_size,+ out_block_idx,+ batch_block_idx,+ simd_group,+ simd_lane);} else {qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, false>(- w,+ weights,scales,zero_points,biases,- x,- y,+ input,+ output,Xs,Ws,- k,- n,- m,- tid,- lid,- simd_gid,- simd_lid+ in_vec_size,+ out_vec_size,+ batch_size,+ out_block_idx,+ batch_block_idx,+ simd_group,+ simd_lane);}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metalindex 527711d3a..a35ed5b9d 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal@@ -2,59 +2,260 @@#include "../common/dsl.h"#include "quant_matmul.h"-template <typename T, int GROUP_SIZE, int BITS>+template <typename T, uint GROUP_SIZE, uint BITS>VARIANTS(T, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)PUBLIC KERNEL(QuantizedMatmulQmv)(- const device uint32_t* w,+ const device uint32_t* weights,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),const device T* biases OPTIONAL(use_mlx_quant),- const device T* x,- device T* y,- const constant int& k,- const constant int& n,- const constant int& m,+ const device T* input,+ device T* output,+ const constant uint& in_vec_size,+ const constant uint& out_vec_size,+ const constant uint& batch_size,const bool use_zero_points SPECIALIZE,const bool use_mlx_quant SPECIALIZE,- const uint tgid_x GROUPS(m),- const uint tgid_y GROUPS((n + 8 - 1) / 8),- const uint tgid_z GROUPS(1),- const uint tid_x THREADS(32),- const uint tid_y THREADS(2)+ const uint batch_idx GROUPS(batch_size),+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(8)),+ const uint simd_lane THREADS(32),+ const uint simd_group THREADS(2)) {- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);- const uint simd_gid = tid_y;- const uint simd_lid = tid_x;-- if (use_mlx_quant) {- qmv_impl<T, GROUP_SIZE, BITS, true>(- w,- scales,- zero_points,- biases,- x,- y,- k,- n,- tid,- simd_gid,- simd_lid- );+ constexpr uint num_simdgroups = 2;+ constexpr uint results_per_simdgroup = 4;+ constexpr uint packs_per_thread = 1;+ constexpr uint pack_factor = get_pack_factor<BITS, 32>();+ constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();++ constexpr uint values_per_thread = pack_factor * packs_per_thread;+ constexpr uint block_size = values_per_thread * 32;+ constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;++ const device uint8_t* ws = (const device uint8_t*)weights;+ typedef float U;+ thread U x_thread[values_per_thread];+ thread U result[results_per_simdgroup] = {0};++ const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;+ const uint in_vec_size_g = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;+ const device T* scales_base = scales;+ const uint out_row =+ out_block_idx * (num_simdgroups * results_per_simdgroup) ++ simd_group * results_per_simdgroup;+ const uint used_out_row = min(out_vec_size - results_per_simdgroup, out_row);++ if (out_row >= out_vec_size) {+ return;+ }++ if (out_vec_size < (num_simdgroups * results_per_simdgroup)) {+ ws +=+ out_row * in_vec_size_w + simd_lane * packs_per_thread * bytes_per_pack;+ scales += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;++ const uint zp_stride =+ BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;+ const device uint8_t* zps_row_base = nullptr;+ const device T* biases_row_base = nullptr;+ if (use_mlx_quant) {+ biases_row_base = biases + out_row * in_vec_size_g;+ } else {+ zps_row_base = zero_points + out_row * zp_stride;+ }++ input += batch_idx * in_vec_size + simd_lane * values_per_thread;+ output += batch_idx * out_vec_size + out_row;++ uint k = 0;+ for (; k + block_size <= in_vec_size; k += block_size) {+ U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);++ for (uint row = 0;+ out_row + row < out_vec_size && row < results_per_simdgroup;+ row++) {+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);+ const uint row_idx = out_row + row;+ const device T* sr = scales_base + row_idx * in_vec_size_g;++ uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;+ U s = static_cast<U>(sr[g]);+ if (use_mlx_quant) {+ const device T* bl = biases_row_base + row * in_vec_size_g;+ U b = static_cast<U>(bl[g]);+ result[row] +=+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);+ } else {+ const device uint8_t* zl = zps_row_base + row * zp_stride;+ U zp;+ if (BITS == 4) {+ uint8_t zp_b = zl[g >> 1];+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));+ } else {+ zp = static_cast<U>(zl[g]);+ }+ result[row] +=+ qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);+ }+ }++ ws += block_size * bytes_per_pack / pack_factor;+ scales += block_size / GROUP_SIZE;+ input += block_size;+ }+ const uint thread_offset = simd_lane * values_per_thread;+ const int remaining =+ (k + thread_offset < in_vec_size)+ ? min(in_vec_size - k - thread_offset, values_per_thread)+ : 0;+ if (remaining > 0) {+ U sum = load_vector_safe<T, U, values_per_thread, BITS>(+ input,+ x_thread,+ remaining+ );++ for (uint row = 0;+ out_row + row < out_vec_size && row < results_per_simdgroup;+ row++) {+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);+ const uint row_idx = out_row + row;+ const device T* sr = scales_base + row_idx * in_vec_size_g;++ uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;+ U s = static_cast<U>(sr[g]);+ if (use_mlx_quant) {+ const device T* bl = biases_row_base + row * in_vec_size_g;+ U b = static_cast<U>(bl[g]);+ result[row] +=+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);+ } else {+ const device uint8_t* zl = zps_row_base + row * zp_stride;+ U zp;+ if (BITS == 4) {+ uint8_t zp_b = zl[g >> 1];+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));+ } else {+ zp = static_cast<U>(zl[g]);+ }+ result[row] +=+ qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);+ }+ }+ }++ for (uint row = 0;+ out_row + row < out_vec_size && row < results_per_simdgroup;+ row++) {+ result[row] = simd_sum(result[row]);+ if (simd_lane == 0) {+ output[row] = static_cast<T>(result[row]);+ }+ }} else {- qmv_impl<T, GROUP_SIZE, BITS, false>(- w,- scales,- zero_points,- biases,- x,- y,- k,- n,- tid,- simd_gid,- simd_lid- );+ ws += used_out_row * in_vec_size_w ++ simd_lane * packs_per_thread * bytes_per_pack;+ scales += used_out_row * in_vec_size_g + simd_lane / scale_step_per_thread;++ const uint zp_stride =+ BITS == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;+ const device uint8_t* zps_row_base = nullptr;+ const device T* biases_row_base = nullptr;+ if (use_mlx_quant) {+ biases_row_base = biases + used_out_row * in_vec_size_g;+ } else {+ zps_row_base = zero_points + used_out_row * zp_stride;+ }++ input += batch_idx * in_vec_size + simd_lane * values_per_thread;+ output += batch_idx * out_vec_size + used_out_row;++ uint k = 0;+ for (; k + block_size <= in_vec_size; k += block_size) {+ U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);++ for (uint row = 0; row < results_per_simdgroup; row++) {+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);+ const uint row_idx = used_out_row + row;+ const device T* sr = scales_base + row_idx * in_vec_size_g;++ uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;+ U s = static_cast<U>(sr[g]);+ if (use_mlx_quant) {+ const device T* bl = biases_row_base + row * in_vec_size_g;+ U b = static_cast<U>(bl[g]);+ result[row] +=+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, b, sum);+ } else {+ const device uint8_t* zl = zps_row_base + row * zp_stride;+ U zp;+ if (BITS == 4) {+ uint8_t zp_b = zl[g >> 1];+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));+ } else {+ zp = static_cast<U>(zl[g]);+ }+ result[row] +=+ qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);+ }+ }++ ws += block_size * bytes_per_pack / pack_factor;+ scales += block_size / GROUP_SIZE;+ input += block_size;+ }+ const uint thread_offset = simd_lane * values_per_thread;+ const int remaining =+ (k + thread_offset < in_vec_size)+ ? min(in_vec_size - k - thread_offset, values_per_thread)+ : 0;++ if (remaining > 0) {+ U sum = load_vector_safe<T, U, values_per_thread, BITS>(+ input,+ x_thread,+ remaining+ );++ for (uint row = 0; row < results_per_simdgroup; row++) {+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);+ const uint row_idx = used_out_row + row;+ const device T* sr = scales_base + row_idx * in_vec_size_g;++ uint g = (k + simd_lane * values_per_thread) / GROUP_SIZE;+ U s = static_cast<U>(sr[g]);+ if (use_mlx_quant) {+ const device T* bl = biases_row_base + row * in_vec_size_g;+ U b = static_cast<U>(bl[g]);+ result[row] += qdot_safe<U, values_per_thread, BITS>(+ wl,+ x_thread,+ s,+ b,+ sum,+ remaining+ );+ } else {+ const device uint8_t* zl = zps_row_base + row * zp_stride;+ U zp;+ if (BITS == 4) {+ uint8_t zp_b = zl[g >> 1];+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));+ } else {+ zp = static_cast<U>(zl[g]);+ }+ result[row] +=+ qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);+ }+ }+ }++ for (uint row = 0; row < results_per_simdgroup; row++) {+ result[row] = simd_sum(result[row]);+ if (simd_lane == 0) {+ output[row] = static_cast<T>(result[row]);+ }+ }}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metalindex db8eda84f..129af75b7 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal@@ -2,59 +2,150 @@#include "../common/dsl.h"#include "quant_matmul.h"-template <typename T, int GROUP_SIZE, int BITS>+template <typename T, uint GROUP_SIZE, uint BITS>VARIANTS(T, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)PUBLIC KERNEL(QuantizedMatmulQmvFast)(- const device uint32_t* w,+ const device uint32_t* weights,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),const device T* biases OPTIONAL(use_mlx_quant),- const device T* x,- device T* y,- const constant int& k,- const constant int& n,- const constant int& m,+ const device T* input,+ device T* output,+ const constant uint& in_vec_size,+ const constant uint& out_vec_size,+ const constant uint& batch_size,const bool use_zero_points SPECIALIZE,const bool use_mlx_quant SPECIALIZE,- const uint tgid_x GROUPS(m),- const uint tgid_y GROUPS((n + 32 - 1) / 32),- const uint tgid_z GROUPS(1),- const uint tid_x THREADS(32),- const uint tid_y THREADS(8)+ const uint batch_idx GROUPS(batch_size),+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),+ const uint simd_lane THREADS(32),+ const uint simd_group THREADS(8)) {- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);- const uint simd_gid = tid_y;- const uint simd_lid = tid_x;+ constexpr uint packs_per_thread = BITS == 2 ? 1 : 2;+ constexpr uint num_simdgroups = 8;+ constexpr uint results_per_simdgroup = 4;+ constexpr uint pack_factor = get_pack_factor<BITS, 32>();+ constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();+ constexpr uint values_per_thread = pack_factor * packs_per_thread;+ constexpr uint block_size = values_per_thread * METAL_SIMD_SIZE;+ constexpr uint scale_step_per_thread = GROUP_SIZE / values_per_thread;+ const device uint8_t* ws = (const device uint8_t*)weights;+ typedef float U;+ thread U x_thread[values_per_thread];+ thread U result[results_per_simdgroup] = {0};++ const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;+ const uint in_vec_size_g = in_vec_size / GROUP_SIZE;+ const uint out_row =+ out_block_idx * (num_simdgroups * results_per_simdgroup) ++ simd_group * results_per_simdgroup;+ ws += out_row * in_vec_size_w + simd_lane * packs_per_thread * bytes_per_pack;+ scales += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;++ uint zp_stride = 0;+ const device uint8_t* zps = nullptr;+ bool high_nibble = false;if (use_mlx_quant) {- qmv_fast_impl<T, GROUP_SIZE, BITS, true>(- w,- scales,- zero_points,- biases,- x,- y,- k,- n,- tid,- simd_gid,- simd_lid- );+ biases += out_row * in_vec_size_g + simd_lane / scale_step_per_thread;} else {- qmv_fast_impl<T, GROUP_SIZE, BITS, false>(- w,- scales,- zero_points,- biases,- x,- y,- k,- n,- tid,- simd_gid,- simd_lid- );+ if (BITS == 4) {+ zp_stride = (in_vec_size_g + 1) / 2;+ zps = zero_points + out_row * zp_stride;+ uint g_offset = simd_lane / scale_step_per_thread;+ zps += g_offset / 2;+ high_nibble = (g_offset & 1);+ } else {+ zp_stride = in_vec_size_g;+ zps = zero_points + out_row * zp_stride;+ zps += simd_lane / scale_step_per_thread;+ }+ }++ input += batch_idx * in_vec_size + simd_lane * values_per_thread;+ output += batch_idx * out_vec_size + out_row;++ for (uint k = 0; k < in_vec_size; k += block_size) {+ U sum = load_vector<T, U, values_per_thread, BITS>(input, x_thread);++ {+ auto wl0 = (const device uint8_t*)(ws);+ auto wl1 = (const device uint8_t*)(ws + in_vec_size_w);+ auto wl2 = (const device uint8_t*)(ws + 2 * in_vec_size_w);+ auto wl3 = (const device uint8_t*)(ws + 3 * in_vec_size_w);++ U s0 = static_cast<U>(scales[0]);+ U s1 = static_cast<U>(scales[in_vec_size_g]);+ U s2 = static_cast<U>(scales[2 * in_vec_size_g]);+ U s3 = static_cast<U>(scales[3 * in_vec_size_g]);++ if (use_mlx_quant) {+ U b0 = static_cast<U>(biases[0]);+ U b1 = static_cast<U>(biases[in_vec_size_g]);+ U b2 = static_cast<U>(biases[2 * in_vec_size_g]);+ U b3 = static_cast<U>(biases[3 * in_vec_size_g]);+ result[0] +=+ qdot<U, values_per_thread, BITS>(wl0, x_thread, s0, b0, sum);+ result[1] +=+ qdot<U, values_per_thread, BITS>(wl1, x_thread, s1, b1, sum);+ result[2] +=+ qdot<U, values_per_thread, BITS>(wl2, x_thread, s2, b2, sum);+ result[3] +=+ qdot<U, values_per_thread, BITS>(wl3, x_thread, s3, b3, sum);+ } else {+ uint8_t zp_byte0 = zps[0];+ uint8_t zp_byte1 = zps[zp_stride];+ uint8_t zp_byte2 = zps[2 * zp_stride];+ uint8_t zp_byte3 = zps[3 * zp_stride];+ U zp0 = static_cast<U>(+ (BITS == 4 && high_nibble) ? (zp_byte0 >> 4) : (zp_byte0 & 0x0F)+ );+ U zp1 = static_cast<U>(+ (BITS == 4 && high_nibble) ? (zp_byte1 >> 4) : (zp_byte1 & 0x0F)+ );+ U zp2 = static_cast<U>(+ (BITS == 4 && high_nibble) ? (zp_byte2 >> 4) : (zp_byte2 & 0x0F)+ );+ U zp3 = static_cast<U>(+ (BITS == 4 && high_nibble) ? (zp_byte3 >> 4) : (zp_byte3 & 0x0F)+ );+ if (BITS == 8) {+ zp0 = static_cast<U>(zp_byte0);+ zp1 = static_cast<U>(zp_byte1);+ zp2 = static_cast<U>(zp_byte2);+ zp3 = static_cast<U>(zp_byte3);+ }+ result[0] +=+ qdot_zero_point<U, values_per_thread, BITS>(wl0, x_thread, s0, zp0);+ result[1] +=+ qdot_zero_point<U, values_per_thread, BITS>(wl1, x_thread, s1, zp1);+ result[2] +=+ qdot_zero_point<U, values_per_thread, BITS>(wl2, x_thread, s2, zp2);+ result[3] +=+ qdot_zero_point<U, values_per_thread, BITS>(wl3, x_thread, s3, zp3);+ }+ }++ ws += block_size * bytes_per_pack / pack_factor;+ scales += block_size / GROUP_SIZE;+ if (use_mlx_quant) {+ biases += block_size / GROUP_SIZE;+ } else {+ if (BITS == 4) {+ zps += (block_size / GROUP_SIZE) / 2;+ } else {+ zps += block_size / GROUP_SIZE;+ }+ }+ input += block_size;+ }++ for (uint row = 0; row < results_per_simdgroup; row++) {+ result[row] = simd_sum(result[row]);+ if (simd_lane == 0) {+ output[row] = static_cast<T>(result[row]);+ }}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.hindex 6da2bfedf..a27c349e9 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h@@ -1,3 +1,5 @@+#pragma once+#include <metal_simdgroup>#include <metal_stdlib>@@ -627,228 +629,6 @@ struct QuantizedBlockLoaderZp {}};-template <- typename LoaderW,- typename LoaderX,- typename Mma,- typename T,- const bool aligned_K,- const int BM,- const int BK,- const int BN>-inline void qmm_core(- thread LoaderX& loader_x,- thread LoaderW& loader_w,- thread Mma& mma_op,- const short num_els,- const short num_outs,- const int K,- device T* y,- const constant int& N,- threadgroup T* Xs,- threadgroup T* Ws,- uint lid [[thread_index_in_threadgroup]]-) {- if (num_els < BM) {- if ((K % BK) != 0) {- const int k_blocks = K / BK;- for (int k = 0; k < k_blocks; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_safe(short2(BK, num_els));- loader_w.load_unsafe();- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(Xs, Ws);- loader_x.next();- loader_w.next();- }- const short num_k = K - k_blocks * BK;- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_safe(short2(num_k, num_els));- loader_w.load_safe(short2(BN, num_k));- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(Xs, Ws);- } else {- for (int k = 0; k < K; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_safe(short2(BK, num_els));- loader_w.load_unsafe();- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(Xs, Ws);- loader_x.next();- loader_w.next();- }- }- } else {- if ((K % BK) != 0) {- const int k_blocks = K / BK;- for (int k = 0; k < k_blocks; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_unsafe();- loader_w.load_unsafe();- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(Xs, Ws);- loader_x.next();- loader_w.next();- }- const short num_k = K - k_blocks * BK;- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_safe(short2(num_k, BM));- loader_w.load_safe(short2(BN, num_k));- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(Xs, Ws);- } else {- for (int k = 0; k < K; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_unsafe();- loader_w.load_unsafe();- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(Xs, Ws);- loader_x.next();- loader_w.next();- }- }- }-- // Store results to device memory- threadgroup_barrier(mem_flags::mem_threadgroup);- if (num_els < BM) {- mma_op.store_result_safe(y, N, short2(BN, num_els));- } else {- mma_op.store_result(y, N);- }-}--template <- typename T,- const int group_size,- const int bits,- const bool aligned_K = false,- const int BM = 32,- const int BK = 32,- const int BN = 32,- const bool use_mlx_quant = false>-void qmm_impl(- const device uint32_t* w,- const device T* scales,- const device uint8_t* zero_points,- const device T* biases,- const device T* x,- device T* y,- threadgroup T* Xs,- threadgroup T* Ws,- const constant int& K,- const constant int& N,- const constant int& M,- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");- static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");-- (void)lid;-- constexpr int WM = 2;- constexpr int WN = 2;- constexpr int pack_factor = get_pack_factor<bits, 8>();- constexpr int bytes_per_pack = get_bytes_per_pack<bits>();-- constexpr int BK_padded = (BK + 16 / sizeof(T));- constexpr int BN_padded = (BN + 16 / sizeof(T));-- using mma_t = matmul_utils::- BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;- using loader_x_t =- matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;-- auto wl = (const device uint8_t*)w;-- const int y_row = tid.y * BM;- const int y_col = tid.x * BN;- x += y_row * static_cast<int64_t>(K);- wl += y_col * bytes_per_pack / pack_factor;- y += y_row * static_cast<int64_t>(N) + y_col;-- const device T* scales_base = scales;- const device T* biases_base = biases;-- const short num_els = min(BM, M - y_row);- const short num_outs = min(BN, N - y_col);- loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);- mma_t mma_op(simd_gid, simd_lid);-- if (use_mlx_quant) {- using loader_w_t = QuantizedBlockLoaderMlx<- T,- BK,- BN,- BN_padded,- 0,- WM * WN * 32,- group_size,- bits>;- const device T* scales_mlx = scales_base + (y_col / group_size);- const device T* biases_mlx = biases_base + (y_col / group_size);- loader_w_t loader_w(wl, scales_mlx, biases_mlx, N, Ws, simd_gid, simd_lid);- qmm_core<loader_w_t, loader_x_t, mma_t, T, aligned_K, BM, BK, BN>(- loader_x,- loader_w,- mma_op,- num_els,- num_outs,- K,- y,- N,- Xs,- Ws,- lid- );- } else {- const int out_groups_total = (N + group_size - 1) / group_size;- const int groups_per_row = out_groups_total;- const int out_group = y_col / group_size;- const int zp_stride_out =- (bits == 4) ? ((out_groups_total + 1) / 2) : out_groups_total;- using loader_w_t = QuantizedBlockLoaderZp<- T,- BK,- BN,- BN_padded,- 0,- WM * WN * 32,- group_size,- bits,- true>;- loader_w_t loader_w(- wl,- scales_base,- zero_points,- N,- groups_per_row,- Ws,- simd_gid,- simd_lid,- out_group,- out_groups_total,- zp_stride_out- );- qmm_core<loader_w_t, loader_x_t, mma_t, T, aligned_K, BM, BK, BN>(- loader_x,- loader_w,- mma_op,- num_els,- num_outs,- K,- y,- N,- Xs,- Ws,- lid- );- }-}-template <typename LoaderW,typename LoaderX,@@ -864,16 +644,15 @@ inline void qmm_transposed_core(thread Mma& mma_op,const short num_els,const short num_outs,- const int K,- device T* y,- const constant int& N,+ const int in_vec_size,+ device T* output,+ const int out_vec_size,threadgroup T* Xs,- threadgroup T* Ws,- uint lid [[thread_index_in_threadgroup]]+ threadgroup T* Ws) {if (num_els < BM) {if (!aligned_N && num_outs < BN) {- for (int k = 0; k < K; k += BK) {+ for (int k = 0; k < in_vec_size; k += BK) {threadgroup_barrier(mem_flags::mem_threadgroup);loader_x.load_safe(short2(BK, num_els));loader_w.load_safe(short2(BK, num_outs));@@ -883,7 +662,7 @@ inline void qmm_transposed_core(loader_w.next();}} else {- for (int k = 0; k < K; k += BK) {+ for (int k = 0; k < in_vec_size; k += BK) {threadgroup_barrier(mem_flags::mem_threadgroup);loader_x.load_safe(short2(BK, num_els));loader_w.load_unsafe();@@ -895,7 +674,7 @@ inline void qmm_transposed_core(}} else {if (!aligned_N && num_outs < BN) {- for (int k = 0; k < K; k += BK) {+ for (int k = 0; k < in_vec_size; k += BK) {threadgroup_barrier(mem_flags::mem_threadgroup);loader_x.load_unsafe();loader_w.load_safe(short2(BK, num_outs));@@ -905,7 +684,7 @@ inline void qmm_transposed_core(loader_w.next();}} else {- for (int k = 0; k < K; k += BK) {+ for (int k = 0; k < in_vec_size; k += BK) {threadgroup_barrier(mem_flags::mem_threadgroup);loader_x.load_unsafe();loader_w.load_unsafe();@@ -921,37 +700,37 @@ inline void qmm_transposed_core(// Store results to device memorythreadgroup_barrier(mem_flags::mem_threadgroup);if (num_els < BM || num_outs < BN) {- mma_op.store_result_safe(y, N, short2(num_outs, num_els));+ mma_op.store_result_safe(output, out_vec_size, short2(num_outs, num_els));} else {- mma_op.store_result(y, N);+ mma_op.store_result(output, out_vec_size);}}template <typename T,- const int group_size,- const int bits,+ const uint group_size,+ const uint bits,const bool aligned_N,const int BM = 32,const int BK = 32,const int BN = 32,const bool use_mlx_quant = false>void qmm_transposed_impl(- const device uint32_t* w,+ const device uint32_t* weights,const device T* scales,const device uint8_t* zero_points,const device T* biases,- const device T* x,- device T* y,+ const device T* input,+ device T* output,threadgroup T* Xs,threadgroup T* Ws,- const constant int& K,- const constant int& N,- const constant int& M,- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]+ const int in_vec_size,+ const int out_vec_size,+ const int batch_size,+ uint out_block_idx,+ uint batch_block_idx,+ uint simd_group,+ uint simd_lane) {static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");@@ -967,23 +746,24 @@ void qmm_transposed_impl(using loader_x_t =matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32>;- const int K_w = K * bytes_per_pack / pack_factor;- const int K_g = (K + group_size - 1) / group_size;- const int y_row = tid.y * BM;- const int y_col = tid.x * BN;+ const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;+ const int in_vec_size_g = (in_vec_size + group_size - 1) / group_size;+ const int out_row = batch_block_idx * BM;+ const int out_col = out_block_idx * BN;- auto wl = (const device uint8_t*)w;+ auto wl = (const device uint8_t*)weights;- const device T* x_block = x + y_row * static_cast<int64_t>(K);- const device uint8_t* w_block = wl + y_col * K_w;- scales += y_col * K_g;- biases += y_col * K_g;- device T* y_block = y + y_row * static_cast<int64_t>(N) + y_col;+ const device T* x_block = input + out_row * static_cast<int64_t>(in_vec_size);+ const device uint8_t* w_block = wl + out_col * in_vec_size_w;+ scales += out_col * in_vec_size_g;+ biases += out_col * in_vec_size_g;+ device T* y_block =+ output + out_row * static_cast<int64_t>(out_vec_size) + out_col;- const short num_els = min(BM, M - y_row);- const short num_outs = min(BN, N - y_col);- loader_x_t loader_x(x_block, K, Xs, simd_gid, simd_lid);- mma_t mma_op(simd_gid, simd_lid);+ const short num_els = min(BM, batch_size - out_row);+ const short num_outs = min(BN, out_vec_size - out_col);+ loader_x_t loader_x(x_block, in_vec_size, Xs, simd_group, simd_lane);+ mma_t mma_op(simd_group, simd_lane);if (use_mlx_quant) {using loader_w_t = QuantizedBlockLoaderMlx<@@ -995,7 +775,15 @@ void qmm_transposed_impl(WM * WN * 32,group_size,bits>;- loader_w_t loader_w(w_block, scales, biases, K, Ws, simd_gid, simd_lid);+ loader_w_t loader_w(+ w_block,+ scales,+ biases,+ in_vec_size,+ Ws,+ simd_group,+ simd_lane+ );qmm_transposed_core<loader_w_t,loader_x_t,@@ -1010,12 +798,11 @@ void qmm_transposed_impl(mma_op,num_els,num_outs,- K,+ in_vec_size,y_block,- N,+ out_vec_size,Xs,- Ws,- lid+ Ws);} else {using loader_w_t = QuantizedBlockLoaderZp<@@ -1028,16 +815,17 @@ void qmm_transposed_impl(group_size,bits>;const device uint8_t* zero_points_row =- zero_points + y_col * (bits == 4 ? ((K_g + 1) / 2) : K_g);+ zero_points ++ out_col * (bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g);loader_w_t loader_w(w_block,scales,zero_points_row,- K,- K_g,+ in_vec_size,+ in_vec_size_g,Ws,- simd_gid,- simd_lid+ simd_group,+ simd_lane);qmm_transposed_core<loader_w_t,@@ -1053,661 +841,11 @@ void qmm_transposed_impl(mma_op,num_els,num_outs,- K,+ in_vec_size,y_block,- N,+ out_vec_size,Xs,- Ws,- lid- );- }-}--template <typename T, int group_size, int bits, bool UseMlx>-void qmv_impl_dispatch(- const device uint32_t* w,- const device T* scales,- const device uint8_t* zero_points,- const device T* biases,- const device T* x,- device T* y,- const constant int& K, // in_vec_size- const constant int& N, // out_vec_size- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- constexpr int num_simdgroups = 2;- constexpr int results_per_simdgroup = 4;- constexpr int packs_per_thread = 1;- constexpr int pack_factor = get_pack_factor<bits, 32>();- constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();-- constexpr int values_per_thread = pack_factor * packs_per_thread;- constexpr int block_size = values_per_thread * 32;- constexpr int scale_step_per_thread = group_size / values_per_thread;-- const device uint8_t* ws = (const device uint8_t*)w;- typedef float U;- thread U x_thread[values_per_thread];- thread U result[results_per_simdgroup] = {0};-- const int in_vec_size_w = K * bytes_per_pack / pack_factor;- const int in_vec_size_g =- (K + group_size - 1) / group_size; // ceil(K / group_size)- const device T* scales_base = scales;- const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) +- simd_gid * results_per_simdgroup;- const int used_out_row = min(N - results_per_simdgroup, out_row);-- if (out_row >= N) {- return;- }-- if (N < (num_simdgroups * results_per_simdgroup)) {- ws +=- out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;- scales += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;-- const int zp_stride = bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;- const device uint8_t* zps_row_base = nullptr;- const device T* biases_row_base = nullptr;- if (UseMlx) {- biases_row_base = biases + out_row * in_vec_size_g;- } else {- zps_row_base = zero_points + out_row * zp_stride;- }-- x += tid.x * K + simd_lid * values_per_thread;- y += tid.x * N + out_row;-- int k = 0;- for (; k < K - block_size; k += block_size) {- U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);-- for (int row = 0; out_row + row < N; row++) {- if (row >= results_per_simdgroup)- break;- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const int row_idx = out_row + row;- const device T* sr = scales_base + row_idx * in_vec_size_g;-- int g = (k + simd_lid * values_per_thread) / group_size;- U s = static_cast<U>(sr[g]);- if (UseMlx) {- const device T* bl = biases_row_base + row * in_vec_size_g;- U b = static_cast<U>(bl[g]);- result[row] +=- qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);- } else {- const device uint8_t* zl = zps_row_base + row * zp_stride;- U zp;- if (bits == 4) {- uint8_t zp_b = zl[g >> 1];- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));- } else {- zp = static_cast<U>(zl[g]);- }- result[row] +=- qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);- }- }-- ws += block_size * bytes_per_pack / pack_factor;- scales += block_size / group_size;- x += block_size;- }- const int remaining = clamp(- static_cast<int>(K - k - simd_lid * values_per_thread),- 0,- values_per_thread- );- if (remaining > 0) {- U sum = load_vector_safe<T, U, values_per_thread, bits>(- x,- x_thread,- remaining- );-- for (int row = 0; out_row + row < N; row++) {- if (row >= results_per_simdgroup)- break;- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const int row_idx = out_row + row;- const device T* sr = scales_base + row_idx * in_vec_size_g;-- int g = (k + simd_lid * values_per_thread) / group_size;- U s = static_cast<U>(sr[g]);- if (UseMlx) {- const device T* bl = biases_row_base + row * in_vec_size_g;- U b = static_cast<U>(bl[g]);- result[row] +=- qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);- } else {- const device uint8_t* zl = zps_row_base + row * zp_stride;- U zp;- if (bits == 4) {- uint8_t zp_b = zl[g >> 1];- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));- } else {- zp = static_cast<U>(zl[g]);- }- result[row] +=- qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);- }- }- }-- for (int row = 0; out_row + row < N; row++) {- if (row >= results_per_simdgroup)- break;- result[row] = simd_sum(result[row]);- if (simd_lid == 0) {- y[row] = static_cast<T>(result[row]);- }- }- } else {- ws += used_out_row * in_vec_size_w +- simd_lid * packs_per_thread * bytes_per_pack;- scales += used_out_row * in_vec_size_g + simd_lid / scale_step_per_thread;-- const int zp_stride = bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;- const device uint8_t* zps_row_base = nullptr;- const device T* biases_row_base = nullptr;- if (UseMlx) {- biases_row_base = biases + used_out_row * in_vec_size_g;- } else {- zps_row_base = zero_points + used_out_row * zp_stride;- }-- x += tid.x * K + simd_lid * values_per_thread;- y += tid.x * N + used_out_row;-- int k = 0;- for (; k < K - block_size; k += block_size) {- U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);-- for (int row = 0; row < results_per_simdgroup; row++) {- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const int row_idx = used_out_row + row;- const device T* sr = scales_base + row_idx * in_vec_size_g;-- int g = (k + simd_lid * values_per_thread) / group_size;- U s = static_cast<U>(sr[g]);- if (UseMlx) {- const device T* bl = biases_row_base + row * in_vec_size_g;- U b = static_cast<U>(bl[g]);- result[row] +=- qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);- } else {- const device uint8_t* zl = zps_row_base + row * zp_stride;- U zp;- if (bits == 4) {- uint8_t zp_b = zl[g >> 1];- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));- } else {- zp = static_cast<U>(zl[g]);- }- result[row] +=- qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);- }- }-- ws += block_size * bytes_per_pack / pack_factor;- scales += block_size / group_size;- x += block_size;- }- const int remaining = clamp(- static_cast<int>(K - k - simd_lid * values_per_thread),- 0,- values_per_thread- );-- if (remaining > 0) {- U sum = load_vector_safe<T, U, values_per_thread, bits>(- x,- x_thread,- remaining- );-- for (int row = 0; row < results_per_simdgroup; row++) {- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const int row_idx = used_out_row + row;- const device T* sr = scales_base + row_idx * in_vec_size_g;-- int g = (k + simd_lid * values_per_thread) / group_size;- U s = static_cast<U>(sr[g]);- if (UseMlx) {- const device T* bl = biases_row_base + row * in_vec_size_g;- U b = static_cast<U>(bl[g]);- result[row] += qdot_safe<U, values_per_thread, bits>(- wl,- x_thread,- s,- b,- sum,- remaining- );- } else {- const device uint8_t* zl = zps_row_base + row * zp_stride;- U zp;- if (bits == 4) {- uint8_t zp_b = zl[g >> 1];- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));- } else {- zp = static_cast<U>(zl[g]);- }- result[row] +=- qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);- }- }- }-- for (int row = 0; row < results_per_simdgroup; row++) {- result[row] = simd_sum(result[row]);- if (simd_lid == 0) {- y[row] = static_cast<T>(result[row]);- }- }- }-}--template <typename T, int group_size, int bits, bool use_mlx_quant>-void qmv_impl(- const device uint32_t* w,- const device T* scales,- const device uint8_t* zero_points,- const device T* biases,- const device T* x,- device T* y,- const constant int& K,- const constant int& N,- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- if (use_mlx_quant) {- qmv_impl_dispatch<T, group_size, bits, true>(- w,- scales,- zero_points,- biases,- x,- y,- K,- N,- tid,- simd_gid,- simd_lid- );- } else {- qmv_impl_dispatch<T, group_size, bits, false>(- w,- scales,- zero_points,- biases,- x,- y,- K,- N,- tid,- simd_gid,- simd_lid- );- }-}--template <typename T, int group_size, int bits, bool use_mlx_quant>-void qmv_fast_impl(- const device uint32_t* w,- const device T* scales,- const device uint8_t* zero_points,- const device T* biases,- const device T* x,- device T* y,- const constant int& in_vec_size,- const constant int& out_vec_size,- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- constexpr int packs_per_thread = bits == 2 ? 1 : 2;- constexpr int num_simdgroups = 8;- constexpr int results_per_simdgroup = 4;- constexpr int pack_factor = get_pack_factor<bits, 32>();- constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();- constexpr int values_per_thread = pack_factor * packs_per_thread;- constexpr int block_size = values_per_thread * METAL_SIMD_SIZE;- constexpr int scale_step_per_thread = group_size / values_per_thread;- const device uint8_t* ws = (const device uint8_t*)w;- typedef float U;- thread U x_thread[values_per_thread];- thread U result[results_per_simdgroup] = {0};-- const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;- const int in_vec_size_g = in_vec_size / group_size;- const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) +- simd_gid * results_per_simdgroup;- ws += out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;- scales += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;-- int zp_stride = 0;- const device uint8_t* zps = nullptr;- bool high_nibble = false;-- if (use_mlx_quant) {- biases += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;- } else {- if (bits == 4) {- zp_stride = (in_vec_size_g + 1) / 2;- zps = zero_points + out_row * zp_stride;- int g_offset = simd_lid / scale_step_per_thread;- zps += g_offset / 2;- high_nibble = (g_offset & 1);- } else {- zp_stride = in_vec_size_g;- zps = zero_points + out_row * zp_stride;- zps += simd_lid / scale_step_per_thread;- }- }-- x += tid.x * in_vec_size + simd_lid * values_per_thread;- y += tid.x * out_vec_size + out_row;-- for (int k = 0; k < in_vec_size; k += block_size) {- U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);-- {- auto wl0 = (const device uint8_t*)(ws);- auto wl1 = (const device uint8_t*)(ws + in_vec_size_w);- auto wl2 = (const device uint8_t*)(ws + 2 * in_vec_size_w);- auto wl3 = (const device uint8_t*)(ws + 3 * in_vec_size_w);-- U s0 = static_cast<U>(scales[0]);- U s1 = static_cast<U>(scales[in_vec_size_g]);- U s2 = static_cast<U>(scales[2 * in_vec_size_g]);- U s3 = static_cast<U>(scales[3 * in_vec_size_g]);-- if (use_mlx_quant) {- U b0 = static_cast<U>(biases[0]);- U b1 = static_cast<U>(biases[in_vec_size_g]);- U b2 = static_cast<U>(biases[2 * in_vec_size_g]);- U b3 = static_cast<U>(biases[3 * in_vec_size_g]);- result[0] +=- qdot<U, values_per_thread, bits>(wl0, x_thread, s0, b0, sum);- result[1] +=- qdot<U, values_per_thread, bits>(wl1, x_thread, s1, b1, sum);- result[2] +=- qdot<U, values_per_thread, bits>(wl2, x_thread, s2, b2, sum);- result[3] +=- qdot<U, values_per_thread, bits>(wl3, x_thread, s3, b3, sum);- } else {- uint8_t zp_byte0 = zps[0];- uint8_t zp_byte1 = zps[zp_stride];- uint8_t zp_byte2 = zps[2 * zp_stride];- uint8_t zp_byte3 = zps[3 * zp_stride];- U zp0 = static_cast<U>(- (bits == 4 && high_nibble) ? (zp_byte0 >> 4) : (zp_byte0 & 0x0F)- );- U zp1 = static_cast<U>(- (bits == 4 && high_nibble) ? (zp_byte1 >> 4) : (zp_byte1 & 0x0F)- );- U zp2 = static_cast<U>(- (bits == 4 && high_nibble) ? (zp_byte2 >> 4) : (zp_byte2 & 0x0F)- );- U zp3 = static_cast<U>(- (bits == 4 && high_nibble) ? (zp_byte3 >> 4) : (zp_byte3 & 0x0F)- );- if (bits == 8) {- zp0 = static_cast<U>(zp_byte0);- zp1 = static_cast<U>(zp_byte1);- zp2 = static_cast<U>(zp_byte2);- zp3 = static_cast<U>(zp_byte3);- }- result[0] +=- qdot_zero_point<U, values_per_thread, bits>(wl0, x_thread, s0, zp0);- result[1] +=- qdot_zero_point<U, values_per_thread, bits>(wl1, x_thread, s1, zp1);- result[2] +=- qdot_zero_point<U, values_per_thread, bits>(wl2, x_thread, s2, zp2);- result[3] +=- qdot_zero_point<U, values_per_thread, bits>(wl3, x_thread, s3, zp3);- }- }-- ws += block_size * bytes_per_pack / pack_factor;- scales += block_size / group_size;- if (use_mlx_quant) {- biases += block_size / group_size;- } else {- if (bits == 4) {- zps += (block_size / group_size) / 2;- } else {- zps += block_size / group_size;- }- }- x += block_size;- }-- for (int row = 0; row < results_per_simdgroup; row++) {- result[row] = simd_sum(result[row]);- if (simd_lid == 0) {- y[row] = static_cast<T>(result[row]);- }- }-}--template <typename T, int group_size, int bits, bool use_zero_points>-void qvm_impl_core(- const device uint32_t* ws,- const device T* scales,- const device T* biases,- const device uint8_t* zero_points,- const device T* x,- device T* y,- const int in_vec_size,- const int out_vec_size,- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- constexpr int num_simdgroups = 2;- constexpr int pack_factor = get_pack_factor<bits, 32>();- constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();- constexpr int tn = 32 / pack_factor;- constexpr int block_size = 32;-- typedef float U;- typedef struct {- uint8_t wi[tn * bytes_per_pack];- } vec_w;-- thread vec_w w_local;- thread U result[tn * pack_factor] = {0};-- const int out_vec_size_w = out_vec_size * bytes_per_pack / pack_factor;- const int out_vec_size_g = out_vec_size / group_size;- const int out_col = pack_factor * tn * (tid.y * num_simdgroups + simd_gid);-- if (out_col >= out_vec_size) {- return;- }-- const int out_group = out_col / group_size;- const int zp_row_stride =- use_zero_points- ? ((bits == 4) ? ((out_vec_size_g + 1) / 2) : out_vec_size_g)- : 0;-- const device uint8_t* ws_ptr = (const device uint8_t*)ws +- out_col * bytes_per_pack / pack_factor +- simd_lid * out_vec_size_w;- const device T* scales_base = scales;- const device T* biases_base = biases;- x += tid.x * in_vec_size + simd_lid;- y += tid.x * out_vec_size + out_col;-- for (int k_base = 0; k_base < in_vec_size; k_base += block_size) {- const int k_index = k_base + simd_lid;- const bool active = k_index < in_vec_size;-- U x_local = active ? static_cast<U>(*x) : U(0);- U scale =- active- ? static_cast<U>(scales_base[k_index * out_vec_size_g + out_group])- : U(0);- U bias;-- if (use_zero_points && active) {- U zp;- if (bits == 4) {- const device uint8_t* row_ptr = zero_points + k_index * zp_row_stride;- const device uint8_t* zp_ptr = row_ptr + (out_group >> 1);- uint8_t zp_byte = *zp_ptr;- bool high_nibble = (out_group & 1) != 0;- zp = static_cast<U>(- high_nibble ? ((zp_byte >> 4) & 0x0F) : (zp_byte & 0x0F)- );- } else {- const device uint8_t* row_ptr = zero_points + k_index * zp_row_stride;- const device uint8_t* zp_ptr = row_ptr + out_group;- zp = static_cast<U>(*zp_ptr);- }- bias = -scale * zp;- } else if (active) {- bias = static_cast<U>(biases_base[k_index * out_vec_size_g + out_group]);- } else {- bias = U(0);- }-- if (active) {- w_local = *((device vec_w*)ws_ptr);- }-- qouter<U, tn * pack_factor, bits>(- (thread uint8_t*)&w_local,- x_local,- scale,- bias,- result- );-- x += block_size;- scales += block_size * out_vec_size_g;- if (!use_zero_points) {- biases += block_size * out_vec_size_g;- }- ws_ptr += block_size * out_vec_size_w;- }--#pragma clang loop unroll(full)- for (int k = 0; k < tn * pack_factor; k++) {- result[k] = simd_sum(result[k]);- }-- if (simd_lid == 0) {-#pragma clang loop unroll(full)- for (int k = 0; k < tn * pack_factor; k++) {- y[k] = static_cast<T>(result[k]);- }- }-}--template <typename T, int group_size, int bits>-void qvm_impl_mlx(- const device uint32_t* ws,- const device T* scales,- const device T* biases,- const device T* x,- device T* y,- const int in_vec_size,- const int out_vec_size,- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- qvm_impl_core<T, group_size, bits, false>(- ws,- scales,- biases,- nullptr,- x,- y,- in_vec_size,- out_vec_size,- tid,- simd_gid,- simd_lid- );-}--template <typename T, int group_size, int bits>-void qvm_impl_zeropoint(- const device uint32_t* w,- const device T* scales,- const device uint8_t* zero_points,- const device T* x,- device T* y,- const int in_vec_size,- const int out_vec_size,- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- qvm_impl_core<T, group_size, bits, true>(- w,- scales,- nullptr,- zero_points,- x,- y,- in_vec_size,- out_vec_size,- tid,- simd_gid,- simd_lid- );-}--template <typename T, int group_size, int bits, bool use_mlx_quant>-void qvm_impl(- const device uint32_t* w,- const device T* scales,- const device uint8_t* zero_points,- const device T* biases,- const device T* x,- device T* y,- const constant int& K, // in_vec_size- const constant int& N, // out_vec_size- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- if (use_mlx_quant) {- qvm_impl_mlx<T, group_size, bits>(- w,- scales,- biases,- x,- y,- K,- N,- tid,- simd_gid,- simd_lid- );- } else {- qvm_impl_zeropoint<T, group_size, bits>(- w,- scales,- zero_points,- x,- y,- K,- N,- tid,- simd_gid,- simd_lid+ Ws);}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metaldeleted file mode 100644index 27356c4a1..000000000--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal+++ /dev/null@@ -1,60 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"-#include "quant_matmul.h"--template <typename T, int GROUP_SIZE, int BITS>-VARIANTS(T, float, half, bfloat)-VARIANTS(GROUP_SIZE, 32, 64, 128)-VARIANTS(BITS, 4, 8)-PUBLIC KERNEL(QuantizedMatmulQvm)(- const device uint32_t* w,- const device T* scales,- const device uint8_t* zero_points OPTIONAL(use_zero_points),- const device T* biases OPTIONAL(use_mlx_quant),- const device T* x,- device T* y,- const constant int& k,- const constant int& n,- const constant int& m,- const bool use_zero_points SPECIALIZE,- const bool use_mlx_quant SPECIALIZE,- const uint tgid_x GROUPS(m),- const uint tgid_y GROUPS((n + 64 - 1) / 64),- const uint tgid_z GROUPS(1),- const uint tid_x THREADS(32),- const uint tid_y THREADS(2)-) {- const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);- const uint simd_gid = tid_y;- const uint simd_lid = tid_x;-- if (use_mlx_quant) {- qvm_impl<T, GROUP_SIZE, BITS, true>(- w,- scales,- zero_points,- biases,- x,- y,- k,- n,- tid,- simd_gid,- simd_lid- );- } else {- qvm_impl<T, GROUP_SIZE, BITS, false>(- w,- scales,- zero_points,- biases,- x,- y,- k,- n,- tid,- simd_gid,- simd_lid- );- }-}diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsindex 1204b63e9..a03e223be 100644--- a/crates/uzu/src/encodable_block/embedding.rs+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -297,7 +297,6 @@ impl<B: Backend> Embedding<B> {output_dim: vocab_size as usize,mode: *embedding_quantization_mode,quantization_type: QuantizedMatmulType::Mlx,- weights_transposed: true,},)?;@@ -377,7 +376,6 @@ impl<B: Backend> Embedding<B> {output_dim: vocab_size as usize,mode: *embedding_quantization_mode,quantization_type: QuantizedMatmulType::Mlx,- weights_transposed: true,},)?;@@ -447,7 +445,6 @@ impl<B: Backend> Embedding<B> {output_dim: vocab_size as usize,mode: *embedding_quantization_mode,quantization_type: QuantizedMatmulType::Mlx,- weights_transposed: true,},)?;@@ -654,10 +651,7 @@ impl<B: Backend> Embedding<B> {scales_buffer: scales,zero_points_or_biases_buffer: biases,output_buffer: output,- batch: batch_dim,- input_dim: self.model_dim as usize,- output_dim: self.vocab_size as usize,- quantization_type: QuantizedMatmulType::Mlx,+ batch_dim,},)?;},diff --git a/crates/uzu/src/encodable_block/linear/quantized.rs b/crates/uzu/src/encodable_block/linear/quantized.rsindex 6450399ba..0d235415e 100644--- a/crates/uzu/src/encodable_block/linear/quantized.rs+++ b/crates/uzu/src/encodable_block/linear/quantized.rs@@ -92,8 +92,6 @@ pub struct QuantizedLinear<B: Backend> {weights_buffer: Rc<RefCell<B::Buffer>>,scales_buffer: Rc<RefCell<B::Buffer>>,zero_points_or_biases_buffer: Rc<RefCell<B::Buffer>>,- quantization_type: QuantizedMatmulType,- input_dim: usize,output_dim: usize,input_array_id: ArrayId,output_array_id: ArrayId,@@ -135,8 +133,6 @@ impl<B: Backend> QuantizedLinear<B> {let k_g = (input_dim + config.group_size - 1) / config.group_size;let weights_shape = weights.shape().to_vec();let scales_shape = scales.shape().to_vec();- let weights_transposed = weights_shape[0] == output_dim;-let (quantization_type, zero_points_or_biases_buffer) = match parameter_tree.leaf_array("deq_biases") {Ok(deq_biases) => {let deq_biases_shape = deq_biases.shape().to_vec();@@ -224,7 +220,6 @@ impl<B: Backend> QuantizedLinear<B> {output_dim,mode: config.weight_quantization_mode,quantization_type,- weights_transposed,},).map_err(QuantizedLinearError::QuantizedMatmulError)?;@@ -236,8 +231,6 @@ impl<B: Backend> QuantizedLinear<B> {weights_buffer: weights.buffer(),scales_buffer: scales.buffer(),zero_points_or_biases_buffer,- quantization_type,- input_dim,output_dim,input_array_id,output_array_id,@@ -268,10 +261,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {scales_buffer: self.scales_buffer.borrow().deref(),zero_points_or_biases_buffer: self.zero_points_or_biases_buffer.borrow().deref(),output_buffer: output_buf_borrow.deref_mut(),- batch: batch_dim,- input_dim: self.input_dim,- output_dim: self.output_dim,- quantization_type: self.quantization_type,+ batch_dim,},).expect("Failed to encode quantized matmul");diff --git a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rsindex 38e8c653f..a2a42b2a9 100644--- a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs+++ b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs@@ -62,34 +62,19 @@ struct ExecutionResult {fn create_test_weights(output_dim: usize,input_dim: usize,- weights_transposed: bool,bits: usize,) -> Vec<u8> {let mut weights_quant: Vec<u8> = Vec::with_capacity(output_dim * input_dim);- if weights_transposed {- // Weights stored as [output_dim × input_dim]- for row in 0..output_dim {- for _col in 0..input_dim {- let v = if bits == 4 {- ((row + 1) & 0x0F) as u8- } else {- ((row + 1) & 0xFF) as u8- };- weights_quant.push(v);- }- }- } else {- // Weights stored as [input_dim × output_dim]- for _row in 0..input_dim {- for col in 0..output_dim {- let v = if bits == 4 {- ((col + 1) & 0x0F) as u8- } else {- (col & 0xFF) as u8 // Allow 0..255 range- };- weights_quant.push(v);- }+ // Weights stored as [output_dim × input_dim]+ for row in 0..output_dim {+ for _col in 0..input_dim {+ let v = if bits == 4 {+ ((row + 1) & 0x0F) as u8+ } else {+ ((row + 1) & 0xFF) as u8+ };+ weights_quant.push(v);}}@@ -154,7 +139,6 @@ fn cpu_reference(b_quant: &[u8],scales: &[f32],biases: &[f32],- weights_transposed: bool,group_size: usize,dtype: DataType,bits: usize,@@ -162,26 +146,27 @@ fn cpu_reference(zero_points: &[u8],zero_points_stride: usize,) -> Vec<f32> {- let num_groups_k = (input_dim + group_size - 1) / group_size;- let num_groups_n = (output_dim + group_size - 1) / group_size;- let num_groups = if !weights_transposed {- num_groups_n- } else {- num_groups_k- };+ let num_groups = (input_dim + group_size - 1) / group_size;let mut y = vec![0.0f32; batch * output_dim];for i in 0..batch {for j in 0..output_dim {let mut acc = 0.0f32;- if !weights_transposed {- let group_idx = j / group_size;- for l in 0..input_dim {- let weight_linear_idx = if weights_transposed {- j * input_dim + l- } else {- l * output_dim + j- };+ for g in 0..num_groups {+ let scale = scales[j * num_groups + g];+ let bias = match quantization_type {+ QuantizedMatmulType::ZeroPoint => {+ let zp = get_zp_value(zero_points, zero_points_stride, j, g, bits);+ -scale * zp+ },+ QuantizedMatmulType::Mlx => biases[j * num_groups + g],+ };+ let l_start = g * group_size;+ let l_end = (l_start + group_size).min(input_dim);+ let mut group_acc = 0.0f32;+ let mut group_sum = 0.0f32;+ for l in l_start..l_end {+ let weight_linear_idx = j * input_dim + l;let val_q = if bits == 4 {get_4bit_value(b_quant, weight_linear_idx)@@ -190,48 +175,10 @@ fn cpu_reference(};let val_a = a[i * input_dim + l];- let scale = scales[l * num_groups + group_idx];- let bias = if quantization_type == QuantizedMatmulType::ZeroPoint {- let zp_val_qvm = get_zp_value(zero_points, zero_points_stride, l, group_idx, bits);- -scale * zp_val_qvm- } else {- biases[l * num_groups + group_idx]- };- acc += val_a * (scale * val_q + bias);- }- } else {- for g in 0..num_groups {- let scale = scales[j * num_groups + g];- let bias = match quantization_type {- QuantizedMatmulType::ZeroPoint => {- let zp = get_zp_value(zero_points, zero_points_stride, j, g, bits);- -scale * zp- },- QuantizedMatmulType::Mlx => biases[j * num_groups + g],- };- let l_start = g * group_size;- let l_end = (l_start + group_size).min(input_dim);- let mut group_acc = 0.0f32;- let mut group_sum = 0.0f32;- for l in l_start..l_end {- let weight_linear_idx = if weights_transposed {- j * input_dim + l- } else {- l * output_dim + j- };-- let val_q = if bits == 4 {- get_4bit_value(b_quant, weight_linear_idx)- } else {- b_quant[weight_linear_idx] as f32- };-- let val_a = a[i * input_dim + l];- group_acc += val_a * val_q;- group_sum += val_a;- }- acc += scale * group_acc + bias * group_sum;+ group_acc += val_a * val_q;+ group_sum += val_a;}+ acc += scale * group_acc + bias * group_sum;}y[i * output_dim + j] = quantize_value(acc, dtype);}@@ -254,23 +201,11 @@ fn generate_test_quant_params(data_type: DataType,bits: usize,quantization_type: QuantizedMatmulType,- weights_transposed: bool,randomize_zp: bool,) -> TestQuantParams {- let num_groups_k = (input_dim + group_size - 1) / group_size;- let num_groups_n = (output_dim + group_size - 1) / group_size;- let num_groups = if !weights_transposed {- num_groups_n- } else {- num_groups_k- };- let primary_dim = if !weights_transposed {- input_dim- } else {- output_dim- };+ let num_groups = (input_dim + group_size - 1) / group_size;- let len = primary_dim * num_groups;+ let len = output_dim * num_groups;let scales = vec![1.0; len];let scales_quant = quantize_slice(&scales, data_type);let mut biases = vec![0.0; len];@@ -280,104 +215,43 @@ fn generate_test_quant_params(} else {num_groups};- let zero_points_len = if !weights_transposed {- input_dim * zero_points_stride- } else {- output_dim * zero_points_stride- };- let mut zero_points = vec![0u8; zero_points_len];+ let mut zero_points = vec![0u8; output_dim * zero_points_stride];if quantization_type == QuantizedMatmulType::ZeroPoint && randomize_zp {- if !weights_transposed {- // GMM/QVM non-transposed: Use [K][N_groups] layout for generation- for k in 0..input_dim {- for g in 0..num_groups {- let k_eff = k / group_size;- let base_val = ((k_eff * 5 + g * 7) & 0xFF) as u8;- let zp_val_u8 = if bits == 4 {- base_val & 0x0F- } else {- base_val- };- if bits == 4 {- let byte_index = k * zero_points_stride + (g >> 1);- if (g & 1) == 0 {- zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val_u8 & 0x0F);- } else {- zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val_u8 & 0x0F) << 4);- }- } else {- zero_points[k * zero_points_stride + g] = zp_val_u8;- }-- // Note: scales are [input_dim][num_groups] in this case?- // No, scales are 1.0.- // Biases logic needs to use same indexing!- // But biases is flat buffer.- // Index: k * num_groups + g.- let s = scales_quant[k * num_groups + g];- let zp_val = if bits == 4 {- (zp_val_u8 & 0x0F) as f32- } else {- zp_val_u8 as f32- };- biases[k * num_groups + g] = quantize_value(-s * zp_val, data_type);- }- }- } else {- for j in 0..output_dim {- for g in 0..num_groups {- let base_val = if weights_transposed {- j + 3 * g- } else {- 3 * g- };- let zp_val: u8 = if bits == 4 {- (base_val as u8) & 0x0F- } else {- base_val as u8- };+ for j in 0..output_dim {+ for g in 0..num_groups {+ let base_val = j + 3 * g;+ let zp_val: u8 = if bits == 4 {+ (base_val as u8) & 0x0F+ } else {+ base_val as u8+ };- if bits == 4 {- let byte_index = j * zero_points_stride + (g >> 1);- if (g & 1) == 0 {- zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val & 0x0F);- } else {- zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val & 0x0F) << 4);- }+ if bits == 4 {+ let byte_index = j * zero_points_stride + (g >> 1);+ if (g & 1) == 0 {+ zero_points[byte_index] = (zero_points[byte_index] & 0xF0) | (zp_val & 0x0F);} else {- let byte_index = j * zero_points_stride + g;- zero_points[byte_index] = zp_val;+ zero_points[byte_index] = (zero_points[byte_index] & 0x0F) | ((zp_val & 0x0F) << 4);}-- let s = scales_quant[j * num_groups + g];- let b = quantize_value(-s * (zp_val as f32), data_type);- biases[j * num_groups + g] = b;+ } else {+ let byte_index = j * zero_points_stride + g;+ zero_points[byte_index] = zp_val;}++ let s = scales_quant[j * num_groups + g];+ let b = quantize_value(-s * (zp_val as f32), data_type);+ biases[j * num_groups + g] = b;}}}if quantization_type == QuantizedMatmulType::Mlx {- if !weights_transposed {- for k in 0..input_dim {- for g in 0..num_groups {- let base_val = g * 3;- let bias_val = (base_val % 19) as f32 * 0.125;- biases[k * num_groups + g] = quantize_value(bias_val, data_type);- }- }- } else {- for j in 0..output_dim {- for g in 0..num_groups {- let base_val = if weights_transposed {- j * 7 + g * 3- } else {- g * 3- };- let bias_val = (base_val % 19) as f32 * 0.125;- biases[j * num_groups + g] = quantize_value(bias_val, data_type);- }+ for j in 0..output_dim {+ for g in 0..num_groups {+ let base_val = j * 7 + g * 3;+ let bias_val = (base_val % 19) as f32 * 0.125;+ biases[j * num_groups + g] = quantize_value(bias_val, data_type);}}}@@ -424,7 +298,6 @@ fn execute_quantized_matmul(batch: usize,input_dim: usize,output_dim: usize,- weights_transposed: bool,iterations: usize,validate: bool,quantization_type: QuantizedMatmulType,@@ -433,23 +306,15 @@ fn execute_quantized_matmul(data_type: DataType,bits: usize,) -> ExecutionResult {- let weights_quant = create_test_weights(output_dim, input_dim, weights_transposed, bits);+ let weights_quant = create_test_weights(output_dim, input_dim, bits);let weights_packed = if bits == 4 {pack_u4_weights(&weights_quant)} else {weights_quant.clone()};- let params = generate_test_quant_params(- output_dim,- input_dim,- group_size,- data_type,- bits,- quantization_type,- weights_transposed,- randomize_zp,- );+ let params =+ generate_test_quant_params(output_dim, input_dim, group_size, data_type, bits, quantization_type, randomize_zp);// X is batch × input_dimlet x_f32: Vec<f32> = if batch == 1 {// Vector case: 1 × input_dim@@ -494,7 +359,6 @@ fn execute_quantized_matmul(_ => panic!("Unsupported bits: {}", bits),},quantization_type,- weights_transposed,},).unwrap();@@ -508,10 +372,7 @@ fn execute_quantized_matmul(scales_buffer: &s_buf,zero_points_or_biases_buffer: &b_buf,output_buffer: &mut y_buf,- batch,- input_dim,- output_dim,- quantization_type,+ batch_dim: batch,};let mut encoder = Encoder::new(ctx).unwrap();kernel.encode(&mut encoder, args).unwrap();@@ -528,10 +389,7 @@ fn execute_quantized_matmul(scales_buffer: &s_buf,zero_points_or_biases_buffer: &b_buf,output_buffer: &mut y_buf,- batch,- input_dim,- output_dim,- quantization_type,+ batch_dim: batch,};let mut encoder = Encoder::new(ctx).unwrap();kernel.encode(&mut encoder, args).unwrap();@@ -548,7 +406,6 @@ fn execute_quantized_matmul(&weights_packed,¶ms.scales_quant,¶ms.biases,- weights_transposed,group_size,data_type,bits,@@ -629,7 +486,6 @@ struct TestConfig {}const QMV_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];-const QVM_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];const QMM_DIMS: &[(usize, usize, usize)] = &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];fn run_kernel_test(@@ -637,7 +493,6 @@ fn run_kernel_test(batch: usize,output_dim: usize,input_dim: usize,- weights_transposed: bool,config: &TestConfig,validate: bool,iterations: usize,@@ -649,7 +504,6 @@ fn run_kernel_test(batch,input_dim,output_dim,- weights_transposed,iterations,validate,config.quant_type,@@ -699,95 +553,7 @@ fn test_quant_gmv() {for config in &configs {for &(output_dim, input_dim) in QMV_DIMS {- run_kernel_test(&ctx, 1, output_dim, input_dim, true, config, true, 1);- }- }-}--#[test]-fn test_quant_qvm() {- let ctx = match <Metal as Backend>::Context::new() {- Ok(c) => c,- Err(_) => {- println!("Metal not available — skipping QVM test");- return;- },- };-- let configs = vec![- TestConfig {- quant_type: QuantizedMatmulType::ZeroPoint,- bits: 4,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_type: QuantizedMatmulType::ZeroPoint,- bits: 8,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_type: QuantizedMatmulType::Mlx,- bits: 4,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_type: QuantizedMatmulType::Mlx,- bits: 8,- data_type: DataType::F32,- group_size: 64,- },- ];-- for config in &configs {- for &(output_dim, input_dim) in QVM_DIMS {- run_kernel_test(&ctx, 1, output_dim, input_dim, false, config, true, 1);- }- }-}--#[test]-fn test_quant_gmm() {- let ctx = match <Metal as Backend>::Context::new() {- Ok(c) => c,- Err(_) => {- println!("Metal not available — skipping QMM test");- return;- },- };-- let configs = vec![- TestConfig {- quant_type: QuantizedMatmulType::ZeroPoint,- bits: 4,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_type: QuantizedMatmulType::ZeroPoint,- bits: 8,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_type: QuantizedMatmulType::Mlx,- bits: 4,- data_type: DataType::F32,- group_size: 64,- },- TestConfig {- quant_type: QuantizedMatmulType::Mlx,- bits: 8,- data_type: DataType::F32,- group_size: 64,- },- ];-- for config in &configs {- for &(batch, output_dim, input_dim) in QMM_DIMS {- run_kernel_test(&ctx, batch, output_dim, input_dim, false, config, true, 1);+ run_kernel_test(&ctx, 1, output_dim, input_dim, config, true, 1);}}}@@ -831,7 +597,7 @@ fn test_quant_gmm_transposed() {for config in &configs {for &(batch, output_dim, input_dim) in QMM_DIMS {- run_kernel_test(&ctx, batch, output_dim, input_dim, true, config, true, 1);+ run_kernel_test(&ctx, batch, output_dim, input_dim, config, true, 1);}}}@@ -896,7 +662,7 @@ fn test_quant_matmul_perf() {for config in &configs {for &(batch, output_dim, input_dim) in &shapes {- let result = run_kernel_test(&ctx, batch, output_dim, input_dim, false, config, false, 20);+ let result = run_kernel_test(&ctx, batch, output_dim, input_dim, config, false, 20);let avg = result.elapsed / 20.0 * 1000.0; // msprintln!(