A high-performance inference engine for AI models
dslify quant matmul
eugene2e1014428f0f
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rsindex ad92a5179..62f28b658 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs@@ -5,11 +5,11 @@ use super::EmbeddingError;use crate::{DataType,backends::{- common::Context,- metal::{- Metal, MetalContext, MetalError,- kernel::quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},+ common::{+ Context,+ kernel::matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulType},},+ metal::{Metal, MetalContext, MetalError, kernel::quant_matmul::QuantizedMatmulKernel},},config::QuantizationMode,encodable_block::{EncodableBlock, EncodingParameters},@@ -161,13 +161,15 @@ impl QuantizedEmbeddingReadout {let kernel = QuantizedMatmulKernel::new(mtl_context,- data_type,- group_size,- model_dim,- vocab_size,- mode,- QuantizationType::Mlx,- weights_transposed,+ QuantizedMatmulConfiguration {+ data_type,+ group_size,+ input_dim: model_dim,+ output_dim: vocab_size,+ mode,+ quantization_type: QuantizedMatmulType::Mlx,+ weights_transposed,+ },).map_err(|e| EmbeddingError::MetalError(MetalError::Generic(format!("Failed to create kernel: {:?}", e))))?;@@ -225,15 +227,15 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {let args = QuantizedMatmulArguments {a_buffer: input_buffer,- a_offset,+ a_offset: a_offset as usize,b_buffer: &self.weights_buffer,scales_buffer: &self.scales_buffer,zero_points_or_biases_buffer: &self.biases_buffer,output_buffer,- batch: batch_size as i32,- input_dim: self.model_dim as i32,- output_dim: self.vocab_size as i32,- quantization_type: QuantizationType::Mlx,+ batch: batch_size,+ input_dim: self.model_dim,+ output_dim: self.vocab_size,+ quantization_type: QuantizedMatmulType::Mlx,};self.kernel.encode(encoder, args).expect("Failed to encode quantized embedding readout kernel");diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/common.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/common.hnew file mode 100644index 000000000..2535a2ea4--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/common.h@@ -0,0 +1,4 @@+#pragma once++// Shared quantized matmul helper implementations.+#include "quant_matmul.metal"diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rsindex 876dc6407..f22db7374 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs@@ -1,30 +1,31 @@use std::collections::HashMap;-use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLFunctionConstantValues, MTLSize};+use metal::{MTLBuffer, MTLComputeCommandEncoder};use objc2::{rc::Retained, runtime::ProtocolObject};use crate::{DataType,backends::{- common::kernel::matmul::{- QuantizedMatmulArguments as GenericQuantizedMatmulArguments, QuantizedMatmulConfiguration,- QuantizedMatmulKernel as QuantizedMatmulKernelTrait, QuantizedMatmulType,+ common::kernel::{+ QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,+ QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, QuantizedMatmulQvmKernel,+ matmul::{+ QuantizedMatmulArguments as GenericQuantizedMatmulArguments, QuantizedMatmulConfiguration,+ QuantizedMatmulKernel as QuantizedMatmulKernelTrait, QuantizedMatmulType,+ },},metal::{- FunctionConstantValuesSetValue, Metal, MetalContext, MetalError, metal_extensions::ComputeEncoderSetValue,+ Metal, MetalContext, MetalError,+ kernel::dsl::{+ QuantizedMatmulQmmMetalKernel, QuantizedMatmulQmmTransposed64x64MetalKernel,+ QuantizedMatmulQmmTransposedMetalKernel, QuantizedMatmulQmvFastMetalKernel,+ QuantizedMatmulQmvMetalKernel, QuantizedMatmulQvmMetalKernel,+ },},},config::QuantizationMode,};-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum QuantizationType {- /// GroupQuantized style: uses zero-points- ZeroPoint,- /// MLX style: uses pre-computed biases- Mlx,-}-#[derive(Debug, thiserror::Error)]pub enum QuantizedMatmulError {#[error("Metal error: {0}")]@@ -33,205 +34,237 @@ pub enum QuantizedMatmulError {UnsupportedDataType(DataType),#[error("Unsupported group size: {0}")]UnsupportedGroupSize(usize),- #[error("Invalid dimensions: M={m}, N={n}, K={k}")]- InvalidDimensions {- m: usize,- n: usize,- k: usize,- },#[error("Unsupported bits: {0}")]UnsupportedBits(usize),+ #[error("Value `{name}` does not fit i32: {value}")]+ ValueOutOfRange {+ name: &'static str,+ value: usize,+ },+ #[error("Quantization type mismatch: kernel={kernel:?}, args={args:?}")]+ QuantizationTypeMismatch {+ kernel: QuantizedMatmulType,+ args: QuantizedMatmulType,+ },+ #[error("Missing kernel for key: {0}")]+ MissingKernel(&'static str),}pub struct QuantizedMatmulKernel {- pipelines: HashMap<KernelKind, (Retained<ProtocolObject<dyn MTLComputePipelineState>>, u64, u64)>,+ kernels: HashMap<KernelKey, EncodableVariant>,+ matrix_vector_key: KernelKey,+ matrix_matrix_key: KernelKey,output_dim: usize,- weights_transposed: bool,+ quantization_type: QuantizedMatmulType,}-#[derive(Debug, Clone, Copy)]-pub struct QuantizedMatmulArguments<'a> {- pub a_buffer: &'a ProtocolObject<dyn MTLBuffer>,- /// Byte offset into `a_buffer` (used for slicing the batch dimension).- pub a_offset: u64,- pub b_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub scales_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub zero_points_or_biases_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub batch: i32,- pub input_dim: i32,- pub output_dim: i32,- pub quantization_type: QuantizationType,+enum EncodableVariant {+ Qmv(QuantizedMatmulQmvMetalKernel),+ QmvFast(QuantizedMatmulQmvFastMetalKernel),+ Qvm(QuantizedMatmulQvmMetalKernel),+ Qmm(QuantizedMatmulQmmMetalKernel),+ QmmTransposed(QuantizedMatmulQmmTransposedMetalKernel),+ QmmTransposed64x64(QuantizedMatmulQmmTransposed64x64MetalKernel),}-fn dtype_suffix(data_type: DataType) -> Option<&'static str> {- match data_type {- DataType::F16 => Some("f16"),- DataType::BF16 => Some("bf16"),- DataType::F32 => Some("f32"),- _ => None,- }+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+enum KernelKey {+ MatrixVector(MatrixVectorFamily),+ MatrixMatrix(MatrixMatrixFamily),}-fn base_qmm_kernel_name(- type_suffix: &str,- group_size: usize,- transpose_infix: &str,- bits: usize,-) -> Result<String, QuantizedMatmulError> {- if bits != 4 && bits != 8 {- return Err(QuantizedMatmulError::UnsupportedBits(bits));- }+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+enum MatrixVectorFamily {+ Qmv,+ QmvFast,+ Qvm,+}- let kernel_name = match (type_suffix, group_size) {- ("f16", 32) => format!("qmm{}_f16_g32_b{}", transpose_infix, bits),- ("f16", 64) => format!("qmm{}_f16_g64_b{}", transpose_infix, bits),- ("f16", 128) => format!("qmm{}_f16_g128_b{}", transpose_infix, bits),- ("bf16", 32) => format!("qmm{}_bf16_g32_b{}", transpose_infix, bits),- ("bf16", 64) => format!("qmm{}_bf16_g64_b{}", transpose_infix, bits),- ("bf16", 128) => format!("qmm{}_bf16_g128_b{}", transpose_infix, bits),- ("f32", 32) => format!("qmm{}_f32_g32_b{}", transpose_infix, bits),- ("f32", 64) => format!("qmm{}_f32_g64_b{}", transpose_infix, bits),- ("f32", 128) => format!("qmm{}_f32_g128_b{}", transpose_infix, bits),- _ => {- return Err(QuantizedMatmulError::UnsupportedGroupSize(group_size));- },- };- Ok(kernel_name)+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+enum MatrixMatrixFamily {+ QmmAlignedK,+ QmmUnalignedK,+ QmmTransposedAlignedN,+ QmmTransposedUnalignedN,+ QmmTransposed64x64,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum RuntimeVariant {+ MatrixVector,+ MatrixMatrix,}impl QuantizedMatmulKernel {pub fn new(- mtl_context: &MetalContext,- data_type: DataType,- group_size: usize,- input_dim: usize,- output_dim: usize,- mode: QuantizationMode,- quantization_type: QuantizationType,- weights_transposed: bool,+ context: &MetalContext,+ configuration: QuantizedMatmulConfiguration,) -> Result<Self, QuantizedMatmulError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(QuantizedMatmulError::UnsupportedDataType(data_type));- }-- let function_constants = MTLFunctionConstantValues::new();- let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);- function_constants.set_value(&use_mlx_quant, 40);-- let mut pipelines = HashMap::new();-- let kernel_name_mv =- select_matrix_vector_kernel_name(data_type, group_size, weights_transposed, output_dim, input_dim, mode)?;-- let cache_key_mv = format!("{}_mlx_{}", kernel_name_mv, use_mlx_quant);- let pipeline_mv = mtl_context- .compute_pipeline_state_cached(&cache_key_mv, &kernel_name_mv, Some(&function_constants))- .map_err(QuantizedMatmulError::MetalError)?;- pipelines.insert(KernelKind::MatrixVector, (pipeline_mv, 32, 32));-- let kernel_name_mm =- select_qmm_kernel_name(data_type, group_size, weights_transposed, output_dim, input_dim, mode)?;-- let cache_key_mm = format!("{}_mlx_{}", kernel_name_mm, use_mlx_quant);- let pipeline_mm = mtl_context- .compute_pipeline_state_cached(&cache_key_mm, &kernel_name_mm, Some(&function_constants))- .map_err(QuantizedMatmulError::MetalError)?;-- let (bm, bn) = if kernel_name_mm.contains("_64x64") {- (64, 64)- } else if kernel_name_mm.contains("_64x128") {- (64, 128)- } else if kernel_name_mm.contains("_128x64") {- (128, 64)- } else {- (32, 32)- };- pipelines.insert(KernelKind::MatrixMatrix, (pipeline_mm, bm, bn));+ validate_configuration(&configuration)?;++ let bits = quant_bits(configuration.mode)?;+ let use_mlx_quant = matches!(configuration.quantization_type, QuantizedMatmulType::Mlx);++ let matrix_vector_family = select_matrix_vector_family(&configuration);+ let matrix_matrix_family = select_matrix_matrix_family(&configuration, bits);+ let matrix_vector_key = KernelKey::MatrixVector(matrix_vector_family);+ let matrix_matrix_key = KernelKey::MatrixMatrix(matrix_matrix_family);++ let mut kernels = HashMap::new();+ kernels.insert(+ matrix_vector_key,+ create_matrix_vector_kernel(+ context,+ configuration.data_type,+ configuration.group_size,+ bits,+ use_mlx_quant,+ matrix_vector_family,+ )?,+ );+ kernels.insert(+ matrix_matrix_key,+ create_matrix_matrix_kernel(+ context,+ configuration.data_type,+ configuration.group_size,+ bits,+ use_mlx_quant,+ matrix_matrix_family,+ )?,+ );Ok(Self {- pipelines,- output_dim,- weights_transposed,+ kernels,+ matrix_vector_key,+ matrix_matrix_key,+ output_dim: configuration.output_dim,+ quantization_type: configuration.quantization_type,})}- #[allow(dead_code)]- fn kernel_name_for_config(data_type: DataType) -> String {- let type_suffix = match data_type {- DataType::F16 => "f16",- DataType::BF16 => "bf16",- DataType::F32 => "f32",- _ => unreachable!(),- };-- format!("qmm_t_{}_gs_64_b_4_batch_0", type_suffix)- }-pub fn encode(&self,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: QuantizedMatmulArguments,+ arguments: GenericQuantizedMatmulArguments<Metal>,) -> Result<(), QuantizedMatmulError> {- let variant = self.select_variant(args.batch as usize);- let (pipeline, bm, bn) =- self.pipelines.get(&variant).ok_or_else(|| QuantizedMatmulError::InvalidDimensions {- m: args.batch as usize,- n: args.output_dim as usize,- k: args.input_dim as usize,- })?;-- encoder.set_compute_pipeline_state(pipeline);-- // Set buffers- encoder.set_buffer(Some(args.b_buffer), 0, 0);- encoder.set_buffer(Some(args.scales_buffer), 0, 1);- encoder.set_buffer(Some(args.zero_points_or_biases_buffer), 0, 2);- encoder.set_buffer(Some(args.a_buffer), args.a_offset as usize, 3);- encoder.set_buffer(Some(args.output_buffer), 0, 4);-- let k: i32 = args.input_dim;- let m = args.batch;- let n = args.output_dim;-- encoder.set_value(&k, 5);- encoder.set_value(&n, 6);- encoder.set_value(&m, 7);-- match variant {- KernelKind::MatrixVector => {- let bk = 32;- let bn = if self.weights_transposed {- 8- } else {- 64- };- let n_tgp_y = (n + bn - 1) / bn;- let threadgroups = MTLSize::new(m as usize, n_tgp_y as usize, 1);- let threads_per_threadgroup = MTLSize::new(bk as usize, 2, 1);- encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);+ if arguments.quantization_type != self.quantization_type {+ return Err(QuantizedMatmulError::QuantizationTypeMismatch {+ kernel: self.quantization_type,+ args: arguments.quantization_type,+ });+ }++ let key = match self.select_runtime_variant(arguments.batch) {+ RuntimeVariant::MatrixVector => self.matrix_vector_key,+ RuntimeVariant::MatrixMatrix => self.matrix_matrix_key,+ };++ let kernel = self.kernels.get(&key).ok_or(QuantizedMatmulError::MissingKernel(kernel_key_name(key)))?;+ let k = to_i32("input_dim", arguments.input_dim)?;+ let n = to_i32("output_dim", arguments.output_dim)?;+ let m = to_i32("batch", arguments.batch)?;+ let (zero_points, biases) = quant_buffers(arguments.zero_points_or_biases_buffer, self.quantization_type);+ let a_with_offset = (arguments.a_buffer, arguments.a_offset);++ match kernel {+ EncodableVariant::Qmv(kernel) => {+ kernel.encode(+ arguments.b_buffer,+ arguments.scales_buffer,+ zero_points,+ biases,+ a_with_offset,+ arguments.output_buffer,+ k,+ n,+ m,+ encoder,+ );},- KernelKind::MatrixMatrix => {- let wm = 2;- let wn = 2;- let threads_per_threadgroup = MTLSize::new(32, wn as usize, wm as usize);- let threadgroups =- MTLSize::new(((n as u64 + bn - 1) / bn) as usize, ((m as u64 + bm - 1) / bm) as usize, 1);- encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);+ EncodableVariant::QmvFast(kernel) => {+ kernel.encode(+ arguments.b_buffer,+ arguments.scales_buffer,+ zero_points,+ biases,+ a_with_offset,+ arguments.output_buffer,+ k,+ n,+ m,+ encoder,+ );+ },+ EncodableVariant::Qvm(kernel) => {+ kernel.encode(+ arguments.b_buffer,+ arguments.scales_buffer,+ zero_points,+ biases,+ a_with_offset,+ arguments.output_buffer,+ k,+ n,+ m,+ encoder,+ );+ },+ EncodableVariant::Qmm(kernel) => {+ kernel.encode(+ arguments.b_buffer,+ arguments.scales_buffer,+ zero_points,+ biases,+ a_with_offset,+ arguments.output_buffer,+ k,+ n,+ m,+ encoder,+ );+ },+ EncodableVariant::QmmTransposed(kernel) => {+ kernel.encode(+ arguments.b_buffer,+ arguments.scales_buffer,+ zero_points,+ biases,+ a_with_offset,+ arguments.output_buffer,+ k,+ n,+ m,+ encoder,+ );+ },+ EncodableVariant::QmmTransposed64x64(kernel) => {+ kernel.encode(+ arguments.b_buffer,+ arguments.scales_buffer,+ zero_points,+ biases,+ a_with_offset,+ arguments.output_buffer,+ k,+ n,+ m,+ encoder,+ );},}Ok(())}- fn select_variant(+ fn select_runtime_variant(&self,batch: usize,- ) -> KernelKind {+ ) -> RuntimeVariant {if batch < 32 || self.output_dim == 1 {- KernelKind::MatrixVector+ RuntimeVariant::MatrixVector} else {- KernelKind::MatrixMatrix+ RuntimeVariant::MatrixMatrix}}}@@ -243,22 +276,7 @@ impl QuantizedMatmulKernelTrait for QuantizedMatmulKernel {context: &MetalContext,configuration: QuantizedMatmulConfiguration,) -> Result<Self, MetalError> {- let quantization_type = match configuration.quantization_type {- QuantizedMatmulType::ZeroPoint => QuantizationType::ZeroPoint,- QuantizedMatmulType::Mlx => QuantizationType::Mlx,- };-- QuantizedMatmulKernel::new(- context,- configuration.data_type,- configuration.group_size,- configuration.input_dim,- configuration.output_dim,- configuration.mode,- quantization_type,- configuration.weights_transposed,- )- .map_err(|error| MetalError::Generic(format!("{:?}", error)))+ QuantizedMatmulKernel::new(context, configuration).map_err(|error| MetalError::Generic(format!("{error:?}")))}fn encode(@@ -266,270 +284,212 @@ impl QuantizedMatmulKernelTrait for QuantizedMatmulKernel {encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,arguments: GenericQuantizedMatmulArguments<Metal>,) {- let quantization_type = match arguments.quantization_type {- QuantizedMatmulType::ZeroPoint => QuantizationType::ZeroPoint,- QuantizedMatmulType::Mlx => QuantizationType::Mlx,- };-- let backend_arguments = QuantizedMatmulArguments {- a_buffer: arguments.a_buffer,- a_offset: arguments.a_offset as u64,- b_buffer: arguments.b_buffer,- scales_buffer: arguments.scales_buffer,- zero_points_or_biases_buffer: arguments.zero_points_or_biases_buffer,- output_buffer: arguments.output_buffer,- batch: arguments.batch as i32,- input_dim: arguments.input_dim as i32,- output_dim: arguments.output_dim as i32,- quantization_type,- };-- QuantizedMatmulKernel::encode(self, encoder, backend_arguments).expect("Failed to encode quantized matmul");+ QuantizedMatmulKernel::encode(self, encoder, arguments).expect("Failed to encode quantized matmul");}}-fn select_matrix_vector_kernel_name(+fn create_matrix_vector_kernel(+ context: &MetalContext,data_type: DataType,group_size: usize,- weights_transposed: bool,- output_dim: usize,- input_dim: usize,- mode: QuantizationMode,-) -> Result<String, QuantizedMatmulError> {- let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;- let bits = match mode {- QuantizationMode::UInt4 => 4,- QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,+ bits: usize,+ use_mlx_quant: bool,+ family: MatrixVectorFamily,+) -> Result<EncodableVariant, QuantizedMatmulError> {+ let group_size = to_i32("group_size", group_size)?;+ let bits = to_i32("bits", bits)?;+ let use_zero_points = !use_mlx_quant;++ let kernel = match family {+ MatrixVectorFamily::Qmv => EncodableVariant::Qmv(QuantizedMatmulQmvMetalKernel::new(+ context,+ data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )?),+ MatrixVectorFamily::QmvFast => EncodableVariant::QmvFast(QuantizedMatmulQmvFastMetalKernel::new(+ context,+ data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )?),+ MatrixVectorFamily::Qvm => EncodableVariant::Qvm(QuantizedMatmulQvmMetalKernel::new(+ context,+ data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )?),};- if weights_transposed {- let mut name = format!("qmv_{}_g{}_b{}", type_suffix, group_size, bits);- if output_dim % 8 == 0 && input_dim % 512 == 0 {- name.push_str("_fast");- }- return Ok(name);- }-- Ok(format!("qvm_{}_g{}_b{}", type_suffix, group_size, bits))+ Ok(kernel)}-fn select_qmm_kernel_name(+fn create_matrix_matrix_kernel(+ context: &MetalContext,data_type: DataType,group_size: usize,- weights_transposed: bool,- output_dim: usize,- input_dim: usize,- mode: QuantizationMode,-) -> Result<String, QuantizedMatmulError> {- let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;- let bits = match mode {- QuantizationMode::UInt4 => 4,- QuantizationMode::Int8 | QuantizationMode::UInt8 => 8,+ bits: usize,+ use_mlx_quant: bool,+ family: MatrixMatrixFamily,+) -> Result<EncodableVariant, QuantizedMatmulError> {+ let group_size = to_i32("group_size", group_size)?;+ let bits = to_i32("bits", bits)?;+ let use_zero_points = !use_mlx_quant;++ let kernel = match family {+ MatrixMatrixFamily::QmmAlignedK => EncodableVariant::Qmm(QuantizedMatmulQmmMetalKernel::new(+ context,+ data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ true,+ )?),+ MatrixMatrixFamily::QmmUnalignedK => EncodableVariant::Qmm(QuantizedMatmulQmmMetalKernel::new(+ context,+ data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ false,+ )?),+ MatrixMatrixFamily::QmmTransposedAlignedN => {+ EncodableVariant::QmmTransposed(QuantizedMatmulQmmTransposedMetalKernel::new(+ context,+ data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ true,+ )?)+ },+ MatrixMatrixFamily::QmmTransposedUnalignedN => {+ EncodableVariant::QmmTransposed(QuantizedMatmulQmmTransposedMetalKernel::new(+ context,+ data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ false,+ )?)+ },+ MatrixMatrixFamily::QmmTransposed64x64 => {+ EncodableVariant::QmmTransposed64x64(QuantizedMatmulQmmTransposed64x64MetalKernel::new(+ context,+ data_type,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )?)+ },};- let transpose_infix = if weights_transposed {- "_transposed"- } else {- ""- };- let mut kernel_name = base_qmm_kernel_name(type_suffix, group_size, transpose_infix, bits)?;- if weights_transposed {- if output_dim % 32 != 0 {- kernel_name.push_str("_unaligned");- } else if type_suffix == "bf16" && (group_size == 128 || group_size == 64) && (bits == 4 || bits == 8) {- kernel_name.push_str("_64x64");- }- } else if input_dim % 32 == 0 {- kernel_name.push_str("_alignedk");- }- Ok(kernel_name)+ Ok(kernel)}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-enum KernelKind {- MatrixMatrix,- MatrixVector,-}+fn validate_configuration(configuration: &QuantizedMatmulConfiguration) -> Result<(), QuantizedMatmulError> {+ if !matches!(configuration.data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(QuantizedMatmulError::UnsupportedDataType(configuration.data_type));+ }-/// Arguments for MLP fused quantized GEMV-#[derive(Debug)]-pub struct MlpFusedQmvArguments<'a> {- /// Weight matrix [2*hidden_dim, input_dim] (up and gate concatenated, quantized)- pub weights: &'a ProtocolObject<dyn MTLBuffer>,- /// Scales buffer- pub scales: &'a ProtocolObject<dyn MTLBuffer>,- /// Zero points or biases buffer (depends on quantization type)- pub zero_points_or_biases: &'a ProtocolObject<dyn MTLBuffer>,- /// Input vector [input_dim]- pub input: &'a ProtocolObject<dyn MTLBuffer>,- /// Input byte offset- pub input_offset: u64,- /// Output vector [hidden_dim]- pub output: &'a ProtocolObject<dyn MTLBuffer>,- /// Input dimension (K)- pub input_dim: i32,- /// Hidden dimension (output size, half of weight rows)- pub hidden_dim: i32,- /// Batch count- pub batch_count: i32,-}+ if !matches!(configuration.group_size, 32 | 64 | 128) {+ return Err(QuantizedMatmulError::UnsupportedGroupSize(configuration.group_size));+ }-/// MLP Fused Quantized GEMV Kernel-/// Computes paired up/gate projections with fused activation: out = up * activation(gate)-pub struct MlpFusedQmvKernel {- pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,+ let _ = quant_bits(configuration.mode)?;+ Ok(())}-impl MlpFusedQmvKernel {- pub fn new(- context: &MetalContext,- data_type: DataType,- group_size: usize,- mode: QuantizationMode,- quantization_type: QuantizationType,- ) -> Result<Self, QuantizedMatmulError> {- let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;-- let bits = match mode {- QuantizationMode::UInt4 => 4,- QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,- };-- if bits != 4 {- return Err(QuantizedMatmulError::UnsupportedBits(bits));+fn select_matrix_vector_family(configuration: &QuantizedMatmulConfiguration) -> MatrixVectorFamily {+ if configuration.weights_transposed {+ if configuration.output_dim % 8 == 0 && configuration.input_dim % 512 == 0 {+ MatrixVectorFamily::QmvFast+ } else {+ MatrixVectorFamily::Qmv}-- let kernel_name = format!("qmv_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);-- let function_constants = MTLFunctionConstantValues::new();- let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);- function_constants.set_value(&use_mlx_quant, 40);-- let pipeline = context- .compute_pipeline_state(&kernel_name, Some(&function_constants))- .map_err(QuantizedMatmulError::MetalError)?;-- Ok(Self {- pipeline,- })+ } else {+ MatrixVectorFamily::Qvm}+}- pub fn encode(- &self,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: &MlpFusedQmvArguments,- ) -> Result<(), QuantizedMatmulError> {- encoder.set_compute_pipeline_state(&self.pipeline);-- encoder.set_buffer(Some(args.weights), 0, 0);- encoder.set_buffer(Some(args.scales), 0, 1);- encoder.set_buffer(Some(args.zero_points_or_biases), 0, 2);- encoder.set_buffer(Some(args.input), args.input_offset as usize, 3);- encoder.set_buffer(Some(args.output), 0, 4);-- encoder.set_value(&args.input_dim, 5);- encoder.set_value(&args.hidden_dim, 6);-- // Dispatch: one threadgroup per 8 output rows (num_simdgroups * results_per_simdgroup)- let rows_per_threadgroup = 8;- let n_tgp_y = ((args.hidden_dim + rows_per_threadgroup - 1) / rows_per_threadgroup) as u64;- let threadgroups = MTLSize::new(args.batch_count.max(1) as usize, n_tgp_y as usize, 1);- let threads_per_threadgroup = MTLSize::new(32, 2, 1); // 2 simdgroups, 32 threads each-- encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);- Ok(())+fn select_matrix_matrix_family(+ configuration: &QuantizedMatmulConfiguration,+ bits: usize,+) -> MatrixMatrixFamily {+ if configuration.weights_transposed {+ let aligned_n = configuration.output_dim % 32 == 0;+ let use_64x64 = aligned_n+ && configuration.data_type == DataType::BF16+ && matches!(configuration.group_size, 64 | 128)+ && matches!(bits, 4 | 8);+ if use_64x64 {+ MatrixMatrixFamily::QmmTransposed64x64+ } else if aligned_n {+ MatrixMatrixFamily::QmmTransposedAlignedN+ } else {+ MatrixMatrixFamily::QmmTransposedUnalignedN+ }+ } else if configuration.input_dim % 32 == 0 {+ MatrixMatrixFamily::QmmAlignedK+ } else {+ MatrixMatrixFamily::QmmUnalignedK}}-/// Arguments for MLP fused quantized GEMM (prefill)-#[derive(Debug)]-pub struct MlpFusedQmmArguments<'a> {- /// Weight matrix [K, 2*hidden_dim] quantized (up and gate concatenated)- pub weights: &'a ProtocolObject<dyn MTLBuffer>,- /// Scales buffer- pub scales: &'a ProtocolObject<dyn MTLBuffer>,- /// Zero points or biases buffer (depends on quantization type)- pub zero_points_or_biases: &'a ProtocolObject<dyn MTLBuffer>,- /// Input activations [M, K]- pub input: &'a ProtocolObject<dyn MTLBuffer>,- /// Input byte offset- pub input_offset: u64,- /// Output [M, hidden_dim]- pub output: &'a ProtocolObject<dyn MTLBuffer>,- /// Batch size (M)- pub batch: i32,- /// Input dimension (K)- pub input_dim: i32,- /// Hidden dimension (output size, half of weight columns)- pub hidden_dim: i32,+fn quant_bits(mode: QuantizationMode) -> Result<usize, QuantizedMatmulError> {+ let bits = match mode {+ QuantizationMode::UInt4 => 4,+ QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,+ };+ if matches!(bits, 4 | 8) {+ Ok(bits)+ } else {+ Err(QuantizedMatmulError::UnsupportedBits(bits))+ }}-/// MLP Fused Quantized GEMM Kernel for prefill path-/// Computes paired up/gate projections with fused activation: out = up * activation(gate)-pub struct MlpFusedQmmKernel {- pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,+fn to_i32(+ name: &'static str,+ value: usize,+) -> Result<i32, QuantizedMatmulError> {+ i32::try_from(value).map_err(|_| QuantizedMatmulError::ValueOutOfRange {+ name,+ value,+ })}-impl MlpFusedQmmKernel {- pub fn new(- context: &MetalContext,- data_type: DataType,- group_size: usize,- mode: QuantizationMode,- quantization_type: QuantizationType,- ) -> Result<Self, QuantizedMatmulError> {- let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;-- let bits = match mode {- QuantizationMode::UInt4 => 4,- QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,- };-- if bits != 4 {- return Err(QuantizedMatmulError::UnsupportedBits(bits));- }-- let kernel_name = format!("qmm_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);-- let function_constants = MTLFunctionConstantValues::new();- let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);- function_constants.set_value(&use_mlx_quant, 40);-- let pipeline = context- .compute_pipeline_state(&kernel_name, Some(&function_constants))- .map_err(QuantizedMatmulError::MetalError)?;-- Ok(Self {- pipeline,- })+fn quant_buffers<'a>(+ buffer: &'a Retained<ProtocolObject<dyn MTLBuffer>>,+ quantization_type: QuantizedMatmulType,+) -> (Option<&'a Retained<ProtocolObject<dyn MTLBuffer>>>, Option<&'a Retained<ProtocolObject<dyn MTLBuffer>>>) {+ match quantization_type {+ QuantizedMatmulType::ZeroPoint => (Some(buffer), None),+ QuantizedMatmulType::Mlx => (None, Some(buffer)),}+}- pub fn encode(- &self,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: &MlpFusedQmmArguments,- ) -> Result<(), QuantizedMatmulError> {- encoder.set_compute_pipeline_state(&self.pipeline);-- encoder.set_buffer(Some(args.weights), 0, 0);- encoder.set_buffer(Some(args.scales), 0, 1);- encoder.set_buffer(Some(args.zero_points_or_biases), 0, 2);- encoder.set_buffer(Some(args.input), args.input_offset as usize, 3);- encoder.set_buffer(Some(args.output), 0, 4);-- encoder.set_value(&args.input_dim, 5);- encoder.set_value(&args.hidden_dim, 6);- encoder.set_value(&args.batch, 7);-- // Dispatch: BM=32, BN=32- let bm = 32;- let bn = 32;- let tiles_m = ((args.batch + bm - 1) / bm) as u64;- let tiles_n = ((args.hidden_dim + bn - 1) / bn) as u64;- let threadgroups = MTLSize::new(tiles_n as usize, tiles_m as usize, 1);- let threads_per_threadgroup = MTLSize::new(32, 2, 2); // WM=2, WN=2, 32 threads-- encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);- Ok(())+fn kernel_key_name(key: KernelKey) -> &'static str {+ match key {+ KernelKey::MatrixVector(MatrixVectorFamily::Qmv) => "matrix_vector_qmv",+ KernelKey::MatrixVector(MatrixVectorFamily::QmvFast) => "matrix_vector_qmv_fast",+ KernelKey::MatrixVector(MatrixVectorFamily::Qvm) => "matrix_vector_qvm",+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmAlignedK) => "matrix_matrix_qmm_aligned_k",+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmUnalignedK) => "matrix_matrix_qmm_unaligned_k",+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedAlignedN) => "matrix_matrix_qmm_transposed_aligned_n",+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedUnalignedN) => {+ "matrix_matrix_qmm_transposed_unaligned_n"+ },+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposed64x64) => "matrix_matrix_qmm_transposed_64x64",}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metalnew file mode 100644index 000000000..33232756b--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal@@ -0,0 +1,113 @@+#include <metal_stdlib>+#include "../definitions.metal"+#include "common.h"++template <typename T, int group_size, int bits>+VARIANTS(T, float, half, bfloat)+VARIANTS(group_size, 32, 64, 128)+VARIANTS(bits, 4, 8)+KERNEL(QuantizedMatmulQmm)(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points OPTIONAL(use_zero_points),+ const device T* biases OPTIONAL(use_mlx_quant),+ const device T* x,+ device T* y,+ const constant int& k,+ const constant int& n,+ const constant int& m,+ threadgroup T Xs[32 * (32 + 16 / sizeof(T))],+ threadgroup T Ws[32 * (32 + 16 / sizeof(T))],+ const bool use_zero_points SPECIALIZE,+ const bool use_mlx_quant SPECIALIZE,+ const bool aligned_k SPECIALIZE,+ const uint tgid_x GROUPS((n + 32 - 1) / 32),+ const uint tgid_y GROUPS((m + 32 - 1) / 32),+ const uint tgid_z GROUPS(1),+ const uint tid_x THREADS(32),+ const uint tid_y THREADS(2),+ const uint tid_z THREADS(2)+) {+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);+ const uint lid = tid_z * 64 + tid_y * 32 + tid_x;+ const uint simd_gid = tid_z * 2 + tid_y;+ const uint simd_lid = tid_x;++ if (use_mlx_quant) {+ if (aligned_k) {+ qmm_impl<T, group_size, bits, true, 32, 32, 32, true>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ } else {+ qmm_impl<T, group_size, bits, false, 32, 32, 32, true>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ }+ } else {+ if (aligned_k) {+ qmm_impl<T, group_size, bits, true, 32, 32, 32, false>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ } else {+ qmm_impl<T, group_size, bits, false, 32, 32, 32, false>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ }+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metalnew file mode 100644index 000000000..2afb794fa--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal@@ -0,0 +1,113 @@+#include <metal_stdlib>+#include "../definitions.metal"+#include "common.h"++template <typename T, int group_size, int bits>+VARIANTS(T, float, half, bfloat)+VARIANTS(group_size, 32, 64, 128)+VARIANTS(bits, 4, 8)+KERNEL(QuantizedMatmulQmmTransposed)(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points OPTIONAL(use_zero_points),+ const device T* biases OPTIONAL(use_mlx_quant),+ const device T* x,+ device T* y,+ const constant int& k,+ const constant int& n,+ const constant int& m,+ threadgroup T Xs[32 * (32 + 16 / sizeof(T))],+ threadgroup T Ws[32 * (32 + 16 / sizeof(T))],+ const bool use_zero_points SPECIALIZE,+ const bool use_mlx_quant SPECIALIZE,+ const bool aligned_n SPECIALIZE,+ const uint tgid_x GROUPS((n + 32 - 1) / 32),+ const uint tgid_y GROUPS((m + 32 - 1) / 32),+ const uint tgid_z GROUPS(1),+ const uint tid_x THREADS(32),+ const uint tid_y THREADS(2),+ const uint tid_z THREADS(2)+) {+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);+ const uint lid = tid_z * 64 + tid_y * 32 + tid_x;+ const uint simd_gid = tid_z * 2 + tid_y;+ const uint simd_lid = tid_x;++ if (use_mlx_quant) {+ if (aligned_n) {+ qmm_transposed_impl<T, group_size, bits, true, 32, 32, 32, true>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ } else {+ qmm_transposed_impl<T, group_size, bits, false, 32, 32, 32, true>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ }+ } else {+ if (aligned_n) {+ qmm_transposed_impl<T, group_size, bits, true, 32, 32, 32, false>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ } else {+ qmm_transposed_impl<T, group_size, bits, false, 32, 32, 32, false>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ }+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metalnew file mode 100644index 000000000..0f7323e35--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal@@ -0,0 +1,72 @@+#include <metal_stdlib>+#include "../definitions.metal"+#include "common.h"++template <typename T, int group_size, int bits>+VARIANTS(T, bfloat)+VARIANTS(group_size, 64, 128)+VARIANTS(bits, 4, 8)+KERNEL(QuantizedMatmulQmmTransposed64x64)(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points OPTIONAL(use_zero_points),+ const device T* biases OPTIONAL(use_mlx_quant),+ const device T* x,+ device T* y,+ const constant int& k,+ const constant int& n,+ const constant int& m,+ threadgroup T Xs[64 * (64 + 16 / sizeof(T))],+ threadgroup T Ws[64 * (64 + 16 / sizeof(T))],+ const bool use_zero_points SPECIALIZE,+ const bool use_mlx_quant SPECIALIZE,+ const uint tgid_x GROUPS((n + 64 - 1) / 64),+ const uint tgid_y GROUPS((m + 64 - 1) / 64),+ const uint tgid_z GROUPS(1),+ const uint tid_x THREADS(32),+ const uint tid_y THREADS(2),+ const uint tid_z THREADS(2)+) {+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);+ const uint lid = tid_z * 64 + tid_y * 32 + tid_x;+ const uint simd_gid = tid_z * 2 + tid_y;+ const uint simd_lid = tid_x;++ if (use_mlx_quant) {+ qmm_transposed_impl<T, group_size, bits, true, 64, 64, 64, true>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ } else {+ qmm_transposed_impl<T, group_size, bits, true, 64, 64, 64, false>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ k,+ n,+ m,+ tid,+ lid,+ simd_gid,+ simd_lid+ );+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metalnew file mode 100644index 000000000..ac62ebb48--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal@@ -0,0 +1,60 @@+#include <metal_stdlib>+#include "../definitions.metal"+#include "common.h"++template <typename T, int group_size, int bits>+VARIANTS(T, float, half, bfloat)+VARIANTS(group_size, 32, 64, 128)+VARIANTS(bits, 4, 8)+KERNEL(QuantizedMatmulQmv)(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points OPTIONAL(use_zero_points),+ const device T* biases OPTIONAL(use_mlx_quant),+ const device T* x,+ device T* y,+ const constant int& k,+ const constant int& n,+ const constant int& m,+ const bool use_zero_points SPECIALIZE,+ const bool use_mlx_quant SPECIALIZE,+ const uint tgid_x GROUPS(m),+ const uint tgid_y GROUPS((n + 8 - 1) / 8),+ const uint tgid_z GROUPS(1),+ const uint tid_x THREADS(32),+ const uint tid_y THREADS(2)+) {+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);+ const uint simd_gid = tid_y;+ const uint simd_lid = tid_x;++ if (use_mlx_quant) {+ qmv_impl<T, group_size, bits, true>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ k,+ n,+ tid,+ simd_gid,+ simd_lid+ );+ } else {+ qmv_impl<T, group_size, bits, false>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ k,+ n,+ tid,+ simd_gid,+ simd_lid+ );+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metalnew file mode 100644index 000000000..c911b1b3e--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal@@ -0,0 +1,60 @@+#include <metal_stdlib>+#include "../definitions.metal"+#include "common.h"++template <typename T, int group_size, int bits>+VARIANTS(T, float, half, bfloat)+VARIANTS(group_size, 32, 64, 128)+VARIANTS(bits, 4, 8)+KERNEL(QuantizedMatmulQmvFast)(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points OPTIONAL(use_zero_points),+ const device T* biases OPTIONAL(use_mlx_quant),+ const device T* x,+ device T* y,+ const constant int& k,+ const constant int& n,+ const constant int& m,+ const bool use_zero_points SPECIALIZE,+ const bool use_mlx_quant SPECIALIZE,+ const uint tgid_x GROUPS(m),+ const uint tgid_y GROUPS((n + 8 - 1) / 8),+ const uint tgid_z GROUPS(1),+ const uint tid_x THREADS(32),+ const uint tid_y THREADS(2)+) {+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);+ const uint simd_gid = tid_y;+ const uint simd_lid = tid_x;++ if (use_mlx_quant) {+ qmv_fast_impl<T, group_size, bits, true>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ k,+ n,+ tid,+ simd_gid,+ simd_lid+ );+ } else {+ qmv_fast_impl<T, group_size, bits, false>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ k,+ n,+ tid,+ simd_gid,+ simd_lid+ );+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalindex 9e83f09e3..c2408b656 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -4,12 +4,6 @@using namespace metal;#include "mma.h"-#include "../common/mlp_epilogue.h"--// Function constant: true = MLX-style (pre-computed biases), false = AWQ-style-// (zero-points)-constant bool kUseMlxQuant [[function_constant(40)]];-constant bool kUseZeroPoints = !kUseMlxQuant;template <int bits, int wsize = 8>inline constexpr short get_pack_factor() {@@ -731,7 +725,8 @@ template <const bool aligned_K = false,const int BM = 32,const int BK = 32,- const int BN = 32>+ const int BN = 32,+ const bool use_mlx_quant = false>void qmm_impl(const device uint32_t* w,const device T* scales,@@ -783,7 +778,7 @@ void qmm_impl(loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);mma_t mma_op(simd_gid, simd_lid);- if (kUseMlxQuant) {+ if (use_mlx_quant) {using loader_w_t = QuantizedBlockLoaderMlx<T,BK,@@ -939,7 +934,8 @@ template <const bool aligned_N,const int BM = 32,const int BK = 32,- const int BN = 32>+ const int BN = 32,+ const bool use_mlx_quant = false>void qmm_transposed_impl(const device uint32_t* w,const device T* scales,@@ -989,7 +985,7 @@ void qmm_transposed_impl(loader_x_t loader_x(x_block, K, Xs, simd_gid, simd_lid);mma_t mma_op(simd_gid, simd_lid);- if (kUseMlxQuant) {+ if (use_mlx_quant) {using loader_w_t = QuantizedBlockLoaderMlx<T,BN,@@ -1317,7 +1313,7 @@ void qmv_impl_dispatch(}}-template <typename T, int group_size, int bits>+template <typename T, int group_size, int bits, bool use_mlx_quant>void qmv_impl(const device uint32_t* w,const device T* scales,@@ -1331,7 +1327,7 @@ void qmv_impl(uint simd_gid [[simdgroup_index_in_threadgroup]],uint simd_lid [[thread_index_in_simdgroup]]) {- if (kUseMlxQuant) {+ if (use_mlx_quant) {qmv_impl_dispatch<T, group_size, bits, true>(w,scales,@@ -1362,7 +1358,7 @@ void qmv_impl(}}-template <typename T, int group_size, int bits>+template <typename T, int group_size, int bits, bool use_mlx_quant>void qmv_fast_impl(const device uint32_t* w,const device T* scales,@@ -1400,7 +1396,7 @@ void qmv_fast_impl(const device uint8_t* zps = nullptr;bool high_nibble = false;- if (kUseMlxQuant) {+ if (use_mlx_quant) {biases += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;} else {if (bits == 4) {@@ -1433,7 +1429,7 @@ void qmv_fast_impl(U s2 = static_cast<U>(scales[2 * in_vec_size_g]);U s3 = static_cast<U>(scales[3 * in_vec_size_g]);- if (kUseMlxQuant) {+ if (use_mlx_quant) {U b0 = static_cast<U>(biases[0]);U b1 = static_cast<U>(biases[in_vec_size_g]);U b2 = static_cast<U>(biases[2 * in_vec_size_g]);@@ -1482,7 +1478,7 @@ void qmv_fast_impl(ws += block_size * bytes_per_pack / pack_factor;scales += block_size / group_size;- if (kUseMlxQuant) {+ if (use_mlx_quant) {biases += block_size / group_size;} else {if (bits == 4) {@@ -1674,7 +1670,7 @@ void qvm_impl_zeropoint();}-template <typename T, int group_size, int bits>+template <typename T, int group_size, int bits, bool use_mlx_quant>void qvm_impl(const device uint32_t* w,const device T* scales,@@ -1688,7 +1684,7 @@ void qvm_impl(uint simd_gid [[simdgroup_index_in_threadgroup]],uint simd_lid [[thread_index_in_simdgroup]]) {- if (kUseMlxQuant) {+ if (use_mlx_quant) {qvm_impl_mlx<T, group_size, bits>(w,scales,@@ -1716,3304 +1712,3 @@ void qvm_impl();}}--template <- typename T,- const int group_size,- const int bits,- const bool aligned_K,- const int BM = 32,- const int BK = 32,- const int BN = 32>-[[kernel, max_total_threads_per_threadgroup(128)]] void qmm(- const device uint32_t* w [[buffer(0)]],- const device T* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device T* x [[buffer(3)]],- device T* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {-- (void)lid;-- constexpr int BK_padded = (BK + 16 / sizeof(T));- constexpr int BN_padded = (BN + 16 / sizeof(T));-- threadgroup T Xs[BM * BK_padded];- threadgroup T Ws[BK * BN_padded];-- qmm_impl<T, group_size, bits, aligned_K, BM, BK, BN>(- w,- scales,- zero_points,- biases,- x,- y,- Xs,- Ws,- K,- N,- M,- tid,- lid,- simd_gid,- simd_lid- );-}--template <- typename T,- const int group_size,- const int bits,- const bool aligned_N,- const int BM = 32,- const int BK = 32,- const int BN = 32>-[[kernel, max_total_threads_per_threadgroup(128)]] void qmm_transposed(- const device uint32_t* w [[buffer(0)]],- const device T* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device T* x [[buffer(3)]],- device T* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {-- (void)lid;-- constexpr int BK_padded = (BK + 16 / sizeof(T));-- threadgroup T Xs[BM * BK_padded];- threadgroup T Ws[BN * BK_padded];-- qmm_transposed_impl<T, group_size, bits, aligned_N, BM, BK, BN>(- w,- scales,- zero_points,- biases,- x,- y,- Xs,- Ws,- K,- N,- M,- tid,- lid,- simd_gid,- simd_lid- );-}--template <typename T, int group_size, int bits>-[[kernel, max_total_threads_per_threadgroup(64)]] void qmv(- const device uint32_t* w [[buffer(0)]],- const device T* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device T* x [[buffer(3)]],- device T* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {-- qmv_impl<T, group_size, bits>(- w,- scales,- zero_points,- biases,- x,- y,- K,- N,- tid,- simd_gid,- simd_lid- );-}--template <typename T, int group_size, int bits>-[[kernel, max_total_threads_per_threadgroup(64)]] void qvm(- const device uint32_t* w [[buffer(0)]],- const device T* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device T* x [[buffer(3)]],- device T* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {-- qvm_impl<T, group_size, bits>(- w,- scales,- zero_points,- biases,- x,- y,- K,- N,- tid,- simd_gid,- simd_lid- );-}--template <typename T, int group_size, int bits>-[[kernel, max_total_threads_per_threadgroup(64)]] void qmv_fast(- const device uint32_t* w [[buffer(0)]],- const device T* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device T* x [[buffer(3)]],- device T* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- qmv_fast_impl<T, group_size, bits>(- w,- scales,- zero_points,- biases,- x,- y,- K,- N,- tid,- simd_gid,- simd_lid- );-}--template [[host_name("qmv_f16_g32_b4_fast")]] [[kernel]] void qmv_fast<- half,- 32,- 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g64_b4_fast")]] [[kernel]] void qmv_fast<- half,- 64,- 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g128_b4_fast")]] [[kernel]] void qmv_fast<- half,- 128,- 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g32_b4_fast")]] [[kernel]] void qmv_fast<- bfloat,- 32,- 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g64_b4_fast")]] [[kernel]] void qmv_fast<- bfloat,- 64,- 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g128_b4_fast")]] [[kernel]] void qmv_fast<- bfloat,- 128,- 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g32_b4_fast")]] [[kernel]] void qmv_fast<- float,- 32,- 4>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g64_b4_fast")]] [[kernel]] void qmv_fast<- float,- 64,- 4>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g128_b4_fast")]] [[kernel]] void qmv_fast<- float,- 128,- 4>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g32_b8_fast")]] [[kernel]] void qmv_fast<- half,- 32,- 8>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g64_b8_fast")]] [[kernel]] void qmv_fast<- half,- 64,- 8>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g128_b8_fast")]] [[kernel]] void qmv_fast<- half,- 128,- 8>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g32_b8_fast")]] [[kernel]] void qmv_fast<- bfloat,- 32,- 8>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g64_b8_fast")]] [[kernel]] void qmv_fast<- bfloat,- 64,- 8>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g128_b8_fast")]] [[kernel]] void qmv_fast<- bfloat,- 128,- 8>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g32_b8_fast")]] [[kernel]] void qmv_fast<- float,- 32,- 8>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g64_b8_fast")]] [[kernel]] void qmv_fast<- float,- 64,- 8>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g128_b8_fast")]] [[kernel]] void qmv_fast<- float,- 128,- 8>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 32 (F16)-template [[host_name("qmm_f16_g32_b4")]] [[kernel]] void qmm<- half,- 32,- 4,- false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g32_b4")]] [[kernel]] void-qmm_transposed<half, 32, 4, true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g32_b4_unaligned")]] [[kernel]] void-qmm_transposed<half, 32, 4, false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f16_g32_b4_alignedk")]] [[kernel]] void qmm<- half,- 32,- 4,- true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g32_b4")]] [[kernel]] void qmv<half, 32, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f16_g32_b4")]] [[kernel]] void qvm<half, 32, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 32 (F16) - 8 bit-template [[host_name("qmm_f16_g32_b8")]] [[kernel]] void qmm<- half,- 32,- 8,- false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g32_b8")]] [[kernel]] void-qmm_transposed<half, 32, 8, true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g32_b8_unaligned")]] [[kernel]] void-qmm_transposed<half, 32, 8, false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f16_g32_b8_alignedk")]] [[kernel]] void qmm<- half,- 32,- 8,- true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g32_b8")]] [[kernel]] void qmv<half, 32, 8>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f16_g32_b8")]] [[kernel]] void qvm<half, 32, 8>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 64 (F16)-template [[host_name("qmm_f16_g64_b4")]] [[kernel]] void qmm<- half,- 64,- 4,- false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f16_g64_b4_alignedk")]] [[kernel]] void qmm<- half,- 64,- 4,- true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g64_b4")]] [[kernel]] void-qmm_transposed<half, 64, 4, true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g64_b4_unaligned")]] [[kernel]] void-qmm_transposed<half, 64, 4, false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g64_b4")]] [[kernel]] void qmv<half, 64, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f16_g64_b4")]] [[kernel]] void qvm<half, 64, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 64 (F16) - 8 bit-template [[host_name("qmm_f16_g64_b8")]] [[kernel]] void qmm<- half,- 64,- 8,- false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f16_g64_b8_alignedk")]] [[kernel]] void qmm<- half,- 64,- 8,- true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g64_b8")]] [[kernel]] void-qmm_transposed<half, 64, 8, true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g64_b8_unaligned")]] [[kernel]] void-qmm_transposed<half, 64, 8, false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g64_b8")]] [[kernel]] void qmv<half, 64, 8>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f16_g64_b8")]] [[kernel]] void qvm<half, 64, 8>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 128 (F16)-template [[host_name("qmm_f16_g128_b4")]] [[kernel]] void qmm<- half,- 128,- 4,- false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f16_g128_b4_alignedk")]] [[kernel]] void qmm<- half,- 128,- 4,- true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g128_b4")]] [[kernel]] void-qmm_transposed<half, 128, 4, true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g128_b4_unaligned")]] [[kernel]] void-qmm_transposed<half, 128, 4, false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g128_b4")]] [[kernel]] void qmv<half, 128, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f16_g128_b4")]] [[kernel]] void qvm<half, 128, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 128 (F16) - 8 bit-template [[host_name("qmm_f16_g128_b8")]] [[kernel]] void qmm<- half,- 128,- 8,- false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f16_g128_b8_alignedk")]] [[kernel]] void qmm<- half,- 128,- 8,- true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g128_b8")]] [[kernel]] void-qmm_transposed<half, 128, 8, true>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f16_g128_b8_unaligned")]] [[kernel]] void-qmm_transposed<half, 128, 8, false>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f16_g128_b8")]] [[kernel]] void qmv<half, 128, 8>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f16_g128_b8")]] [[kernel]] void qvm<half, 128, 8>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 32 (F32)-template [[host_name("qmm_f32_g32_b4")]] [[kernel]] void qmm<- float,- 32,- 4,- false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g32_b4")]] [[kernel]] void-qmm_transposed<float, 32, 4, true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g32_b4_unaligned")]] [[kernel]] void-qmm_transposed<float, 32, 4, false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f32_g32_b4_alignedk")]] [[kernel]] void qmm<- float,- 32,- 4,- true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g32_b4")]] [[kernel]] void qmv<float, 32, 4>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f32_g32_b4")]] [[kernel]] void qvm<float, 32, 4>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 32 (F32) - 8 bit-template [[host_name("qmm_f32_g32_b8")]] [[kernel]] void qmm<- float,- 32,- 8,- false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g32_b8")]] [[kernel]] void-qmm_transposed<float, 32, 8, true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g32_b8_unaligned")]] [[kernel]] void-qmm_transposed<float, 32, 8, false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f32_g32_b8_alignedk")]] [[kernel]] void qmm<- float,- 32,- 8,- true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g32_b8")]] [[kernel]] void qmv<float, 32, 8>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f32_g32_b8")]] [[kernel]] void qvm<float, 32, 8>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 64 (F32)-template [[host_name("qmm_f32_g64_b4")]] [[kernel]] void qmm<- float,- 64,- 4,- false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f32_g64_b4_alignedk")]] [[kernel]] void qmm<- float,- 64,- 4,- true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g64_b4")]] [[kernel]] void-qmm_transposed<float, 64, 4, true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g64_b4_unaligned")]] [[kernel]] void-qmm_transposed<float, 64, 4, false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g64_b4")]] [[kernel]] void qmv<float, 64, 4>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f32_g64_b4")]] [[kernel]] void qvm<float, 64, 4>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 64 (F32) - 8 bit-template [[host_name("qmm_f32_g64_b8")]] [[kernel]] void qmm<- float,- 64,- 8,- false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f32_g64_b8_alignedk")]] [[kernel]] void qmm<- float,- 64,- 8,- true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g64_b8")]] [[kernel]] void-qmm_transposed<float, 64, 8, true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g64_b8_unaligned")]] [[kernel]] void-qmm_transposed<float, 64, 8, false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g64_b8")]] [[kernel]] void qmv<float, 64, 8>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f32_g64_b8")]] [[kernel]] void qvm<float, 64, 8>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 128 (F32)-template [[host_name("qmm_f32_g128_b4")]] [[kernel]] void qmm<- float,- 128,- 4,- false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f32_g128_b4_alignedk")]] [[kernel]] void qmm<- float,- 128,- 4,- true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g128_b4")]] [[kernel]] void-qmm_transposed<float, 128, 4, true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g128_b4_unaligned")]] [[kernel]] void-qmm_transposed<float, 128, 4, false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g128_b4")]] [[kernel]] void qmv<float, 128, 4>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f32_g128_b4")]] [[kernel]] void qvm<float, 128, 4>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 128 (F32) - 8 bit-template [[host_name("qmm_f32_g128_b8")]] [[kernel]] void qmm<- float,- 128,- 8,- false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_f32_g128_b8_alignedk")]] [[kernel]] void qmm<- float,- 128,- 8,- true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g128_b8")]] [[kernel]] void-qmm_transposed<float, 128, 8, true>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_f32_g128_b8_unaligned")]] [[kernel]] void-qmm_transposed<float, 128, 8, false>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_f32_g128_b8")]] [[kernel]] void qmv<float, 128, 8>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device float* biases [[buffer(2)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_f32_g128_b8")]] [[kernel]] void qvm<float, 128, 8>(- const device uint32_t* w [[buffer(0)]],- const device float* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device float* x [[buffer(3)]],- device float* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 32 (BF16)-template [[host_name("qmm_bf16_g32_b4")]] [[kernel]] void qmm<- bfloat,- 32,- 4,- false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_bf16_g32_b4_alignedk")]] [[kernel]] void qmm<- bfloat,- 32,- 4,- true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g32_b4")]] [[kernel]] void-qmm_transposed<bfloat, 32, 4, true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g32_b4_unaligned")]] [[kernel]] void-qmm_transposed<bfloat, 32, 4, false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g32_b4")]] [[kernel]] void qmv<bfloat, 32, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_bf16_g32_b4")]] [[kernel]] void qvm<bfloat, 32, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 32 (BF16) - 8 bit-template [[host_name("qmm_bf16_g32_b8")]] [[kernel]] void qmm<- bfloat,- 32,- 8,- false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_bf16_g32_b8_alignedk")]] [[kernel]] void qmm<- bfloat,- 32,- 8,- true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g32_b8")]] [[kernel]] void-qmm_transposed<bfloat, 32, 8, true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g32_b8_unaligned")]] [[kernel]] void-qmm_transposed<bfloat, 32, 8, false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g32_b8")]] [[kernel]] void qmv<bfloat, 32, 8>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_bf16_g32_b8")]] [[kernel]] void qvm<bfloat, 32, 8>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 64 (BF16)-template [[host_name("qmm_bf16_g64_b4")]] [[kernel]] void qmm<- bfloat,- 64,- 4,- false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_bf16_g64_b4_alignedk")]] [[kernel]] void qmm<- bfloat,- 64,- 4,- true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g64_b4")]] [[kernel]] void-qmm_transposed<bfloat, 64, 4, true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g64_b4_unaligned")]] [[kernel]] void-qmm_transposed<bfloat, 64, 4, false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g64_b4_64x64")]] [[kernel]] void-qmm_transposed<bfloat, 64, 4, false, 64, 32, 64>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g64_b4")]] [[kernel]] void qmv<bfloat, 64, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_bf16_g64_b4")]] [[kernel]] void qvm<bfloat, 64, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 64 (BF16) - 8 bit-template [[host_name("qmm_bf16_g64_b8")]] [[kernel]] void qmm<- bfloat,- 64,- 8,- false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_bf16_g64_b8_alignedk")]] [[kernel]] void qmm<- bfloat,- 64,- 8,- true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g64_b8")]] [[kernel]] void-qmm_transposed<bfloat, 64, 8, true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g64_b8_unaligned")]] [[kernel]] void-qmm_transposed<bfloat, 64, 8, false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g64_b8_64x64")]] [[kernel]] void-qmm_transposed<bfloat, 64, 8, false, 64, 32, 64>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g64_b8")]] [[kernel]] void qmv<bfloat, 64, 8>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_bf16_g64_b8")]] [[kernel]] void qvm<bfloat, 64, 8>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 128 (BF16)-template [[host_name("qmm_bf16_g128_b4")]] [[kernel]] void qmm<- bfloat,- 128,- 4,- false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_bf16_g128_b4_alignedk")]] [[kernel]] void qmm<- bfloat,- 128,- 4,- true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g128_b4")]] [[kernel]] void-qmm_transposed<bfloat, 128, 4, true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g128_b4_unaligned")]] [[kernel]] void-qmm_transposed<bfloat, 128, 4, false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g128_b4")]] [[kernel]] void qmv<bfloat, 128, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_bf16_g128_b4")]] [[kernel]] void qvm<bfloat, 128, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--// Group size 128 (BF16) - 8 bit-template [[host_name("qmm_bf16_g128_b8")]] [[kernel]] void qmm<- bfloat,- 128,- 8,- false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_bf16_g128_b8_alignedk")]] [[kernel]] void qmm<- bfloat,- 128,- 8,- true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g128_b8")]] [[kernel]] void-qmm_transposed<bfloat, 128, 8, true>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g128_b8_unaligned")]] [[kernel]] void-qmm_transposed<bfloat, 128, 8, false>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_bf16_g128_b8")]] [[kernel]] void qmv<bfloat, 128, 8>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qvm_bf16_g128_b8")]] [[kernel]] void qvm<bfloat, 128, 8>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g128_b4_64x64")]] [[kernel]] void-qmm_transposed<bfloat, 128, 4, true, 64, 32, 64>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g128_b8_64x64")]] [[kernel]] void-qmm_transposed<bfloat, 128, 8, true, 64, 32, 64>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g128_b4_64x128")]] [[kernel]] void-qmm_transposed<bfloat, 128, 4, true, 64, 32, 128>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_transposed_bf16_g128_b4_128x64")]] [[kernel]] void-qmm_transposed<bfloat, 128, 4, true, 128, 32, 64>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& N [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--///////////////////////////////////////////////////////////////////////////////-/// MLP Fused QMV - Quantized Matrix-Vector with Fused Activation-///////////////////////////////////////////////////////////////////////////////--template <typename U>-static inline U extract_zp(- const device uint8_t* zps,- int row_offset,- int zp_stride,- int bits,- bool high_nibble-) {- uint8_t zp_byte = zps[row_offset * zp_stride];- return bits == 4- ? static_cast<U>(high_nibble ? (zp_byte >> 4) : (zp_byte & 0x0F))- : static_cast<U>(zp_byte);-};--// MLP fused QMV kernel for decode path (M=1).-// Computes paired up and gate projections, then applies: out = up *-// activation(gate) Weight layout: [up_weights (hidden_dim rows), gate_weights-// (hidden_dim rows)] Output size is hidden_dim.-template <typename T, int group_size, int bits>-void qmv_mlp_fused_impl(- const device uint32_t* w,- const device T* scales,- const device uint8_t* zero_points,- const device T* biases,- const device T* x,- device T* y,- const constant int& in_vec_size,- const constant int& hidden_dim, // Output size (half of weight rows)- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- constexpr int packs_per_thread = bits == 2 ? 1 : 2;- constexpr int num_simdgroups = 2;- constexpr int results_per_simdgroup = 4; // Compute 4 paired rows at a time- constexpr int pack_factor = get_pack_factor<bits, 32>();- constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();- constexpr int values_per_thread = pack_factor * packs_per_thread;- constexpr int block_size = values_per_thread * SIMD_SIZE;- constexpr int scale_step_per_thread = group_size / values_per_thread;- const device uint8_t* ws = (const device uint8_t*)w;- typedef float U;- thread U x_thread[values_per_thread];- thread U result_up[results_per_simdgroup] = {0};- thread U result_gate[results_per_simdgroup] = {0};-- const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;- const int in_vec_size_g = in_vec_size / group_size;-- // Output row for 'up' projection- const int out_row_up = tid.y * (num_simdgroups * results_per_simdgroup) +- simd_gid * results_per_simdgroup;-- // Corresponding 'gate' row is offset by hidden_dim- const int out_row_gate = out_row_up + hidden_dim;-- // Check bounds - only process if up row is within hidden_dim- if (out_row_up >= hidden_dim)- return;-- // Pointer setup for up weights- const device uint8_t* ws_up = ws + out_row_up * in_vec_size_w +- simd_lid * packs_per_thread * bytes_per_pack;- const device T* scales_up =- scales + out_row_up * in_vec_size_g + simd_lid / scale_step_per_thread;-- // Pointer setup for gate weights- const device uint8_t* ws_gate = ws + out_row_gate * in_vec_size_w +- simd_lid * packs_per_thread * bytes_per_pack;- const device T* scales_gate =- scales + out_row_gate * in_vec_size_g + simd_lid / scale_step_per_thread;-- const device T* biases_up = nullptr;- const device T* biases_gate = nullptr;- const device uint8_t* zps_up = nullptr;- const device uint8_t* zps_gate = nullptr;- int zp_stride = 0;- bool high_nibble = false;-- if (kUseMlxQuant) {- biases_up =- biases + out_row_up * in_vec_size_g + simd_lid / scale_step_per_thread;- biases_gate = biases + out_row_gate * in_vec_size_g +- simd_lid / scale_step_per_thread;- } else {- if (bits == 4) {- zp_stride = (in_vec_size_g + 1) / 2;- zps_up = zero_points + out_row_up * zp_stride;- zps_gate = zero_points + out_row_gate * zp_stride;- int g_offset = simd_lid / scale_step_per_thread;- zps_up += g_offset / 2;- zps_gate += g_offset / 2;- high_nibble = (g_offset & 1);- } else {- zp_stride = in_vec_size_g;- zps_up = zero_points + out_row_up * zp_stride +- simd_lid / scale_step_per_thread;- zps_gate = zero_points + out_row_gate * zp_stride +- simd_lid / scale_step_per_thread;- }- }-- x += tid.x * in_vec_size + simd_lid * values_per_thread;- y += tid.x * hidden_dim + out_row_up;-- for (int k = 0; k < in_vec_size; k += block_size) {- U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);-- // Compute both up and gate dot products for 4 rows each- {- auto wl_up0 = ws_up;- auto wl_up1 = ws_up + in_vec_size_w;- auto wl_up2 = ws_up + 2 * in_vec_size_w;- auto wl_up3 = ws_up + 3 * in_vec_size_w;-- auto wl_gate0 = ws_gate;- auto wl_gate1 = ws_gate + in_vec_size_w;- auto wl_gate2 = ws_gate + 2 * in_vec_size_w;- auto wl_gate3 = ws_gate + 3 * in_vec_size_w;-- U s_up0 = static_cast<U>(scales_up[0]);- U s_up1 = static_cast<U>(scales_up[in_vec_size_g]);- U s_up2 = static_cast<U>(scales_up[2 * in_vec_size_g]);- U s_up3 = static_cast<U>(scales_up[3 * in_vec_size_g]);-- U s_gate0 = static_cast<U>(scales_gate[0]);- U s_gate1 = static_cast<U>(scales_gate[in_vec_size_g]);- U s_gate2 = static_cast<U>(scales_gate[2 * in_vec_size_g]);- U s_gate3 = static_cast<U>(scales_gate[3 * in_vec_size_g]);-- if (kUseMlxQuant) {- U b_up0 = static_cast<U>(biases_up[0]);- U b_up1 = static_cast<U>(biases_up[in_vec_size_g]);- U b_up2 = static_cast<U>(biases_up[2 * in_vec_size_g]);- U b_up3 = static_cast<U>(biases_up[3 * in_vec_size_g]);-- U b_gate0 = static_cast<U>(biases_gate[0]);- U b_gate1 = static_cast<U>(biases_gate[in_vec_size_g]);- U b_gate2 = static_cast<U>(biases_gate[2 * in_vec_size_g]);- U b_gate3 = static_cast<U>(biases_gate[3 * in_vec_size_g]);-- result_up[0] += qdot<U, values_per_thread, bits>(- wl_up0,- x_thread,- s_up0,- b_up0,- sum- );- result_up[1] += qdot<U, values_per_thread, bits>(- wl_up1,- x_thread,- s_up1,- b_up1,- sum- );- result_up[2] += qdot<U, values_per_thread, bits>(- wl_up2,- x_thread,- s_up2,- b_up2,- sum- );- result_up[3] += qdot<U, values_per_thread, bits>(- wl_up3,- x_thread,- s_up3,- b_up3,- sum- );-- result_gate[0] += qdot<U, values_per_thread, bits>(- wl_gate0,- x_thread,- s_gate0,- b_gate0,- sum- );- result_gate[1] += qdot<U, values_per_thread, bits>(- wl_gate1,- x_thread,- s_gate1,- b_gate1,- sum- );- result_gate[2] += qdot<U, values_per_thread, bits>(- wl_gate2,- x_thread,- s_gate2,- b_gate2,- sum- );- result_gate[3] += qdot<U, values_per_thread, bits>(- wl_gate3,- x_thread,- s_gate3,- b_gate3,- sum- );- } else {- U zp_up0 = extract_zp<U>(zps_up, 0, zp_stride, bits, high_nibble);- U zp_up1 = extract_zp<U>(zps_up, 1, zp_stride, bits, high_nibble);- U zp_up2 = extract_zp<U>(zps_up, 2, zp_stride, bits, high_nibble);- U zp_up3 = extract_zp<U>(zps_up, 3, zp_stride, bits, high_nibble);-- U zp_gate0 = extract_zp<U>(zps_gate, 0, zp_stride, bits, high_nibble);- U zp_gate1 = extract_zp<U>(zps_gate, 1, zp_stride, bits, high_nibble);- U zp_gate2 = extract_zp<U>(zps_gate, 2, zp_stride, bits, high_nibble);- U zp_gate3 = extract_zp<U>(zps_gate, 3, zp_stride, bits, high_nibble);-- result_up[0] += qdot_zero_point<U, values_per_thread, bits>(- wl_up0,- x_thread,- s_up0,- zp_up0- );- result_up[1] += qdot_zero_point<U, values_per_thread, bits>(- wl_up1,- x_thread,- s_up1,- zp_up1- );- result_up[2] += qdot_zero_point<U, values_per_thread, bits>(- wl_up2,- x_thread,- s_up2,- zp_up2- );- result_up[3] += qdot_zero_point<U, values_per_thread, bits>(- wl_up3,- x_thread,- s_up3,- zp_up3- );-- result_gate[0] += qdot_zero_point<U, values_per_thread, bits>(- wl_gate0,- x_thread,- s_gate0,- zp_gate0- );- result_gate[1] += qdot_zero_point<U, values_per_thread, bits>(- wl_gate1,- x_thread,- s_gate1,- zp_gate1- );- result_gate[2] += qdot_zero_point<U, values_per_thread, bits>(- wl_gate2,- x_thread,- s_gate2,- zp_gate2- );- result_gate[3] += qdot_zero_point<U, values_per_thread, bits>(- wl_gate3,- x_thread,- s_gate3,- zp_gate3- );- }- }-- ws_up += block_size * bytes_per_pack / pack_factor;- ws_gate += block_size * bytes_per_pack / pack_factor;- scales_up += block_size / group_size;- scales_gate += block_size / group_size;- if (kUseMlxQuant) {- biases_up += block_size / group_size;- biases_gate += block_size / group_size;- } else {- if (bits == 4) {- zps_up += (block_size / group_size) / 2;- zps_gate += (block_size / group_size) / 2;- } else {- zps_up += block_size / group_size;- zps_gate += block_size / group_size;- }- }- x += block_size;- }-- // SIMD reduction and fused activation- for (int row = 0; row < results_per_simdgroup; row++) {- result_up[row] = simd_sum(result_up[row]);- result_gate[row] = simd_sum(result_gate[row]);- if (simd_lid == 0) {- // Apply MLP fused epilogue: up * activation(gate)- float fused = mlp_fused_epilogue_f32(result_up[row], result_gate[row]);- y[row] = static_cast<T>(fused);- }- }-}--template <typename T, int group_size, int bits>-[[kernel, max_total_threads_per_threadgroup(64)]] void qmv_mlp_fused(- const device uint32_t* w [[buffer(0)]],- const device T* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device T* x [[buffer(3)]],- device T* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- qmv_mlp_fused_impl<T, group_size, bits>(- w,- scales,- zero_points,- biases,- x,- y,- K,- hidden_dim,- tid,- simd_gid,- simd_lid- );-}--// Instantiate MLP fused QMV kernels for common configurations-template [[host_name("qmv_mlp_fused_f16_g32_b4")]] [[kernel]] void-qmv_mlp_fused<half, 32, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_mlp_fused_f16_g64_b4")]] [[kernel]] void-qmv_mlp_fused<half, 64, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_mlp_fused_f16_g128_b4")]] [[kernel]] void-qmv_mlp_fused<half, 128, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_mlp_fused_bf16_g32_b4")]] [[kernel]] void-qmv_mlp_fused<bfloat, 32, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_mlp_fused_bf16_g64_b4")]] [[kernel]] void-qmv_mlp_fused<bfloat, 64, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmv_mlp_fused_bf16_g128_b4")]] [[kernel]] void-qmv_mlp_fused<bfloat, 128, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- uint3 tid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--///////////////////////////////////////////////////////////////////////////////-/// MLP Fused QMM - Quantized Matrix-Matrix with Fused Activation-///////////////////////////////////////////////////////////////////////////////--// MLP fused QMM kernel for prefill path (M>1).-// Computes paired up and gate projections, then applies: out = up *-// activation(gate) Weight layout: [up_weights (hidden_dim cols), gate_weights-// (hidden_dim cols)] Output size is [M, hidden_dim].-template <- typename T,- int group_size,- int bits,- bool aligned_K,- int BM = 32,- int BK = 32,- int BN = 32>-void qmm_mlp_fused_impl(- const device uint32_t* w,- const device T* scales,- const device uint8_t* zero_points,- const device T* biases,- const device T* x,- device T* y,- threadgroup T* Xs,- threadgroup T* Ws_up,- const constant int& K,- const constant int&- hidden_dim, // Output columns (half of total weight columns)- const constant int& M,- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");- static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");-- (void)lid;-- constexpr int WM = 2;- constexpr int WN = 2;- constexpr int pack_factor = get_pack_factor<bits, 8>();- constexpr int bytes_per_pack = get_bytes_per_pack<bits>();-- constexpr int BK_padded = (BK + 16 / sizeof(T));- constexpr int BN_padded = (BN + 16 / sizeof(T));-- using mma_t = matmul_utils::- BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;- using loader_x_t =- matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;-- auto wl_up = (const device uint8_t*)w;-- const int y_row = tid.y * BM;- const int y_col = tid.x * BN; // Column in output (within hidden_dim)-- // Only process tiles within hidden_dim- if (y_col >= hidden_dim)- return;-- x += y_row * static_cast<int64_t>(K);-- // Up weights at column y_col- wl_up += y_col * bytes_per_pack / pack_factor;- // Gate weights at column y_col + hidden_dim- auto wl_gate = wl_up + hidden_dim * bytes_per_pack / pack_factor;-- y += y_row * static_cast<int64_t>(hidden_dim) + y_col;-- const device T* scales_base = scales;- const device T* biases_base = biases;-- const short num_els = min(BM, M - y_row);- const short num_outs = min(BN, hidden_dim - y_col);-- // Threadgroup memory for gate weights (after Ws_up)- threadgroup T* Ws_gate = Ws_up + BK * BN_padded;-- loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);- mma_t mma_up(simd_gid, simd_lid);- mma_t mma_gate(simd_gid, simd_lid);-- const int k_iterations = (K + BK - 1) / BK;- const int full_iterations = aligned_K ? k_iterations : k_iterations - 1;-- // Total output width for scale/bias strides- const int N_total = hidden_dim * 2;-- if (kUseMlxQuant) {- using loader_w_t = QuantizedBlockLoaderMlx<- T,- BK,- BN,- BN_padded,- 0,- WM * WN * 32,- group_size,- bits>;-- const device T* scales_up = scales_base + (y_col / group_size);- const device T* biases_up = biases_base + (y_col / group_size);- const device T* scales_gate =- scales_base + ((y_col + hidden_dim) / group_size);- const device T* biases_gate =- biases_base + ((y_col + hidden_dim) / group_size);-- loader_w_t loader_w_up(- wl_up,- scales_up,- biases_up,- N_total,- Ws_up,- simd_gid,- simd_lid- );- loader_w_t loader_w_gate(- wl_gate,- scales_gate,- biases_gate,- N_total,- Ws_gate,- simd_gid,- simd_lid- );-- for (int k = 0; k < full_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_unsafe();- loader_w_up.load_unsafe();- loader_w_gate.load_unsafe();- threadgroup_barrier(mem_flags::mem_threadgroup);-- mma_up.mma(Xs, Ws_up);- mma_gate.mma(Xs, Ws_gate);-- loader_x.next();- loader_w_up.next();- loader_w_gate.next();- }-- if (!aligned_K) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_safe(short2(BM, K - full_iterations * BK));- loader_w_up.load_safe(short2(K - full_iterations * BK, num_outs));- loader_w_gate.load_safe(short2(K - full_iterations * BK, num_outs));- threadgroup_barrier(mem_flags::mem_threadgroup);-- mma_up.mma(Xs, Ws_up);- mma_gate.mma(Xs, Ws_gate);- }- } else {- const int out_groups_total = (N_total + group_size - 1) / group_size;- const int groups_per_row = out_groups_total;- const int out_group_up = y_col / group_size;- const int out_group_gate = (y_col + hidden_dim) / group_size;- const int zp_stride_out =- (bits == 4) ? ((out_groups_total + 1) / 2) : out_groups_total;-- using loader_w_t = QuantizedBlockLoaderZp<- T,- BK,- BN,- BN_padded,- 0,- WM * WN * 32,- group_size,- bits,- true>;-- loader_w_t loader_w_up(- wl_up,- scales_base,- zero_points,- N_total,- groups_per_row,- Ws_up,- simd_gid,- simd_lid,- out_group_up,- out_groups_total,- zp_stride_out- );-- loader_w_t loader_w_gate(- wl_gate,- scales_base,- zero_points,- N_total,- groups_per_row,- Ws_gate,- simd_gid,- simd_lid,- out_group_gate,- out_groups_total,- zp_stride_out- );-- for (int k = 0; k < full_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_unsafe();- loader_w_up.load_unsafe();- loader_w_gate.load_unsafe();- threadgroup_barrier(mem_flags::mem_threadgroup);-- mma_up.mma(Xs, Ws_up);- mma_gate.mma(Xs, Ws_gate);-- loader_x.next();- loader_w_up.next();- loader_w_gate.next();- }-- if (!aligned_K) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_x.load_safe(short2(BM, K - full_iterations * BK));- loader_w_up.load_safe(short2(K - full_iterations * BK, num_outs));- loader_w_gate.load_safe(short2(K - full_iterations * BK, num_outs));- threadgroup_barrier(mem_flags::mem_threadgroup);-- mma_up.mma(Xs, Ws_up);- mma_gate.mma(Xs, Ws_gate);- }- }-- // Apply MLP fused epilogue: out = up * activation(gate)- // Access Ctile elements and apply fusion- constexpr int kElemsPerTile = decltype(mma_up.Ctile)::kElemsPerTile;-#pragma unroll- for (short i = 0; i < kElemsPerTile; i++) {- float up_val = static_cast<float>(mma_up.Ctile.elems()[i]);- float gate_val = static_cast<float>(mma_gate.Ctile.elems()[i]);- float fused = mlp_fused_epilogue_f32(up_val, gate_val);- mma_up.Ctile.elems()[i] = static_cast<T>(fused);- }-- // Store result- mma_up.store_result_safe(y, hidden_dim, short2(num_outs, num_els));-}--template <typename T, int group_size, int bits>-[[kernel, max_total_threads_per_threadgroup(128)]] void qmm_mlp_fused(- const device uint32_t* w [[buffer(0)]],- const device T* scales [[buffer(1)]],- const device uint8_t* zero_points- [[buffer(2), function_constant(kUseZeroPoints)]],- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],- const device T* x [[buffer(3)]],- device T* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- constexpr int BM = 32;- constexpr int BK = 32;- constexpr int BN = 32;- constexpr int BK_padded = (BK + 16 / sizeof(T));- constexpr int BN_padded = (BN + 16 / sizeof(T));-- threadgroup T Xs[BM * BK_padded];- threadgroup T Ws[2 * BK * BN_padded]; // Double for up and gate-- bool aligned_K = (K % BK) == 0;- if (aligned_K) {- qmm_mlp_fused_impl<T, group_size, bits, true, BM, BK, BN>(- w,- scales,- zero_points,- biases,- x,- y,- Xs,- Ws,- K,- hidden_dim,- M,- tid,- lid,- simd_gid,- simd_lid- );- } else {- qmm_mlp_fused_impl<T, group_size, bits, false, BM, BK, BN>(- w,- scales,- zero_points,- biases,- x,- y,- Xs,- Ws,- K,- hidden_dim,- M,- tid,- lid,- simd_gid,- simd_lid- );- }-}--// Instantiate MLP fused QMM kernels-template [[host_name("qmm_mlp_fused_f16_g32_b4")]] [[kernel]] void-qmm_mlp_fused<half, 32, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_mlp_fused_f16_g64_b4")]] [[kernel]] void-qmm_mlp_fused<half, 64, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_mlp_fused_f16_g128_b4")]] [[kernel]] void-qmm_mlp_fused<half, 128, 4>(- const device uint32_t* w [[buffer(0)]],- const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device half* biases [[buffer(2)]],- const device half* x [[buffer(3)]],- device half* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_mlp_fused_bf16_g32_b4")]] [[kernel]] void-qmm_mlp_fused<bfloat, 32, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_mlp_fused_bf16_g64_b4")]] [[kernel]] void-qmm_mlp_fused<bfloat, 64, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);--template [[host_name("qmm_mlp_fused_bf16_g128_b4")]] [[kernel]] void-qmm_mlp_fused<bfloat, 128, 4>(- const device uint32_t* w [[buffer(0)]],- const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2)]],- const device bfloat* biases [[buffer(2)]],- const device bfloat* x [[buffer(3)]],- device bfloat* y [[buffer(4)]],- const constant int& K [[buffer(5)]],- const constant int& hidden_dim [[buffer(6)]],- const constant int& M [[buffer(7)]],- uint3 tid [[threadgroup_position_in_grid]],- uint lid [[thread_index_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-);diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metalnew file mode 100644index 000000000..e69a9123c--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal@@ -0,0 +1,60 @@+#include <metal_stdlib>+#include "../definitions.metal"+#include "common.h"++template <typename T, int group_size, int bits>+VARIANTS(T, float, half, bfloat)+VARIANTS(group_size, 32, 64, 128)+VARIANTS(bits, 4, 8)+KERNEL(QuantizedMatmulQvm)(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points OPTIONAL(use_zero_points),+ const device T* biases OPTIONAL(use_mlx_quant),+ const device T* x,+ device T* y,+ const constant int& k,+ const constant int& n,+ const constant int& m,+ const bool use_zero_points SPECIALIZE,+ const bool use_mlx_quant SPECIALIZE,+ const uint tgid_x GROUPS(m),+ const uint tgid_y GROUPS((n + 64 - 1) / 64),+ const uint tgid_z GROUPS(1),+ const uint tid_x THREADS(32),+ const uint tid_y THREADS(2)+) {+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);+ const uint simd_gid = tid_y;+ const uint simd_lid = tid_x;++ if (use_mlx_quant) {+ qvm_impl<T, group_size, bits, true>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ k,+ n,+ tid,+ simd_gid,+ simd_lid+ );+ } else {+ qvm_impl<T, group_size, bits, false>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ k,+ n,+ tid,+ simd_gid,+ simd_lid+ );+ }+}diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rsindex fd3fa8e24..7f07f6282 100644--- a/crates/uzu/tests/quant_matmul_test.rs+++ b/crates/uzu/tests/quant_matmul_test.rs@@ -7,11 +7,11 @@ use objc2::{rc::Retained, runtime::ProtocolObject};use uzu::{DataType,backends::{- common::Context,- metal::{- MetalContext,- kernel::quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},+ common::{+ Context,+ kernel::matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulType},},+ metal::{MetalContext, kernel::quant_matmul::QuantizedMatmulKernel},},config::QuantizationMode,};@@ -152,7 +152,7 @@ fn cpu_reference(group_size: usize,dtype: DataType,bits: usize,- quantization_type: QuantizationType,+ quantization_type: QuantizedMatmulType,zero_points: &[u8],zero_points_stride: usize,) -> Vec<f32> {@@ -185,7 +185,7 @@ fn cpu_reference(let val_a = a[i * input_dim + l];let scale = scales[l * num_groups + group_idx];- let bias = if quantization_type == QuantizationType::ZeroPoint {+ let bias = if quantization_type == QuantizedMatmulType::ZeroPoint {let zp_val_qvm = get_zp_value(zero_points, zero_points_stride, l, group_idx, bits);-scale * zp_val_qvm} else {@@ -197,11 +197,11 @@ fn cpu_reference(for g in 0..num_groups {let scale = scales[j * num_groups + g];let bias = match quantization_type {- QuantizationType::ZeroPoint => {+ QuantizedMatmulType::ZeroPoint => {let zp = get_zp_value(zero_points, zero_points_stride, j, g, bits);-scale * zp},- QuantizationType::Mlx => biases[j * num_groups + g],+ QuantizedMatmulType::Mlx => biases[j * num_groups + g],};let l_start = g * group_size;let l_end = (l_start + group_size).min(input_dim);@@ -247,7 +247,7 @@ fn generate_test_quant_params(group_size: usize,data_type: DataType,bits: usize,- quantization_type: QuantizationType,+ quantization_type: QuantizedMatmulType,weights_transposed: bool,randomize_zp: bool,) -> TestQuantParams {@@ -281,7 +281,7 @@ fn generate_test_quant_params(};let mut zero_points = vec![0u8; zero_points_len];- if quantization_type == QuantizationType::ZeroPoint && randomize_zp {+ if quantization_type == QuantizedMatmulType::ZeroPoint && randomize_zp {if !weights_transposed {// GMM/QVM non-transposed: Use [K][N_groups] layout for generationfor k in 0..input_dim {@@ -352,7 +352,7 @@ fn generate_test_quant_params(}}- if quantization_type == QuantizationType::Mlx {+ if quantization_type == QuantizedMatmulType::Mlx {if !weights_transposed {for k in 0..input_dim {for g in 0..num_groups {@@ -421,7 +421,7 @@ fn execute_quantized_matmul(weights_transposed: bool,iterations: usize,validate: bool,- quantization_type: QuantizationType,+ quantization_type: QuantizedMatmulType,randomize_zp: bool,group_size: usize,data_type: DataType,@@ -466,28 +466,30 @@ fn execute_quantized_matmul(let s_buf = buffer_from_f32_slice(ctx, data_type, ¶ms.scales);let b_buf = match quantization_type {- QuantizationType::ZeroPoint => ctx+ QuantizedMatmulType::ZeroPoint => ctx.device.new_buffer_with_data(¶ms.zero_points, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer"),- QuantizationType::Mlx => buffer_from_f32_slice(ctx, data_type, ¶ms.biases),+ QuantizedMatmulType::Mlx => buffer_from_f32_slice(ctx, data_type, ¶ms.biases),};let x_buf = buffer_from_f32_slice(ctx, data_type, &x_f32);let y_buf = ctx.create_buffer(batch * output_dim * data_type.size_in_bytes()).expect("Failed to create buffer");let kernel = QuantizedMatmulKernel::new(&ctx,- data_type,- group_size,- input_dim,- output_dim,- match bits {- 4 => QuantizationMode::UInt4,- 8 => QuantizationMode::Int8,- _ => panic!("Unsupported bits: {}", bits),+ QuantizedMatmulConfiguration {+ data_type,+ group_size,+ input_dim,+ output_dim,+ mode: match bits {+ 4 => QuantizationMode::UInt4,+ 8 => QuantizationMode::Int8,+ _ => panic!("Unsupported bits: {}", bits),+ },+ quantization_type,+ weights_transposed,},- quantization_type,- weights_transposed,).unwrap();@@ -500,9 +502,9 @@ fn execute_quantized_matmul(scales_buffer: &s_buf,zero_points_or_biases_buffer: &b_buf,output_buffer: &y_buf,- batch: batch as i32,- input_dim: input_dim as i32,- output_dim: output_dim as i32,+ batch,+ input_dim,+ output_dim,quantization_type,};let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");@@ -523,9 +525,9 @@ fn execute_quantized_matmul(scales_buffer: &s_buf,zero_points_or_biases_buffer: &b_buf,output_buffer: &y_buf,- batch: batch as i32,- input_dim: input_dim as i32,- output_dim: output_dim as i32,+ batch,+ input_dim,+ output_dim,quantization_type,};let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");@@ -620,7 +622,7 @@ fn execute_quantized_matmul(}struct TestConfig {- quant_type: QuantizationType,+ quant_type: QuantizedMatmulType,bits: usize,data_type: DataType,group_size: usize,@@ -640,7 +642,7 @@ fn run_kernel_test(validate: bool,iterations: usize,) -> ExecutionResult {- let randomize_zp = config.quant_type == QuantizationType::ZeroPoint;+ let randomize_zp = config.quant_type == QuantizedMatmulType::ZeroPoint;execute_quantized_matmul(ctx,@@ -670,25 +672,25 @@ fn test_quant_gmv() {let configs = vec![TestConfig {- quant_type: QuantizationType::ZeroPoint,+ quant_type: QuantizedMatmulType::ZeroPoint,bits: 4,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::ZeroPoint,+ quant_type: QuantizedMatmulType::ZeroPoint,bits: 8,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::Mlx,+ quant_type: QuantizedMatmulType::Mlx,bits: 4,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::Mlx,+ quant_type: QuantizedMatmulType::Mlx,bits: 8,data_type: DataType::F32,group_size: 64,@@ -714,25 +716,25 @@ fn test_quant_qvm() {let configs = vec![TestConfig {- quant_type: QuantizationType::ZeroPoint,+ quant_type: QuantizedMatmulType::ZeroPoint,bits: 4,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::ZeroPoint,+ quant_type: QuantizedMatmulType::ZeroPoint,bits: 8,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::Mlx,+ quant_type: QuantizedMatmulType::Mlx,bits: 4,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::Mlx,+ quant_type: QuantizedMatmulType::Mlx,bits: 8,data_type: DataType::F32,group_size: 64,@@ -758,25 +760,25 @@ fn test_quant_gmm() {let configs = vec![TestConfig {- quant_type: QuantizationType::ZeroPoint,+ quant_type: QuantizedMatmulType::ZeroPoint,bits: 4,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::ZeroPoint,+ quant_type: QuantizedMatmulType::ZeroPoint,bits: 8,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::Mlx,+ quant_type: QuantizedMatmulType::Mlx,bits: 4,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::Mlx,+ quant_type: QuantizedMatmulType::Mlx,bits: 8,data_type: DataType::F32,group_size: 64,@@ -802,25 +804,25 @@ fn test_quant_gmm_transposed() {let configs = vec![TestConfig {- quant_type: QuantizationType::ZeroPoint,+ quant_type: QuantizedMatmulType::ZeroPoint,bits: 4,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::ZeroPoint,+ quant_type: QuantizedMatmulType::ZeroPoint,bits: 8,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::Mlx,+ quant_type: QuantizedMatmulType::Mlx,bits: 4,data_type: DataType::F32,group_size: 64,},TestConfig {- quant_type: QuantizationType::Mlx,+ quant_type: QuantizedMatmulType::Mlx,bits: 8,data_type: DataType::F32,group_size: 64,@@ -858,28 +860,28 @@ fn test_quant_matmul_perf() {let configs = vec![// 4-bit Mlx BF16TestConfig {- quant_type: QuantizationType::Mlx,+ quant_type: QuantizedMatmulType::Mlx,bits: 4,data_type: DataType::BF16,group_size: 128,},// 8-bit Mlx BF16TestConfig {- quant_type: QuantizationType::Mlx,+ quant_type: QuantizedMatmulType::Mlx,bits: 8,data_type: DataType::BF16,group_size: 128,},// 4-bit ZP F16TestConfig {- quant_type: QuantizationType::ZeroPoint,+ quant_type: QuantizedMatmulType::ZeroPoint,bits: 4,data_type: DataType::F16,group_size: 64,},// 8-bit ZP F16TestConfig {- quant_type: QuantizationType::ZeroPoint,+ quant_type: QuantizedMatmulType::ZeroPoint,bits: 8,data_type: DataType::F16,group_size: 64,